1ff27e85aSGilad Ben-Yossef // SPDX-License-Identifier: GPL-2.0
203963caeSGilad Ben-Yossef /* Copyright (C) 2012-2019 ARM Limited (or its affiliates). */
3ff27e85aSGilad Ben-Yossef
4ff27e85aSGilad Ben-Yossef #include <linux/kernel.h>
5ff27e85aSGilad Ben-Yossef #include <linux/module.h>
6ff27e85aSGilad Ben-Yossef #include <crypto/algapi.h>
7ff27e85aSGilad Ben-Yossef #include <crypto/internal/aead.h>
8ff27e85aSGilad Ben-Yossef #include <crypto/authenc.h>
9b66c1876SGilad Ben-Yossef #include <crypto/gcm.h>
10b66c1876SGilad Ben-Yossef #include <linux/rtnetlink.h>
1100cd6b23SArd Biesheuvel #include <crypto/internal/des.h>
12ff27e85aSGilad Ben-Yossef #include "cc_driver.h"
13ff27e85aSGilad Ben-Yossef #include "cc_buffer_mgr.h"
14ff27e85aSGilad Ben-Yossef #include "cc_aead.h"
15ff27e85aSGilad Ben-Yossef #include "cc_request_mgr.h"
16ff27e85aSGilad Ben-Yossef #include "cc_hash.h"
17ff27e85aSGilad Ben-Yossef #include "cc_sram_mgr.h"
18ff27e85aSGilad Ben-Yossef
19ff27e85aSGilad Ben-Yossef #define template_aead template_u.aead
20ff27e85aSGilad Ben-Yossef
21ff27e85aSGilad Ben-Yossef #define MAX_AEAD_SETKEY_SEQ 12
22ff27e85aSGilad Ben-Yossef #define MAX_AEAD_PROCESS_SEQ 23
23ff27e85aSGilad Ben-Yossef
24ff27e85aSGilad Ben-Yossef #define MAX_HMAC_DIGEST_SIZE (SHA256_DIGEST_SIZE)
25ff27e85aSGilad Ben-Yossef #define MAX_HMAC_BLOCK_SIZE (SHA256_BLOCK_SIZE)
26ff27e85aSGilad Ben-Yossef
27ff27e85aSGilad Ben-Yossef #define MAX_NONCE_SIZE CTR_RFC3686_NONCE_SIZE
28ff27e85aSGilad Ben-Yossef
29ff27e85aSGilad Ben-Yossef struct cc_aead_handle {
301a895f1dSGeert Uytterhoeven u32 sram_workspace_addr;
31ff27e85aSGilad Ben-Yossef struct list_head aead_list;
32ff27e85aSGilad Ben-Yossef };
33ff27e85aSGilad Ben-Yossef
34ff27e85aSGilad Ben-Yossef struct cc_hmac_s {
35ff27e85aSGilad Ben-Yossef u8 *padded_authkey;
36ff27e85aSGilad Ben-Yossef u8 *ipad_opad; /* IPAD, OPAD*/
37ff27e85aSGilad Ben-Yossef dma_addr_t padded_authkey_dma_addr;
38ff27e85aSGilad Ben-Yossef dma_addr_t ipad_opad_dma_addr;
39ff27e85aSGilad Ben-Yossef };
40ff27e85aSGilad Ben-Yossef
41ff27e85aSGilad Ben-Yossef struct cc_xcbc_s {
42ff27e85aSGilad Ben-Yossef u8 *xcbc_keys; /* K1,K2,K3 */
43ff27e85aSGilad Ben-Yossef dma_addr_t xcbc_keys_dma_addr;
44ff27e85aSGilad Ben-Yossef };
45ff27e85aSGilad Ben-Yossef
46ff27e85aSGilad Ben-Yossef struct cc_aead_ctx {
47ff27e85aSGilad Ben-Yossef struct cc_drvdata *drvdata;
48ff27e85aSGilad Ben-Yossef u8 ctr_nonce[MAX_NONCE_SIZE]; /* used for ctr3686 iv and aes ccm */
49ff27e85aSGilad Ben-Yossef u8 *enckey;
50ff27e85aSGilad Ben-Yossef dma_addr_t enckey_dma_addr;
51ff27e85aSGilad Ben-Yossef union {
52ff27e85aSGilad Ben-Yossef struct cc_hmac_s hmac;
53ff27e85aSGilad Ben-Yossef struct cc_xcbc_s xcbc;
54ff27e85aSGilad Ben-Yossef } auth_state;
55ff27e85aSGilad Ben-Yossef unsigned int enc_keylen;
56ff27e85aSGilad Ben-Yossef unsigned int auth_keylen;
57ff27e85aSGilad Ben-Yossef unsigned int authsize; /* Actual (reduced?) size of the MAC/ICv */
58f1e52fd0SYael Chemla unsigned int hash_len;
59ff27e85aSGilad Ben-Yossef enum drv_cipher_mode cipher_mode;
60ff27e85aSGilad Ben-Yossef enum cc_flow_mode flow_mode;
61ff27e85aSGilad Ben-Yossef enum drv_hash_mode auth_mode;
62ff27e85aSGilad Ben-Yossef };
63ff27e85aSGilad Ben-Yossef
cc_aead_exit(struct crypto_aead * tfm)64ff27e85aSGilad Ben-Yossef static void cc_aead_exit(struct crypto_aead *tfm)
65ff27e85aSGilad Ben-Yossef {
66ff27e85aSGilad Ben-Yossef struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
67ff27e85aSGilad Ben-Yossef struct device *dev = drvdata_to_dev(ctx->drvdata);
68ff27e85aSGilad Ben-Yossef
69ff27e85aSGilad Ben-Yossef dev_dbg(dev, "Clearing context @%p for %s\n", crypto_aead_ctx(tfm),
70ff27e85aSGilad Ben-Yossef crypto_tfm_alg_name(&tfm->base));
71ff27e85aSGilad Ben-Yossef
72ff27e85aSGilad Ben-Yossef /* Unmap enckey buffer */
73ff27e85aSGilad Ben-Yossef if (ctx->enckey) {
74ff27e85aSGilad Ben-Yossef dma_free_coherent(dev, AES_MAX_KEY_SIZE, ctx->enckey,
75ff27e85aSGilad Ben-Yossef ctx->enckey_dma_addr);
76ff27e85aSGilad Ben-Yossef dev_dbg(dev, "Freed enckey DMA buffer enckey_dma_addr=%pad\n",
77ff27e85aSGilad Ben-Yossef &ctx->enckey_dma_addr);
78ff27e85aSGilad Ben-Yossef ctx->enckey_dma_addr = 0;
79ff27e85aSGilad Ben-Yossef ctx->enckey = NULL;
80ff27e85aSGilad Ben-Yossef }
81ff27e85aSGilad Ben-Yossef
82ff27e85aSGilad Ben-Yossef if (ctx->auth_mode == DRV_HASH_XCBC_MAC) { /* XCBC authetication */
83ff27e85aSGilad Ben-Yossef struct cc_xcbc_s *xcbc = &ctx->auth_state.xcbc;
84ff27e85aSGilad Ben-Yossef
85ff27e85aSGilad Ben-Yossef if (xcbc->xcbc_keys) {
86ff27e85aSGilad Ben-Yossef dma_free_coherent(dev, CC_AES_128_BIT_KEY_SIZE * 3,
87ff27e85aSGilad Ben-Yossef xcbc->xcbc_keys,
88ff27e85aSGilad Ben-Yossef xcbc->xcbc_keys_dma_addr);
89ff27e85aSGilad Ben-Yossef }
90ff27e85aSGilad Ben-Yossef dev_dbg(dev, "Freed xcbc_keys DMA buffer xcbc_keys_dma_addr=%pad\n",
91ff27e85aSGilad Ben-Yossef &xcbc->xcbc_keys_dma_addr);
92ff27e85aSGilad Ben-Yossef xcbc->xcbc_keys_dma_addr = 0;
93ff27e85aSGilad Ben-Yossef xcbc->xcbc_keys = NULL;
94ff27e85aSGilad Ben-Yossef } else if (ctx->auth_mode != DRV_HASH_NULL) { /* HMAC auth. */
95ff27e85aSGilad Ben-Yossef struct cc_hmac_s *hmac = &ctx->auth_state.hmac;
96ff27e85aSGilad Ben-Yossef
97ff27e85aSGilad Ben-Yossef if (hmac->ipad_opad) {
98ff27e85aSGilad Ben-Yossef dma_free_coherent(dev, 2 * MAX_HMAC_DIGEST_SIZE,
99ff27e85aSGilad Ben-Yossef hmac->ipad_opad,
100ff27e85aSGilad Ben-Yossef hmac->ipad_opad_dma_addr);
101ff27e85aSGilad Ben-Yossef dev_dbg(dev, "Freed ipad_opad DMA buffer ipad_opad_dma_addr=%pad\n",
102ff27e85aSGilad Ben-Yossef &hmac->ipad_opad_dma_addr);
103ff27e85aSGilad Ben-Yossef hmac->ipad_opad_dma_addr = 0;
104ff27e85aSGilad Ben-Yossef hmac->ipad_opad = NULL;
105ff27e85aSGilad Ben-Yossef }
106ff27e85aSGilad Ben-Yossef if (hmac->padded_authkey) {
107ff27e85aSGilad Ben-Yossef dma_free_coherent(dev, MAX_HMAC_BLOCK_SIZE,
108ff27e85aSGilad Ben-Yossef hmac->padded_authkey,
109ff27e85aSGilad Ben-Yossef hmac->padded_authkey_dma_addr);
110ff27e85aSGilad Ben-Yossef dev_dbg(dev, "Freed padded_authkey DMA buffer padded_authkey_dma_addr=%pad\n",
111ff27e85aSGilad Ben-Yossef &hmac->padded_authkey_dma_addr);
112ff27e85aSGilad Ben-Yossef hmac->padded_authkey_dma_addr = 0;
113ff27e85aSGilad Ben-Yossef hmac->padded_authkey = NULL;
114ff27e85aSGilad Ben-Yossef }
115ff27e85aSGilad Ben-Yossef }
116ff27e85aSGilad Ben-Yossef }
117ff27e85aSGilad Ben-Yossef
cc_get_aead_hash_len(struct crypto_aead * tfm)118f1e52fd0SYael Chemla static unsigned int cc_get_aead_hash_len(struct crypto_aead *tfm)
119f1e52fd0SYael Chemla {
120f1e52fd0SYael Chemla struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
121f1e52fd0SYael Chemla
122f1e52fd0SYael Chemla return cc_get_default_hash_len(ctx->drvdata);
123f1e52fd0SYael Chemla }
124f1e52fd0SYael Chemla
cc_aead_init(struct crypto_aead * tfm)125ff27e85aSGilad Ben-Yossef static int cc_aead_init(struct crypto_aead *tfm)
126ff27e85aSGilad Ben-Yossef {
127ff27e85aSGilad Ben-Yossef struct aead_alg *alg = crypto_aead_alg(tfm);
128ff27e85aSGilad Ben-Yossef struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
129ff27e85aSGilad Ben-Yossef struct cc_crypto_alg *cc_alg =
130ff27e85aSGilad Ben-Yossef container_of(alg, struct cc_crypto_alg, aead_alg);
131ff27e85aSGilad Ben-Yossef struct device *dev = drvdata_to_dev(cc_alg->drvdata);
132ff27e85aSGilad Ben-Yossef
133ff27e85aSGilad Ben-Yossef dev_dbg(dev, "Initializing context @%p for %s\n", ctx,
134ff27e85aSGilad Ben-Yossef crypto_tfm_alg_name(&tfm->base));
135ff27e85aSGilad Ben-Yossef
136ff27e85aSGilad Ben-Yossef /* Initialize modes in instance */
137ff27e85aSGilad Ben-Yossef ctx->cipher_mode = cc_alg->cipher_mode;
138ff27e85aSGilad Ben-Yossef ctx->flow_mode = cc_alg->flow_mode;
139ff27e85aSGilad Ben-Yossef ctx->auth_mode = cc_alg->auth_mode;
140ff27e85aSGilad Ben-Yossef ctx->drvdata = cc_alg->drvdata;
141*07547fa7SHerbert Xu crypto_aead_set_reqsize_dma(tfm, sizeof(struct aead_req_ctx));
142ff27e85aSGilad Ben-Yossef
143ff27e85aSGilad Ben-Yossef /* Allocate key buffer, cache line aligned */
144ff27e85aSGilad Ben-Yossef ctx->enckey = dma_alloc_coherent(dev, AES_MAX_KEY_SIZE,
145ff27e85aSGilad Ben-Yossef &ctx->enckey_dma_addr, GFP_KERNEL);
146ff27e85aSGilad Ben-Yossef if (!ctx->enckey) {
147ff27e85aSGilad Ben-Yossef dev_err(dev, "Failed allocating key buffer\n");
148ff27e85aSGilad Ben-Yossef goto init_failed;
149ff27e85aSGilad Ben-Yossef }
150ff27e85aSGilad Ben-Yossef dev_dbg(dev, "Allocated enckey buffer in context ctx->enckey=@%p\n",
151ff27e85aSGilad Ben-Yossef ctx->enckey);
152ff27e85aSGilad Ben-Yossef
153ff27e85aSGilad Ben-Yossef /* Set default authlen value */
154ff27e85aSGilad Ben-Yossef
155ff27e85aSGilad Ben-Yossef if (ctx->auth_mode == DRV_HASH_XCBC_MAC) { /* XCBC authetication */
156ff27e85aSGilad Ben-Yossef struct cc_xcbc_s *xcbc = &ctx->auth_state.xcbc;
157ff27e85aSGilad Ben-Yossef const unsigned int key_size = CC_AES_128_BIT_KEY_SIZE * 3;
158ff27e85aSGilad Ben-Yossef
159ff27e85aSGilad Ben-Yossef /* Allocate dma-coherent buffer for XCBC's K1+K2+K3 */
160ff27e85aSGilad Ben-Yossef /* (and temporary for user key - up to 256b) */
161ff27e85aSGilad Ben-Yossef xcbc->xcbc_keys = dma_alloc_coherent(dev, key_size,
162ff27e85aSGilad Ben-Yossef &xcbc->xcbc_keys_dma_addr,
163ff27e85aSGilad Ben-Yossef GFP_KERNEL);
164ff27e85aSGilad Ben-Yossef if (!xcbc->xcbc_keys) {
165ff27e85aSGilad Ben-Yossef dev_err(dev, "Failed allocating buffer for XCBC keys\n");
166ff27e85aSGilad Ben-Yossef goto init_failed;
167ff27e85aSGilad Ben-Yossef }
168ff27e85aSGilad Ben-Yossef } else if (ctx->auth_mode != DRV_HASH_NULL) { /* HMAC authentication */
169ff27e85aSGilad Ben-Yossef struct cc_hmac_s *hmac = &ctx->auth_state.hmac;
170ff27e85aSGilad Ben-Yossef const unsigned int digest_size = 2 * MAX_HMAC_DIGEST_SIZE;
171ff27e85aSGilad Ben-Yossef dma_addr_t *pkey_dma = &hmac->padded_authkey_dma_addr;
172ff27e85aSGilad Ben-Yossef
173ff27e85aSGilad Ben-Yossef /* Allocate dma-coherent buffer for IPAD + OPAD */
174ff27e85aSGilad Ben-Yossef hmac->ipad_opad = dma_alloc_coherent(dev, digest_size,
175ff27e85aSGilad Ben-Yossef &hmac->ipad_opad_dma_addr,
176ff27e85aSGilad Ben-Yossef GFP_KERNEL);
177ff27e85aSGilad Ben-Yossef
178ff27e85aSGilad Ben-Yossef if (!hmac->ipad_opad) {
179ff27e85aSGilad Ben-Yossef dev_err(dev, "Failed allocating IPAD/OPAD buffer\n");
180ff27e85aSGilad Ben-Yossef goto init_failed;
181ff27e85aSGilad Ben-Yossef }
182ff27e85aSGilad Ben-Yossef
183ff27e85aSGilad Ben-Yossef dev_dbg(dev, "Allocated authkey buffer in context ctx->authkey=@%p\n",
184ff27e85aSGilad Ben-Yossef hmac->ipad_opad);
185ff27e85aSGilad Ben-Yossef
186ff27e85aSGilad Ben-Yossef hmac->padded_authkey = dma_alloc_coherent(dev,
187ff27e85aSGilad Ben-Yossef MAX_HMAC_BLOCK_SIZE,
188ff27e85aSGilad Ben-Yossef pkey_dma,
189ff27e85aSGilad Ben-Yossef GFP_KERNEL);
190ff27e85aSGilad Ben-Yossef
191ff27e85aSGilad Ben-Yossef if (!hmac->padded_authkey) {
192ff27e85aSGilad Ben-Yossef dev_err(dev, "failed to allocate padded_authkey\n");
193ff27e85aSGilad Ben-Yossef goto init_failed;
194ff27e85aSGilad Ben-Yossef }
195ff27e85aSGilad Ben-Yossef } else {
196ff27e85aSGilad Ben-Yossef ctx->auth_state.hmac.ipad_opad = NULL;
197ff27e85aSGilad Ben-Yossef ctx->auth_state.hmac.padded_authkey = NULL;
198ff27e85aSGilad Ben-Yossef }
199f1e52fd0SYael Chemla ctx->hash_len = cc_get_aead_hash_len(tfm);
200ff27e85aSGilad Ben-Yossef
201ff27e85aSGilad Ben-Yossef return 0;
202ff27e85aSGilad Ben-Yossef
203ff27e85aSGilad Ben-Yossef init_failed:
204ff27e85aSGilad Ben-Yossef cc_aead_exit(tfm);
205ff27e85aSGilad Ben-Yossef return -ENOMEM;
206ff27e85aSGilad Ben-Yossef }
207ff27e85aSGilad Ben-Yossef
cc_aead_complete(struct device * dev,void * cc_req,int err)208ff27e85aSGilad Ben-Yossef static void cc_aead_complete(struct device *dev, void *cc_req, int err)
209ff27e85aSGilad Ben-Yossef {
210ff27e85aSGilad Ben-Yossef struct aead_request *areq = (struct aead_request *)cc_req;
211*07547fa7SHerbert Xu struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(areq);
212ff27e85aSGilad Ben-Yossef struct crypto_aead *tfm = crypto_aead_reqtfm(cc_req);
213ff27e85aSGilad Ben-Yossef struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
214ff27e85aSGilad Ben-Yossef
215a108f931SGilad Ben-Yossef /* BACKLOG notification */
216a108f931SGilad Ben-Yossef if (err == -EINPROGRESS)
217a108f931SGilad Ben-Yossef goto done;
218a108f931SGilad Ben-Yossef
219ff27e85aSGilad Ben-Yossef cc_unmap_aead_request(dev, areq);
220ff27e85aSGilad Ben-Yossef
221ff27e85aSGilad Ben-Yossef /* Restore ordinary iv pointer */
222ff27e85aSGilad Ben-Yossef areq->iv = areq_ctx->backup_iv;
223ff27e85aSGilad Ben-Yossef
224ff27e85aSGilad Ben-Yossef if (err)
225ff27e85aSGilad Ben-Yossef goto done;
226ff27e85aSGilad Ben-Yossef
227ff27e85aSGilad Ben-Yossef if (areq_ctx->gen_ctx.op_type == DRV_CRYPTO_DIRECTION_DECRYPT) {
228ff27e85aSGilad Ben-Yossef if (memcmp(areq_ctx->mac_buf, areq_ctx->icv_virt_addr,
229ff27e85aSGilad Ben-Yossef ctx->authsize) != 0) {
230ff27e85aSGilad Ben-Yossef dev_dbg(dev, "Payload authentication failure, (auth-size=%d, cipher=%d)\n",
231ff27e85aSGilad Ben-Yossef ctx->authsize, ctx->cipher_mode);
232ff27e85aSGilad Ben-Yossef /* In case of payload authentication failure, MUST NOT
233ff27e85aSGilad Ben-Yossef * revealed the decrypted message --> zero its memory.
234ff27e85aSGilad Ben-Yossef */
235e88b27c8SGilad Ben-Yossef sg_zero_buffer(areq->dst, sg_nents(areq->dst),
2362a6bc713SGilad Ben-Yossef areq->cryptlen, areq->assoclen);
237ff27e85aSGilad Ben-Yossef err = -EBADMSG;
238ff27e85aSGilad Ben-Yossef }
239e6e6600cSGilad Ben-Yossef /*ENCRYPT*/
240e6e6600cSGilad Ben-Yossef } else if (areq_ctx->is_icv_fragmented) {
241ff27e85aSGilad Ben-Yossef u32 skip = areq->cryptlen + areq_ctx->dst_offset;
242ff27e85aSGilad Ben-Yossef
243e6e6600cSGilad Ben-Yossef cc_copy_sg_portion(dev, areq_ctx->mac_buf, areq_ctx->dst_sgl,
244e6e6600cSGilad Ben-Yossef skip, (skip + ctx->authsize),
245ff27e85aSGilad Ben-Yossef CC_SG_FROM_BUF);
246ff27e85aSGilad Ben-Yossef }
247ff27e85aSGilad Ben-Yossef done:
248ff27e85aSGilad Ben-Yossef aead_request_complete(areq, err);
249ff27e85aSGilad Ben-Yossef }
250ff27e85aSGilad Ben-Yossef
xcbc_setkey(struct cc_hw_desc * desc,struct cc_aead_ctx * ctx)251ff27e85aSGilad Ben-Yossef static unsigned int xcbc_setkey(struct cc_hw_desc *desc,
252ff27e85aSGilad Ben-Yossef struct cc_aead_ctx *ctx)
253ff27e85aSGilad Ben-Yossef {
254ff27e85aSGilad Ben-Yossef /* Load the AES key */
255ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[0]);
256ff27e85aSGilad Ben-Yossef /* We are using for the source/user key the same buffer
257ff27e85aSGilad Ben-Yossef * as for the output keys, * because after this key loading it
258ff27e85aSGilad Ben-Yossef * is not needed anymore
259ff27e85aSGilad Ben-Yossef */
260ff27e85aSGilad Ben-Yossef set_din_type(&desc[0], DMA_DLLI,
261ff27e85aSGilad Ben-Yossef ctx->auth_state.xcbc.xcbc_keys_dma_addr, ctx->auth_keylen,
262ff27e85aSGilad Ben-Yossef NS_BIT);
263ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[0], DRV_CIPHER_ECB);
264ff27e85aSGilad Ben-Yossef set_cipher_config0(&desc[0], DRV_CRYPTO_DIRECTION_ENCRYPT);
265ff27e85aSGilad Ben-Yossef set_key_size_aes(&desc[0], ctx->auth_keylen);
266ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[0], S_DIN_to_AES);
267ff27e85aSGilad Ben-Yossef set_setup_mode(&desc[0], SETUP_LOAD_KEY0);
268ff27e85aSGilad Ben-Yossef
269ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[1]);
270ff27e85aSGilad Ben-Yossef set_din_const(&desc[1], 0x01010101, CC_AES_128_BIT_KEY_SIZE);
271ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[1], DIN_AES_DOUT);
272ff27e85aSGilad Ben-Yossef set_dout_dlli(&desc[1], ctx->auth_state.xcbc.xcbc_keys_dma_addr,
273ff27e85aSGilad Ben-Yossef AES_KEYSIZE_128, NS_BIT, 0);
274ff27e85aSGilad Ben-Yossef
275ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[2]);
276ff27e85aSGilad Ben-Yossef set_din_const(&desc[2], 0x02020202, CC_AES_128_BIT_KEY_SIZE);
277ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[2], DIN_AES_DOUT);
278ff27e85aSGilad Ben-Yossef set_dout_dlli(&desc[2], (ctx->auth_state.xcbc.xcbc_keys_dma_addr
279ff27e85aSGilad Ben-Yossef + AES_KEYSIZE_128),
280ff27e85aSGilad Ben-Yossef AES_KEYSIZE_128, NS_BIT, 0);
281ff27e85aSGilad Ben-Yossef
282ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[3]);
283ff27e85aSGilad Ben-Yossef set_din_const(&desc[3], 0x03030303, CC_AES_128_BIT_KEY_SIZE);
284ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[3], DIN_AES_DOUT);
285ff27e85aSGilad Ben-Yossef set_dout_dlli(&desc[3], (ctx->auth_state.xcbc.xcbc_keys_dma_addr
286ff27e85aSGilad Ben-Yossef + 2 * AES_KEYSIZE_128),
287ff27e85aSGilad Ben-Yossef AES_KEYSIZE_128, NS_BIT, 0);
288ff27e85aSGilad Ben-Yossef
289ff27e85aSGilad Ben-Yossef return 4;
290ff27e85aSGilad Ben-Yossef }
291ff27e85aSGilad Ben-Yossef
hmac_setkey(struct cc_hw_desc * desc,struct cc_aead_ctx * ctx)292798ac398STian Tao static unsigned int hmac_setkey(struct cc_hw_desc *desc,
293798ac398STian Tao struct cc_aead_ctx *ctx)
294ff27e85aSGilad Ben-Yossef {
295ff27e85aSGilad Ben-Yossef unsigned int hmac_pad_const[2] = { HMAC_IPAD_CONST, HMAC_OPAD_CONST };
296ff27e85aSGilad Ben-Yossef unsigned int digest_ofs = 0;
297ff27e85aSGilad Ben-Yossef unsigned int hash_mode = (ctx->auth_mode == DRV_HASH_SHA1) ?
298ff27e85aSGilad Ben-Yossef DRV_HASH_HW_SHA1 : DRV_HASH_HW_SHA256;
299ff27e85aSGilad Ben-Yossef unsigned int digest_size = (ctx->auth_mode == DRV_HASH_SHA1) ?
300ff27e85aSGilad Ben-Yossef CC_SHA1_DIGEST_SIZE : CC_SHA256_DIGEST_SIZE;
301ff27e85aSGilad Ben-Yossef struct cc_hmac_s *hmac = &ctx->auth_state.hmac;
302ff27e85aSGilad Ben-Yossef
303ff27e85aSGilad Ben-Yossef unsigned int idx = 0;
304ff27e85aSGilad Ben-Yossef int i;
305ff27e85aSGilad Ben-Yossef
306ff27e85aSGilad Ben-Yossef /* calc derived HMAC key */
307ff27e85aSGilad Ben-Yossef for (i = 0; i < 2; i++) {
308ff27e85aSGilad Ben-Yossef /* Load hash initial state */
309ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]);
310ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], hash_mode);
311ff27e85aSGilad Ben-Yossef set_din_sram(&desc[idx],
312ff27e85aSGilad Ben-Yossef cc_larval_digest_addr(ctx->drvdata,
313ff27e85aSGilad Ben-Yossef ctx->auth_mode),
314ff27e85aSGilad Ben-Yossef digest_size);
315ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], S_DIN_to_HASH);
316ff27e85aSGilad Ben-Yossef set_setup_mode(&desc[idx], SETUP_LOAD_STATE0);
317ff27e85aSGilad Ben-Yossef idx++;
318ff27e85aSGilad Ben-Yossef
319ff27e85aSGilad Ben-Yossef /* Load the hash current length*/
320ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]);
321ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], hash_mode);
322f1e52fd0SYael Chemla set_din_const(&desc[idx], 0, ctx->hash_len);
323ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], S_DIN_to_HASH);
324ff27e85aSGilad Ben-Yossef set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
325ff27e85aSGilad Ben-Yossef idx++;
326ff27e85aSGilad Ben-Yossef
327ff27e85aSGilad Ben-Yossef /* Prepare ipad key */
328ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]);
329ff27e85aSGilad Ben-Yossef set_xor_val(&desc[idx], hmac_pad_const[i]);
330ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], hash_mode);
331ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], S_DIN_to_HASH);
332ff27e85aSGilad Ben-Yossef set_setup_mode(&desc[idx], SETUP_LOAD_STATE1);
333ff27e85aSGilad Ben-Yossef idx++;
334ff27e85aSGilad Ben-Yossef
335ff27e85aSGilad Ben-Yossef /* Perform HASH update */
336ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]);
337ff27e85aSGilad Ben-Yossef set_din_type(&desc[idx], DMA_DLLI,
338ff27e85aSGilad Ben-Yossef hmac->padded_authkey_dma_addr,
339ff27e85aSGilad Ben-Yossef SHA256_BLOCK_SIZE, NS_BIT);
340ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], hash_mode);
341ff27e85aSGilad Ben-Yossef set_xor_active(&desc[idx]);
342ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], DIN_HASH);
343ff27e85aSGilad Ben-Yossef idx++;
344ff27e85aSGilad Ben-Yossef
345ff27e85aSGilad Ben-Yossef /* Get the digset */
346ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]);
347ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], hash_mode);
348ff27e85aSGilad Ben-Yossef set_dout_dlli(&desc[idx],
349ff27e85aSGilad Ben-Yossef (hmac->ipad_opad_dma_addr + digest_ofs),
350ff27e85aSGilad Ben-Yossef digest_size, NS_BIT, 0);
351ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], S_HASH_to_DOUT);
352ff27e85aSGilad Ben-Yossef set_setup_mode(&desc[idx], SETUP_WRITE_STATE0);
353ff27e85aSGilad Ben-Yossef set_cipher_config1(&desc[idx], HASH_PADDING_DISABLED);
354ff27e85aSGilad Ben-Yossef idx++;
355ff27e85aSGilad Ben-Yossef
356ff27e85aSGilad Ben-Yossef digest_ofs += digest_size;
357ff27e85aSGilad Ben-Yossef }
358ff27e85aSGilad Ben-Yossef
359ff27e85aSGilad Ben-Yossef return idx;
360ff27e85aSGilad Ben-Yossef }
361ff27e85aSGilad Ben-Yossef
validate_keys_sizes(struct cc_aead_ctx * ctx)362ff27e85aSGilad Ben-Yossef static int validate_keys_sizes(struct cc_aead_ctx *ctx)
363ff27e85aSGilad Ben-Yossef {
364ff27e85aSGilad Ben-Yossef struct device *dev = drvdata_to_dev(ctx->drvdata);
365ff27e85aSGilad Ben-Yossef
366ff27e85aSGilad Ben-Yossef dev_dbg(dev, "enc_keylen=%u authkeylen=%u\n",
367ff27e85aSGilad Ben-Yossef ctx->enc_keylen, ctx->auth_keylen);
368ff27e85aSGilad Ben-Yossef
369ff27e85aSGilad Ben-Yossef switch (ctx->auth_mode) {
370ff27e85aSGilad Ben-Yossef case DRV_HASH_SHA1:
371ff27e85aSGilad Ben-Yossef case DRV_HASH_SHA256:
372ff27e85aSGilad Ben-Yossef break;
373ff27e85aSGilad Ben-Yossef case DRV_HASH_XCBC_MAC:
374ff27e85aSGilad Ben-Yossef if (ctx->auth_keylen != AES_KEYSIZE_128 &&
375ff27e85aSGilad Ben-Yossef ctx->auth_keylen != AES_KEYSIZE_192 &&
376ff27e85aSGilad Ben-Yossef ctx->auth_keylen != AES_KEYSIZE_256)
377ff27e85aSGilad Ben-Yossef return -ENOTSUPP;
378ff27e85aSGilad Ben-Yossef break;
379ff27e85aSGilad Ben-Yossef case DRV_HASH_NULL: /* Not authenc (e.g., CCM) - no auth_key) */
380ff27e85aSGilad Ben-Yossef if (ctx->auth_keylen > 0)
381ff27e85aSGilad Ben-Yossef return -EINVAL;
382ff27e85aSGilad Ben-Yossef break;
383ff27e85aSGilad Ben-Yossef default:
384c7b31c88SGilad Ben-Yossef dev_dbg(dev, "Invalid auth_mode=%d\n", ctx->auth_mode);
385ff27e85aSGilad Ben-Yossef return -EINVAL;
386ff27e85aSGilad Ben-Yossef }
387ff27e85aSGilad Ben-Yossef /* Check cipher key size */
388ff27e85aSGilad Ben-Yossef if (ctx->flow_mode == S_DIN_to_DES) {
389ff27e85aSGilad Ben-Yossef if (ctx->enc_keylen != DES3_EDE_KEY_SIZE) {
390c7b31c88SGilad Ben-Yossef dev_dbg(dev, "Invalid cipher(3DES) key size: %u\n",
391ff27e85aSGilad Ben-Yossef ctx->enc_keylen);
392ff27e85aSGilad Ben-Yossef return -EINVAL;
393ff27e85aSGilad Ben-Yossef }
394ff27e85aSGilad Ben-Yossef } else { /* Default assumed to be AES ciphers */
395ff27e85aSGilad Ben-Yossef if (ctx->enc_keylen != AES_KEYSIZE_128 &&
396ff27e85aSGilad Ben-Yossef ctx->enc_keylen != AES_KEYSIZE_192 &&
397ff27e85aSGilad Ben-Yossef ctx->enc_keylen != AES_KEYSIZE_256) {
398c7b31c88SGilad Ben-Yossef dev_dbg(dev, "Invalid cipher(AES) key size: %u\n",
399ff27e85aSGilad Ben-Yossef ctx->enc_keylen);
400ff27e85aSGilad Ben-Yossef return -EINVAL;
401ff27e85aSGilad Ben-Yossef }
402ff27e85aSGilad Ben-Yossef }
403ff27e85aSGilad Ben-Yossef
404ff27e85aSGilad Ben-Yossef return 0; /* All tests of keys sizes passed */
405ff27e85aSGilad Ben-Yossef }
406ff27e85aSGilad Ben-Yossef
407ff27e85aSGilad Ben-Yossef /* This function prepers the user key so it can pass to the hmac processing
408ff27e85aSGilad Ben-Yossef * (copy to intenral buffer or hash in case of key longer than block
409ff27e85aSGilad Ben-Yossef */
cc_get_plain_hmac_key(struct crypto_aead * tfm,const u8 * authkey,unsigned int keylen)410e8662a6aSGilad Ben-Yossef static int cc_get_plain_hmac_key(struct crypto_aead *tfm, const u8 *authkey,
411ff27e85aSGilad Ben-Yossef unsigned int keylen)
412ff27e85aSGilad Ben-Yossef {
413ff27e85aSGilad Ben-Yossef dma_addr_t key_dma_addr = 0;
414ff27e85aSGilad Ben-Yossef struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
415ff27e85aSGilad Ben-Yossef struct device *dev = drvdata_to_dev(ctx->drvdata);
416e431cc04SGeert Uytterhoeven u32 larval_addr;
417ff27e85aSGilad Ben-Yossef struct cc_crypto_req cc_req = {};
418ff27e85aSGilad Ben-Yossef unsigned int blocksize;
419ff27e85aSGilad Ben-Yossef unsigned int digestsize;
420ff27e85aSGilad Ben-Yossef unsigned int hashmode;
421ff27e85aSGilad Ben-Yossef unsigned int idx = 0;
422ff27e85aSGilad Ben-Yossef int rc = 0;
423e8662a6aSGilad Ben-Yossef u8 *key = NULL;
424ff27e85aSGilad Ben-Yossef struct cc_hw_desc desc[MAX_AEAD_SETKEY_SEQ];
425ff27e85aSGilad Ben-Yossef dma_addr_t padded_authkey_dma_addr =
426ff27e85aSGilad Ben-Yossef ctx->auth_state.hmac.padded_authkey_dma_addr;
427ff27e85aSGilad Ben-Yossef
428ff27e85aSGilad Ben-Yossef switch (ctx->auth_mode) { /* auth_key required and >0 */
429ff27e85aSGilad Ben-Yossef case DRV_HASH_SHA1:
430ff27e85aSGilad Ben-Yossef blocksize = SHA1_BLOCK_SIZE;
431ff27e85aSGilad Ben-Yossef digestsize = SHA1_DIGEST_SIZE;
432ff27e85aSGilad Ben-Yossef hashmode = DRV_HASH_HW_SHA1;
433ff27e85aSGilad Ben-Yossef break;
434ff27e85aSGilad Ben-Yossef case DRV_HASH_SHA256:
435ff27e85aSGilad Ben-Yossef default:
436ff27e85aSGilad Ben-Yossef blocksize = SHA256_BLOCK_SIZE;
437ff27e85aSGilad Ben-Yossef digestsize = SHA256_DIGEST_SIZE;
438ff27e85aSGilad Ben-Yossef hashmode = DRV_HASH_HW_SHA256;
439ff27e85aSGilad Ben-Yossef }
440ff27e85aSGilad Ben-Yossef
441ff27e85aSGilad Ben-Yossef if (keylen != 0) {
442e8662a6aSGilad Ben-Yossef
443e8662a6aSGilad Ben-Yossef key = kmemdup(authkey, keylen, GFP_KERNEL);
444e8662a6aSGilad Ben-Yossef if (!key)
445e8662a6aSGilad Ben-Yossef return -ENOMEM;
446e8662a6aSGilad Ben-Yossef
447f4274eecSGeert Uytterhoeven key_dma_addr = dma_map_single(dev, key, keylen, DMA_TO_DEVICE);
448ff27e85aSGilad Ben-Yossef if (dma_mapping_error(dev, key_dma_addr)) {
449ff27e85aSGilad Ben-Yossef dev_err(dev, "Mapping key va=0x%p len=%u for DMA failed\n",
450ff27e85aSGilad Ben-Yossef key, keylen);
451453431a5SWaiman Long kfree_sensitive(key);
452ff27e85aSGilad Ben-Yossef return -ENOMEM;
453ff27e85aSGilad Ben-Yossef }
454ff27e85aSGilad Ben-Yossef if (keylen > blocksize) {
455ff27e85aSGilad Ben-Yossef /* Load hash initial state */
456ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]);
457ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], hashmode);
458e431cc04SGeert Uytterhoeven larval_addr = cc_larval_digest_addr(ctx->drvdata,
459e431cc04SGeert Uytterhoeven ctx->auth_mode);
460ff27e85aSGilad Ben-Yossef set_din_sram(&desc[idx], larval_addr, digestsize);
461ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], S_DIN_to_HASH);
462ff27e85aSGilad Ben-Yossef set_setup_mode(&desc[idx], SETUP_LOAD_STATE0);
463ff27e85aSGilad Ben-Yossef idx++;
464ff27e85aSGilad Ben-Yossef
465ff27e85aSGilad Ben-Yossef /* Load the hash current length*/
466ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]);
467ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], hashmode);
468f1e52fd0SYael Chemla set_din_const(&desc[idx], 0, ctx->hash_len);
469ff27e85aSGilad Ben-Yossef set_cipher_config1(&desc[idx], HASH_PADDING_ENABLED);
470ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], S_DIN_to_HASH);
471ff27e85aSGilad Ben-Yossef set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
472ff27e85aSGilad Ben-Yossef idx++;
473ff27e85aSGilad Ben-Yossef
474ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]);
475ff27e85aSGilad Ben-Yossef set_din_type(&desc[idx], DMA_DLLI,
476ff27e85aSGilad Ben-Yossef key_dma_addr, keylen, NS_BIT);
477ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], DIN_HASH);
478ff27e85aSGilad Ben-Yossef idx++;
479ff27e85aSGilad Ben-Yossef
480ff27e85aSGilad Ben-Yossef /* Get hashed key */
481ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]);
482ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], hashmode);
483ff27e85aSGilad Ben-Yossef set_dout_dlli(&desc[idx], padded_authkey_dma_addr,
484ff27e85aSGilad Ben-Yossef digestsize, NS_BIT, 0);
485ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], S_HASH_to_DOUT);
486ff27e85aSGilad Ben-Yossef set_setup_mode(&desc[idx], SETUP_WRITE_STATE0);
487ff27e85aSGilad Ben-Yossef set_cipher_config1(&desc[idx], HASH_PADDING_DISABLED);
488ff27e85aSGilad Ben-Yossef set_cipher_config0(&desc[idx],
489ff27e85aSGilad Ben-Yossef HASH_DIGEST_RESULT_LITTLE_ENDIAN);
490ff27e85aSGilad Ben-Yossef idx++;
491ff27e85aSGilad Ben-Yossef
492ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]);
493ff27e85aSGilad Ben-Yossef set_din_const(&desc[idx], 0, (blocksize - digestsize));
494ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], BYPASS);
495ff27e85aSGilad Ben-Yossef set_dout_dlli(&desc[idx], (padded_authkey_dma_addr +
496ff27e85aSGilad Ben-Yossef digestsize), (blocksize - digestsize),
497ff27e85aSGilad Ben-Yossef NS_BIT, 0);
498ff27e85aSGilad Ben-Yossef idx++;
499ff27e85aSGilad Ben-Yossef } else {
500ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]);
501ff27e85aSGilad Ben-Yossef set_din_type(&desc[idx], DMA_DLLI, key_dma_addr,
502ff27e85aSGilad Ben-Yossef keylen, NS_BIT);
503ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], BYPASS);
504ff27e85aSGilad Ben-Yossef set_dout_dlli(&desc[idx], padded_authkey_dma_addr,
505ff27e85aSGilad Ben-Yossef keylen, NS_BIT, 0);
506ff27e85aSGilad Ben-Yossef idx++;
507ff27e85aSGilad Ben-Yossef
508ff27e85aSGilad Ben-Yossef if ((blocksize - keylen) != 0) {
509ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]);
510ff27e85aSGilad Ben-Yossef set_din_const(&desc[idx], 0,
511ff27e85aSGilad Ben-Yossef (blocksize - keylen));
512ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], BYPASS);
513ff27e85aSGilad Ben-Yossef set_dout_dlli(&desc[idx],
514ff27e85aSGilad Ben-Yossef (padded_authkey_dma_addr +
515ff27e85aSGilad Ben-Yossef keylen),
516ff27e85aSGilad Ben-Yossef (blocksize - keylen), NS_BIT, 0);
517ff27e85aSGilad Ben-Yossef idx++;
518ff27e85aSGilad Ben-Yossef }
519ff27e85aSGilad Ben-Yossef }
520ff27e85aSGilad Ben-Yossef } else {
521ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]);
522ff27e85aSGilad Ben-Yossef set_din_const(&desc[idx], 0, (blocksize - keylen));
523ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], BYPASS);
524ff27e85aSGilad Ben-Yossef set_dout_dlli(&desc[idx], padded_authkey_dma_addr,
525ff27e85aSGilad Ben-Yossef blocksize, NS_BIT, 0);
526ff27e85aSGilad Ben-Yossef idx++;
527ff27e85aSGilad Ben-Yossef }
528ff27e85aSGilad Ben-Yossef
529ff27e85aSGilad Ben-Yossef rc = cc_send_sync_request(ctx->drvdata, &cc_req, desc, idx);
530ff27e85aSGilad Ben-Yossef if (rc)
531ff27e85aSGilad Ben-Yossef dev_err(dev, "send_request() failed (rc=%d)\n", rc);
532ff27e85aSGilad Ben-Yossef
533ff27e85aSGilad Ben-Yossef if (key_dma_addr)
534ff27e85aSGilad Ben-Yossef dma_unmap_single(dev, key_dma_addr, keylen, DMA_TO_DEVICE);
535ff27e85aSGilad Ben-Yossef
536453431a5SWaiman Long kfree_sensitive(key);
537e8662a6aSGilad Ben-Yossef
538ff27e85aSGilad Ben-Yossef return rc;
539ff27e85aSGilad Ben-Yossef }
540ff27e85aSGilad Ben-Yossef
cc_aead_setkey(struct crypto_aead * tfm,const u8 * key,unsigned int keylen)541ff27e85aSGilad Ben-Yossef static int cc_aead_setkey(struct crypto_aead *tfm, const u8 *key,
542ff27e85aSGilad Ben-Yossef unsigned int keylen)
543ff27e85aSGilad Ben-Yossef {
544ff27e85aSGilad Ben-Yossef struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
545ff27e85aSGilad Ben-Yossef struct cc_crypto_req cc_req = {};
546ff27e85aSGilad Ben-Yossef struct cc_hw_desc desc[MAX_AEAD_SETKEY_SEQ];
547ff27e85aSGilad Ben-Yossef unsigned int seq_len = 0;
548ff27e85aSGilad Ben-Yossef struct device *dev = drvdata_to_dev(ctx->drvdata);
549dc95b535SEric Biggers const u8 *enckey, *authkey;
550dc95b535SEric Biggers int rc;
551ff27e85aSGilad Ben-Yossef
552ff27e85aSGilad Ben-Yossef dev_dbg(dev, "Setting key in context @%p for %s. key=%p keylen=%u\n",
553ff27e85aSGilad Ben-Yossef ctx, crypto_tfm_alg_name(crypto_aead_tfm(tfm)), key, keylen);
554ff27e85aSGilad Ben-Yossef
555ff27e85aSGilad Ben-Yossef /* STAT_PHASE_0: Init and sanity checks */
556ff27e85aSGilad Ben-Yossef
557ff27e85aSGilad Ben-Yossef if (ctx->auth_mode != DRV_HASH_NULL) { /* authenc() alg. */
558dc95b535SEric Biggers struct crypto_authenc_keys keys;
559dc95b535SEric Biggers
560dc95b535SEric Biggers rc = crypto_authenc_extractkeys(&keys, key, keylen);
561dc95b535SEric Biggers if (rc)
562674f368aSEric Biggers return rc;
563dc95b535SEric Biggers enckey = keys.enckey;
564dc95b535SEric Biggers authkey = keys.authkey;
565dc95b535SEric Biggers ctx->enc_keylen = keys.enckeylen;
566dc95b535SEric Biggers ctx->auth_keylen = keys.authkeylen;
567ff27e85aSGilad Ben-Yossef
568ff27e85aSGilad Ben-Yossef if (ctx->cipher_mode == DRV_CIPHER_CTR) {
569ff27e85aSGilad Ben-Yossef /* the nonce is stored in bytes at end of key */
570ff27e85aSGilad Ben-Yossef if (ctx->enc_keylen <
571ff27e85aSGilad Ben-Yossef (AES_MIN_KEY_SIZE + CTR_RFC3686_NONCE_SIZE))
572674f368aSEric Biggers return -EINVAL;
573ff27e85aSGilad Ben-Yossef /* Copy nonce from last 4 bytes in CTR key to
574ff27e85aSGilad Ben-Yossef * first 4 bytes in CTR IV
575ff27e85aSGilad Ben-Yossef */
576dc95b535SEric Biggers memcpy(ctx->ctr_nonce, enckey + ctx->enc_keylen -
577dc95b535SEric Biggers CTR_RFC3686_NONCE_SIZE, CTR_RFC3686_NONCE_SIZE);
578ff27e85aSGilad Ben-Yossef /* Set CTR key size */
579ff27e85aSGilad Ben-Yossef ctx->enc_keylen -= CTR_RFC3686_NONCE_SIZE;
580ff27e85aSGilad Ben-Yossef }
581ff27e85aSGilad Ben-Yossef } else { /* non-authenc - has just one key */
582dc95b535SEric Biggers enckey = key;
583dc95b535SEric Biggers authkey = NULL;
584ff27e85aSGilad Ben-Yossef ctx->enc_keylen = keylen;
585ff27e85aSGilad Ben-Yossef ctx->auth_keylen = 0;
586ff27e85aSGilad Ben-Yossef }
587ff27e85aSGilad Ben-Yossef
588ff27e85aSGilad Ben-Yossef rc = validate_keys_sizes(ctx);
589ff27e85aSGilad Ben-Yossef if (rc)
590674f368aSEric Biggers return rc;
591ff27e85aSGilad Ben-Yossef
592ff27e85aSGilad Ben-Yossef /* STAT_PHASE_1: Copy key to ctx */
593ff27e85aSGilad Ben-Yossef
594ff27e85aSGilad Ben-Yossef /* Get key material */
595dc95b535SEric Biggers memcpy(ctx->enckey, enckey, ctx->enc_keylen);
596ff27e85aSGilad Ben-Yossef if (ctx->enc_keylen == 24)
597ff27e85aSGilad Ben-Yossef memset(ctx->enckey + 24, 0, CC_AES_KEY_SIZE_MAX - 24);
598ff27e85aSGilad Ben-Yossef if (ctx->auth_mode == DRV_HASH_XCBC_MAC) {
599dc95b535SEric Biggers memcpy(ctx->auth_state.xcbc.xcbc_keys, authkey,
600dc95b535SEric Biggers ctx->auth_keylen);
601ff27e85aSGilad Ben-Yossef } else if (ctx->auth_mode != DRV_HASH_NULL) { /* HMAC */
602dc95b535SEric Biggers rc = cc_get_plain_hmac_key(tfm, authkey, ctx->auth_keylen);
603ff27e85aSGilad Ben-Yossef if (rc)
604674f368aSEric Biggers return rc;
605ff27e85aSGilad Ben-Yossef }
606ff27e85aSGilad Ben-Yossef
607ff27e85aSGilad Ben-Yossef /* STAT_PHASE_2: Create sequence */
608ff27e85aSGilad Ben-Yossef
609ff27e85aSGilad Ben-Yossef switch (ctx->auth_mode) {
610ff27e85aSGilad Ben-Yossef case DRV_HASH_SHA1:
611ff27e85aSGilad Ben-Yossef case DRV_HASH_SHA256:
612ff27e85aSGilad Ben-Yossef seq_len = hmac_setkey(desc, ctx);
613ff27e85aSGilad Ben-Yossef break;
614ff27e85aSGilad Ben-Yossef case DRV_HASH_XCBC_MAC:
615ff27e85aSGilad Ben-Yossef seq_len = xcbc_setkey(desc, ctx);
616ff27e85aSGilad Ben-Yossef break;
617ff27e85aSGilad Ben-Yossef case DRV_HASH_NULL: /* non-authenc modes, e.g., CCM */
618ff27e85aSGilad Ben-Yossef break; /* No auth. key setup */
619ff27e85aSGilad Ben-Yossef default:
620ff27e85aSGilad Ben-Yossef dev_err(dev, "Unsupported authenc (%d)\n", ctx->auth_mode);
621674f368aSEric Biggers return -ENOTSUPP;
622ff27e85aSGilad Ben-Yossef }
623ff27e85aSGilad Ben-Yossef
624ff27e85aSGilad Ben-Yossef /* STAT_PHASE_3: Submit sequence to HW */
625ff27e85aSGilad Ben-Yossef
626ff27e85aSGilad Ben-Yossef if (seq_len > 0) { /* For CCM there is no sequence to setup the key */
627ff27e85aSGilad Ben-Yossef rc = cc_send_sync_request(ctx->drvdata, &cc_req, desc, seq_len);
628ff27e85aSGilad Ben-Yossef if (rc) {
629ff27e85aSGilad Ben-Yossef dev_err(dev, "send_request() failed (rc=%d)\n", rc);
630674f368aSEric Biggers return rc;
631ff27e85aSGilad Ben-Yossef }
632ff27e85aSGilad Ben-Yossef }
633ff27e85aSGilad Ben-Yossef
634ff27e85aSGilad Ben-Yossef /* Update STAT_PHASE_3 */
635ff27e85aSGilad Ben-Yossef return rc;
636ff27e85aSGilad Ben-Yossef }
637ff27e85aSGilad Ben-Yossef
cc_des3_aead_setkey(struct crypto_aead * aead,const u8 * key,unsigned int keylen)6389fbfcefcSHerbert Xu static int cc_des3_aead_setkey(struct crypto_aead *aead, const u8 *key,
6399fbfcefcSHerbert Xu unsigned int keylen)
6409fbfcefcSHerbert Xu {
6419fbfcefcSHerbert Xu struct crypto_authenc_keys keys;
6429fbfcefcSHerbert Xu int err;
6439fbfcefcSHerbert Xu
6449fbfcefcSHerbert Xu err = crypto_authenc_extractkeys(&keys, key, keylen);
6459fbfcefcSHerbert Xu if (unlikely(err))
6469fbfcefcSHerbert Xu return err;
6479fbfcefcSHerbert Xu
64800cd6b23SArd Biesheuvel err = verify_aead_des3_key(aead, keys.enckey, keys.enckeylen) ?:
64900cd6b23SArd Biesheuvel cc_aead_setkey(aead, key, keylen);
65000cd6b23SArd Biesheuvel
65100cd6b23SArd Biesheuvel memzero_explicit(&keys, sizeof(keys));
65200cd6b23SArd Biesheuvel return err;
6539fbfcefcSHerbert Xu }
6549fbfcefcSHerbert Xu
cc_rfc4309_ccm_setkey(struct crypto_aead * tfm,const u8 * key,unsigned int keylen)655ff27e85aSGilad Ben-Yossef static int cc_rfc4309_ccm_setkey(struct crypto_aead *tfm, const u8 *key,
656ff27e85aSGilad Ben-Yossef unsigned int keylen)
657ff27e85aSGilad Ben-Yossef {
658ff27e85aSGilad Ben-Yossef struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
659ff27e85aSGilad Ben-Yossef
660ff27e85aSGilad Ben-Yossef if (keylen < 3)
661ff27e85aSGilad Ben-Yossef return -EINVAL;
662ff27e85aSGilad Ben-Yossef
663ff27e85aSGilad Ben-Yossef keylen -= 3;
664ff27e85aSGilad Ben-Yossef memcpy(ctx->ctr_nonce, key + keylen, 3);
665ff27e85aSGilad Ben-Yossef
666ff27e85aSGilad Ben-Yossef return cc_aead_setkey(tfm, key, keylen);
667ff27e85aSGilad Ben-Yossef }
668ff27e85aSGilad Ben-Yossef
cc_aead_setauthsize(struct crypto_aead * authenc,unsigned int authsize)669ff27e85aSGilad Ben-Yossef static int cc_aead_setauthsize(struct crypto_aead *authenc,
670ff27e85aSGilad Ben-Yossef unsigned int authsize)
671ff27e85aSGilad Ben-Yossef {
672ff27e85aSGilad Ben-Yossef struct cc_aead_ctx *ctx = crypto_aead_ctx(authenc);
673ff27e85aSGilad Ben-Yossef struct device *dev = drvdata_to_dev(ctx->drvdata);
674ff27e85aSGilad Ben-Yossef
675ff27e85aSGilad Ben-Yossef /* Unsupported auth. sizes */
676ff27e85aSGilad Ben-Yossef if (authsize == 0 ||
677ff27e85aSGilad Ben-Yossef authsize > crypto_aead_maxauthsize(authenc)) {
678ff27e85aSGilad Ben-Yossef return -ENOTSUPP;
679ff27e85aSGilad Ben-Yossef }
680ff27e85aSGilad Ben-Yossef
681ff27e85aSGilad Ben-Yossef ctx->authsize = authsize;
682ff27e85aSGilad Ben-Yossef dev_dbg(dev, "authlen=%d\n", ctx->authsize);
683ff27e85aSGilad Ben-Yossef
684ff27e85aSGilad Ben-Yossef return 0;
685ff27e85aSGilad Ben-Yossef }
686ff27e85aSGilad Ben-Yossef
cc_rfc4309_ccm_setauthsize(struct crypto_aead * authenc,unsigned int authsize)687ff27e85aSGilad Ben-Yossef static int cc_rfc4309_ccm_setauthsize(struct crypto_aead *authenc,
688ff27e85aSGilad Ben-Yossef unsigned int authsize)
689ff27e85aSGilad Ben-Yossef {
690ff27e85aSGilad Ben-Yossef switch (authsize) {
691ff27e85aSGilad Ben-Yossef case 8:
692ff27e85aSGilad Ben-Yossef case 12:
693ff27e85aSGilad Ben-Yossef case 16:
694ff27e85aSGilad Ben-Yossef break;
695ff27e85aSGilad Ben-Yossef default:
696ff27e85aSGilad Ben-Yossef return -EINVAL;
697ff27e85aSGilad Ben-Yossef }
698ff27e85aSGilad Ben-Yossef
699ff27e85aSGilad Ben-Yossef return cc_aead_setauthsize(authenc, authsize);
700ff27e85aSGilad Ben-Yossef }
701ff27e85aSGilad Ben-Yossef
cc_ccm_setauthsize(struct crypto_aead * authenc,unsigned int authsize)702ff27e85aSGilad Ben-Yossef static int cc_ccm_setauthsize(struct crypto_aead *authenc,
703ff27e85aSGilad Ben-Yossef unsigned int authsize)
704ff27e85aSGilad Ben-Yossef {
705ff27e85aSGilad Ben-Yossef switch (authsize) {
706ff27e85aSGilad Ben-Yossef case 4:
707ff27e85aSGilad Ben-Yossef case 6:
708ff27e85aSGilad Ben-Yossef case 8:
709ff27e85aSGilad Ben-Yossef case 10:
710ff27e85aSGilad Ben-Yossef case 12:
711ff27e85aSGilad Ben-Yossef case 14:
712ff27e85aSGilad Ben-Yossef case 16:
713ff27e85aSGilad Ben-Yossef break;
714ff27e85aSGilad Ben-Yossef default:
715ff27e85aSGilad Ben-Yossef return -EINVAL;
716ff27e85aSGilad Ben-Yossef }
717ff27e85aSGilad Ben-Yossef
718ff27e85aSGilad Ben-Yossef return cc_aead_setauthsize(authenc, authsize);
719ff27e85aSGilad Ben-Yossef }
720ff27e85aSGilad Ben-Yossef
cc_set_assoc_desc(struct aead_request * areq,unsigned int flow_mode,struct cc_hw_desc desc[],unsigned int * seq_size)721ff27e85aSGilad Ben-Yossef static void cc_set_assoc_desc(struct aead_request *areq, unsigned int flow_mode,
722ff27e85aSGilad Ben-Yossef struct cc_hw_desc desc[], unsigned int *seq_size)
723ff27e85aSGilad Ben-Yossef {
724ff27e85aSGilad Ben-Yossef struct crypto_aead *tfm = crypto_aead_reqtfm(areq);
725ff27e85aSGilad Ben-Yossef struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
726*07547fa7SHerbert Xu struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(areq);
727ff27e85aSGilad Ben-Yossef enum cc_req_dma_buf_type assoc_dma_type = areq_ctx->assoc_buff_type;
728ff27e85aSGilad Ben-Yossef unsigned int idx = *seq_size;
729ff27e85aSGilad Ben-Yossef struct device *dev = drvdata_to_dev(ctx->drvdata);
730ff27e85aSGilad Ben-Yossef
731ff27e85aSGilad Ben-Yossef switch (assoc_dma_type) {
732ff27e85aSGilad Ben-Yossef case CC_DMA_BUF_DLLI:
733ff27e85aSGilad Ben-Yossef dev_dbg(dev, "ASSOC buffer type DLLI\n");
734ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]);
735ff27e85aSGilad Ben-Yossef set_din_type(&desc[idx], DMA_DLLI, sg_dma_address(areq->src),
736da3cf67fSGilad Ben-Yossef areq_ctx->assoclen, NS_BIT);
737ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], flow_mode);
738ff27e85aSGilad Ben-Yossef if (ctx->auth_mode == DRV_HASH_XCBC_MAC &&
739ff27e85aSGilad Ben-Yossef areq_ctx->cryptlen > 0)
740ff27e85aSGilad Ben-Yossef set_din_not_last_indication(&desc[idx]);
741ff27e85aSGilad Ben-Yossef break;
742ff27e85aSGilad Ben-Yossef case CC_DMA_BUF_MLLI:
743ff27e85aSGilad Ben-Yossef dev_dbg(dev, "ASSOC buffer type MLLI\n");
744ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]);
745ff27e85aSGilad Ben-Yossef set_din_type(&desc[idx], DMA_MLLI, areq_ctx->assoc.sram_addr,
746ff27e85aSGilad Ben-Yossef areq_ctx->assoc.mlli_nents, NS_BIT);
747ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], flow_mode);
748ff27e85aSGilad Ben-Yossef if (ctx->auth_mode == DRV_HASH_XCBC_MAC &&
749ff27e85aSGilad Ben-Yossef areq_ctx->cryptlen > 0)
750ff27e85aSGilad Ben-Yossef set_din_not_last_indication(&desc[idx]);
751ff27e85aSGilad Ben-Yossef break;
752ff27e85aSGilad Ben-Yossef case CC_DMA_BUF_NULL:
753ff27e85aSGilad Ben-Yossef default:
754ff27e85aSGilad Ben-Yossef dev_err(dev, "Invalid ASSOC buffer type\n");
755ff27e85aSGilad Ben-Yossef }
756ff27e85aSGilad Ben-Yossef
757ff27e85aSGilad Ben-Yossef *seq_size = (++idx);
758ff27e85aSGilad Ben-Yossef }
759ff27e85aSGilad Ben-Yossef
cc_proc_authen_desc(struct aead_request * areq,unsigned int flow_mode,struct cc_hw_desc desc[],unsigned int * seq_size,int direct)760ff27e85aSGilad Ben-Yossef static void cc_proc_authen_desc(struct aead_request *areq,
761ff27e85aSGilad Ben-Yossef unsigned int flow_mode,
762ff27e85aSGilad Ben-Yossef struct cc_hw_desc desc[],
763ff27e85aSGilad Ben-Yossef unsigned int *seq_size, int direct)
764ff27e85aSGilad Ben-Yossef {
765*07547fa7SHerbert Xu struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(areq);
766ff27e85aSGilad Ben-Yossef enum cc_req_dma_buf_type data_dma_type = areq_ctx->data_buff_type;
767ff27e85aSGilad Ben-Yossef unsigned int idx = *seq_size;
768ff27e85aSGilad Ben-Yossef struct crypto_aead *tfm = crypto_aead_reqtfm(areq);
769ff27e85aSGilad Ben-Yossef struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
770ff27e85aSGilad Ben-Yossef struct device *dev = drvdata_to_dev(ctx->drvdata);
771ff27e85aSGilad Ben-Yossef
772ff27e85aSGilad Ben-Yossef switch (data_dma_type) {
773ff27e85aSGilad Ben-Yossef case CC_DMA_BUF_DLLI:
774ff27e85aSGilad Ben-Yossef {
775ff27e85aSGilad Ben-Yossef struct scatterlist *cipher =
776ff27e85aSGilad Ben-Yossef (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) ?
777ff27e85aSGilad Ben-Yossef areq_ctx->dst_sgl : areq_ctx->src_sgl;
778ff27e85aSGilad Ben-Yossef
779ff27e85aSGilad Ben-Yossef unsigned int offset =
780ff27e85aSGilad Ben-Yossef (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) ?
781ff27e85aSGilad Ben-Yossef areq_ctx->dst_offset : areq_ctx->src_offset;
782ff27e85aSGilad Ben-Yossef dev_dbg(dev, "AUTHENC: SRC/DST buffer type DLLI\n");
783ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]);
784ff27e85aSGilad Ben-Yossef set_din_type(&desc[idx], DMA_DLLI,
785ff27e85aSGilad Ben-Yossef (sg_dma_address(cipher) + offset),
786ff27e85aSGilad Ben-Yossef areq_ctx->cryptlen, NS_BIT);
787ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], flow_mode);
788ff27e85aSGilad Ben-Yossef break;
789ff27e85aSGilad Ben-Yossef }
790ff27e85aSGilad Ben-Yossef case CC_DMA_BUF_MLLI:
791ff27e85aSGilad Ben-Yossef {
792ff27e85aSGilad Ben-Yossef /* DOUBLE-PASS flow (as default)
793ff27e85aSGilad Ben-Yossef * assoc. + iv + data -compact in one table
794ff27e85aSGilad Ben-Yossef * if assoclen is ZERO only IV perform
795ff27e85aSGilad Ben-Yossef */
7961a895f1dSGeert Uytterhoeven u32 mlli_addr = areq_ctx->assoc.sram_addr;
797ff27e85aSGilad Ben-Yossef u32 mlli_nents = areq_ctx->assoc.mlli_nents;
798ff27e85aSGilad Ben-Yossef
799ff27e85aSGilad Ben-Yossef if (areq_ctx->is_single_pass) {
800ff27e85aSGilad Ben-Yossef if (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) {
801ff27e85aSGilad Ben-Yossef mlli_addr = areq_ctx->dst.sram_addr;
802ff27e85aSGilad Ben-Yossef mlli_nents = areq_ctx->dst.mlli_nents;
803ff27e85aSGilad Ben-Yossef } else {
804ff27e85aSGilad Ben-Yossef mlli_addr = areq_ctx->src.sram_addr;
805ff27e85aSGilad Ben-Yossef mlli_nents = areq_ctx->src.mlli_nents;
806ff27e85aSGilad Ben-Yossef }
807ff27e85aSGilad Ben-Yossef }
808ff27e85aSGilad Ben-Yossef
809ff27e85aSGilad Ben-Yossef dev_dbg(dev, "AUTHENC: SRC/DST buffer type MLLI\n");
810ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]);
811ff27e85aSGilad Ben-Yossef set_din_type(&desc[idx], DMA_MLLI, mlli_addr, mlli_nents,
812ff27e85aSGilad Ben-Yossef NS_BIT);
813ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], flow_mode);
814ff27e85aSGilad Ben-Yossef break;
815ff27e85aSGilad Ben-Yossef }
816ff27e85aSGilad Ben-Yossef case CC_DMA_BUF_NULL:
817ff27e85aSGilad Ben-Yossef default:
818ff27e85aSGilad Ben-Yossef dev_err(dev, "AUTHENC: Invalid SRC/DST buffer type\n");
819ff27e85aSGilad Ben-Yossef }
820ff27e85aSGilad Ben-Yossef
821ff27e85aSGilad Ben-Yossef *seq_size = (++idx);
822ff27e85aSGilad Ben-Yossef }
823ff27e85aSGilad Ben-Yossef
cc_proc_cipher_desc(struct aead_request * areq,unsigned int flow_mode,struct cc_hw_desc desc[],unsigned int * seq_size)824ff27e85aSGilad Ben-Yossef static void cc_proc_cipher_desc(struct aead_request *areq,
825ff27e85aSGilad Ben-Yossef unsigned int flow_mode,
826ff27e85aSGilad Ben-Yossef struct cc_hw_desc desc[],
827ff27e85aSGilad Ben-Yossef unsigned int *seq_size)
828ff27e85aSGilad Ben-Yossef {
829ff27e85aSGilad Ben-Yossef unsigned int idx = *seq_size;
830*07547fa7SHerbert Xu struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(areq);
831ff27e85aSGilad Ben-Yossef enum cc_req_dma_buf_type data_dma_type = areq_ctx->data_buff_type;
832ff27e85aSGilad Ben-Yossef struct crypto_aead *tfm = crypto_aead_reqtfm(areq);
833ff27e85aSGilad Ben-Yossef struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
834ff27e85aSGilad Ben-Yossef struct device *dev = drvdata_to_dev(ctx->drvdata);
835ff27e85aSGilad Ben-Yossef
836ff27e85aSGilad Ben-Yossef if (areq_ctx->cryptlen == 0)
837ff27e85aSGilad Ben-Yossef return; /*null processing*/
838ff27e85aSGilad Ben-Yossef
839ff27e85aSGilad Ben-Yossef switch (data_dma_type) {
840ff27e85aSGilad Ben-Yossef case CC_DMA_BUF_DLLI:
841ff27e85aSGilad Ben-Yossef dev_dbg(dev, "CIPHER: SRC/DST buffer type DLLI\n");
842ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]);
843ff27e85aSGilad Ben-Yossef set_din_type(&desc[idx], DMA_DLLI,
844ff27e85aSGilad Ben-Yossef (sg_dma_address(areq_ctx->src_sgl) +
845ff27e85aSGilad Ben-Yossef areq_ctx->src_offset), areq_ctx->cryptlen,
846ff27e85aSGilad Ben-Yossef NS_BIT);
847ff27e85aSGilad Ben-Yossef set_dout_dlli(&desc[idx],
848ff27e85aSGilad Ben-Yossef (sg_dma_address(areq_ctx->dst_sgl) +
849ff27e85aSGilad Ben-Yossef areq_ctx->dst_offset),
850ff27e85aSGilad Ben-Yossef areq_ctx->cryptlen, NS_BIT, 0);
851ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], flow_mode);
852ff27e85aSGilad Ben-Yossef break;
853ff27e85aSGilad Ben-Yossef case CC_DMA_BUF_MLLI:
854ff27e85aSGilad Ben-Yossef dev_dbg(dev, "CIPHER: SRC/DST buffer type MLLI\n");
855ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]);
856ff27e85aSGilad Ben-Yossef set_din_type(&desc[idx], DMA_MLLI, areq_ctx->src.sram_addr,
857ff27e85aSGilad Ben-Yossef areq_ctx->src.mlli_nents, NS_BIT);
858ff27e85aSGilad Ben-Yossef set_dout_mlli(&desc[idx], areq_ctx->dst.sram_addr,
859ff27e85aSGilad Ben-Yossef areq_ctx->dst.mlli_nents, NS_BIT, 0);
860ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], flow_mode);
861ff27e85aSGilad Ben-Yossef break;
862ff27e85aSGilad Ben-Yossef case CC_DMA_BUF_NULL:
863ff27e85aSGilad Ben-Yossef default:
864ff27e85aSGilad Ben-Yossef dev_err(dev, "CIPHER: Invalid SRC/DST buffer type\n");
865ff27e85aSGilad Ben-Yossef }
866ff27e85aSGilad Ben-Yossef
867ff27e85aSGilad Ben-Yossef *seq_size = (++idx);
868ff27e85aSGilad Ben-Yossef }
869ff27e85aSGilad Ben-Yossef
cc_proc_digest_desc(struct aead_request * req,struct cc_hw_desc desc[],unsigned int * seq_size)870ff27e85aSGilad Ben-Yossef static void cc_proc_digest_desc(struct aead_request *req,
871ff27e85aSGilad Ben-Yossef struct cc_hw_desc desc[],
872ff27e85aSGilad Ben-Yossef unsigned int *seq_size)
873ff27e85aSGilad Ben-Yossef {
874ff27e85aSGilad Ben-Yossef struct crypto_aead *tfm = crypto_aead_reqtfm(req);
875ff27e85aSGilad Ben-Yossef struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
876*07547fa7SHerbert Xu struct aead_req_ctx *req_ctx = aead_request_ctx_dma(req);
877ff27e85aSGilad Ben-Yossef unsigned int idx = *seq_size;
878ff27e85aSGilad Ben-Yossef unsigned int hash_mode = (ctx->auth_mode == DRV_HASH_SHA1) ?
879ff27e85aSGilad Ben-Yossef DRV_HASH_HW_SHA1 : DRV_HASH_HW_SHA256;
880ff27e85aSGilad Ben-Yossef int direct = req_ctx->gen_ctx.op_type;
881ff27e85aSGilad Ben-Yossef
882ff27e85aSGilad Ben-Yossef /* Get final ICV result */
883ff27e85aSGilad Ben-Yossef if (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) {
884ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]);
885ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], S_HASH_to_DOUT);
886ff27e85aSGilad Ben-Yossef set_setup_mode(&desc[idx], SETUP_WRITE_STATE0);
887ff27e85aSGilad Ben-Yossef set_dout_dlli(&desc[idx], req_ctx->icv_dma_addr, ctx->authsize,
888ff27e85aSGilad Ben-Yossef NS_BIT, 1);
88927b3b22dSGilad Ben-Yossef set_queue_last_ind(ctx->drvdata, &desc[idx]);
890ff27e85aSGilad Ben-Yossef if (ctx->auth_mode == DRV_HASH_XCBC_MAC) {
891ff27e85aSGilad Ben-Yossef set_aes_not_hash_mode(&desc[idx]);
892ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC);
893ff27e85aSGilad Ben-Yossef } else {
894ff27e85aSGilad Ben-Yossef set_cipher_config0(&desc[idx],
895ff27e85aSGilad Ben-Yossef HASH_DIGEST_RESULT_LITTLE_ENDIAN);
896ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], hash_mode);
897ff27e85aSGilad Ben-Yossef }
898ff27e85aSGilad Ben-Yossef } else { /*Decrypt*/
899ff27e85aSGilad Ben-Yossef /* Get ICV out from hardware */
900ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]);
901ff27e85aSGilad Ben-Yossef set_setup_mode(&desc[idx], SETUP_WRITE_STATE0);
902ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], S_HASH_to_DOUT);
903ff27e85aSGilad Ben-Yossef set_dout_dlli(&desc[idx], req_ctx->mac_buf_dma_addr,
904ff27e85aSGilad Ben-Yossef ctx->authsize, NS_BIT, 1);
90527b3b22dSGilad Ben-Yossef set_queue_last_ind(ctx->drvdata, &desc[idx]);
906ff27e85aSGilad Ben-Yossef set_cipher_config0(&desc[idx],
907ff27e85aSGilad Ben-Yossef HASH_DIGEST_RESULT_LITTLE_ENDIAN);
908ff27e85aSGilad Ben-Yossef set_cipher_config1(&desc[idx], HASH_PADDING_DISABLED);
909ff27e85aSGilad Ben-Yossef if (ctx->auth_mode == DRV_HASH_XCBC_MAC) {
910ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC);
911ff27e85aSGilad Ben-Yossef set_aes_not_hash_mode(&desc[idx]);
912ff27e85aSGilad Ben-Yossef } else {
913ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], hash_mode);
914ff27e85aSGilad Ben-Yossef }
915ff27e85aSGilad Ben-Yossef }
916ff27e85aSGilad Ben-Yossef
917ff27e85aSGilad Ben-Yossef *seq_size = (++idx);
918ff27e85aSGilad Ben-Yossef }
919ff27e85aSGilad Ben-Yossef
cc_set_cipher_desc(struct aead_request * req,struct cc_hw_desc desc[],unsigned int * seq_size)920ff27e85aSGilad Ben-Yossef static void cc_set_cipher_desc(struct aead_request *req,
921ff27e85aSGilad Ben-Yossef struct cc_hw_desc desc[],
922ff27e85aSGilad Ben-Yossef unsigned int *seq_size)
923ff27e85aSGilad Ben-Yossef {
924ff27e85aSGilad Ben-Yossef struct crypto_aead *tfm = crypto_aead_reqtfm(req);
925ff27e85aSGilad Ben-Yossef struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
926*07547fa7SHerbert Xu struct aead_req_ctx *req_ctx = aead_request_ctx_dma(req);
927ff27e85aSGilad Ben-Yossef unsigned int hw_iv_size = req_ctx->hw_iv_size;
928ff27e85aSGilad Ben-Yossef unsigned int idx = *seq_size;
929ff27e85aSGilad Ben-Yossef int direct = req_ctx->gen_ctx.op_type;
930ff27e85aSGilad Ben-Yossef
931ff27e85aSGilad Ben-Yossef /* Setup cipher state */
932ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]);
933ff27e85aSGilad Ben-Yossef set_cipher_config0(&desc[idx], direct);
934ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], ctx->flow_mode);
935ff27e85aSGilad Ben-Yossef set_din_type(&desc[idx], DMA_DLLI, req_ctx->gen_ctx.iv_dma_addr,
936ff27e85aSGilad Ben-Yossef hw_iv_size, NS_BIT);
937ff27e85aSGilad Ben-Yossef if (ctx->cipher_mode == DRV_CIPHER_CTR)
938ff27e85aSGilad Ben-Yossef set_setup_mode(&desc[idx], SETUP_LOAD_STATE1);
939ff27e85aSGilad Ben-Yossef else
940ff27e85aSGilad Ben-Yossef set_setup_mode(&desc[idx], SETUP_LOAD_STATE0);
941ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], ctx->cipher_mode);
942ff27e85aSGilad Ben-Yossef idx++;
943ff27e85aSGilad Ben-Yossef
944ff27e85aSGilad Ben-Yossef /* Setup enc. key */
945ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]);
946ff27e85aSGilad Ben-Yossef set_cipher_config0(&desc[idx], direct);
947ff27e85aSGilad Ben-Yossef set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
948ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], ctx->flow_mode);
949ff27e85aSGilad Ben-Yossef if (ctx->flow_mode == S_DIN_to_AES) {
950ff27e85aSGilad Ben-Yossef set_din_type(&desc[idx], DMA_DLLI, ctx->enckey_dma_addr,
951ff27e85aSGilad Ben-Yossef ((ctx->enc_keylen == 24) ? CC_AES_KEY_SIZE_MAX :
952ff27e85aSGilad Ben-Yossef ctx->enc_keylen), NS_BIT);
953ff27e85aSGilad Ben-Yossef set_key_size_aes(&desc[idx], ctx->enc_keylen);
954ff27e85aSGilad Ben-Yossef } else {
955ff27e85aSGilad Ben-Yossef set_din_type(&desc[idx], DMA_DLLI, ctx->enckey_dma_addr,
956ff27e85aSGilad Ben-Yossef ctx->enc_keylen, NS_BIT);
957ff27e85aSGilad Ben-Yossef set_key_size_des(&desc[idx], ctx->enc_keylen);
958ff27e85aSGilad Ben-Yossef }
959ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], ctx->cipher_mode);
960ff27e85aSGilad Ben-Yossef idx++;
961ff27e85aSGilad Ben-Yossef
962ff27e85aSGilad Ben-Yossef *seq_size = idx;
963ff27e85aSGilad Ben-Yossef }
964ff27e85aSGilad Ben-Yossef
cc_proc_cipher(struct aead_request * req,struct cc_hw_desc desc[],unsigned int * seq_size,unsigned int data_flow_mode)965ff27e85aSGilad Ben-Yossef static void cc_proc_cipher(struct aead_request *req, struct cc_hw_desc desc[],
966ff27e85aSGilad Ben-Yossef unsigned int *seq_size, unsigned int data_flow_mode)
967ff27e85aSGilad Ben-Yossef {
968*07547fa7SHerbert Xu struct aead_req_ctx *req_ctx = aead_request_ctx_dma(req);
969ff27e85aSGilad Ben-Yossef int direct = req_ctx->gen_ctx.op_type;
970ff27e85aSGilad Ben-Yossef unsigned int idx = *seq_size;
971ff27e85aSGilad Ben-Yossef
972ff27e85aSGilad Ben-Yossef if (req_ctx->cryptlen == 0)
973ff27e85aSGilad Ben-Yossef return; /*null processing*/
974ff27e85aSGilad Ben-Yossef
975ff27e85aSGilad Ben-Yossef cc_set_cipher_desc(req, desc, &idx);
976ff27e85aSGilad Ben-Yossef cc_proc_cipher_desc(req, data_flow_mode, desc, &idx);
977ff27e85aSGilad Ben-Yossef if (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) {
978ff27e85aSGilad Ben-Yossef /* We must wait for DMA to write all cipher */
979ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]);
980ff27e85aSGilad Ben-Yossef set_din_no_dma(&desc[idx], 0, 0xfffff0);
981ff27e85aSGilad Ben-Yossef set_dout_no_dma(&desc[idx], 0, 0, 1);
982ff27e85aSGilad Ben-Yossef idx++;
983ff27e85aSGilad Ben-Yossef }
984ff27e85aSGilad Ben-Yossef
985ff27e85aSGilad Ben-Yossef *seq_size = idx;
986ff27e85aSGilad Ben-Yossef }
987ff27e85aSGilad Ben-Yossef
cc_set_hmac_desc(struct aead_request * req,struct cc_hw_desc desc[],unsigned int * seq_size)988ff27e85aSGilad Ben-Yossef static void cc_set_hmac_desc(struct aead_request *req, struct cc_hw_desc desc[],
989ff27e85aSGilad Ben-Yossef unsigned int *seq_size)
990ff27e85aSGilad Ben-Yossef {
991ff27e85aSGilad Ben-Yossef struct crypto_aead *tfm = crypto_aead_reqtfm(req);
992ff27e85aSGilad Ben-Yossef struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
993ff27e85aSGilad Ben-Yossef unsigned int hash_mode = (ctx->auth_mode == DRV_HASH_SHA1) ?
994ff27e85aSGilad Ben-Yossef DRV_HASH_HW_SHA1 : DRV_HASH_HW_SHA256;
995ff27e85aSGilad Ben-Yossef unsigned int digest_size = (ctx->auth_mode == DRV_HASH_SHA1) ?
996ff27e85aSGilad Ben-Yossef CC_SHA1_DIGEST_SIZE : CC_SHA256_DIGEST_SIZE;
997ff27e85aSGilad Ben-Yossef unsigned int idx = *seq_size;
998ff27e85aSGilad Ben-Yossef
999ff27e85aSGilad Ben-Yossef /* Loading hash ipad xor key state */
1000ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]);
1001ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], hash_mode);
1002ff27e85aSGilad Ben-Yossef set_din_type(&desc[idx], DMA_DLLI,
1003ff27e85aSGilad Ben-Yossef ctx->auth_state.hmac.ipad_opad_dma_addr, digest_size,
1004ff27e85aSGilad Ben-Yossef NS_BIT);
1005ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], S_DIN_to_HASH);
1006ff27e85aSGilad Ben-Yossef set_setup_mode(&desc[idx], SETUP_LOAD_STATE0);
1007ff27e85aSGilad Ben-Yossef idx++;
1008ff27e85aSGilad Ben-Yossef
1009ff27e85aSGilad Ben-Yossef /* Load init. digest len (64 bytes) */
1010ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]);
1011ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], hash_mode);
1012ff27e85aSGilad Ben-Yossef set_din_sram(&desc[idx], cc_digest_len_addr(ctx->drvdata, hash_mode),
1013f1e52fd0SYael Chemla ctx->hash_len);
1014ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], S_DIN_to_HASH);
1015ff27e85aSGilad Ben-Yossef set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
1016ff27e85aSGilad Ben-Yossef idx++;
1017ff27e85aSGilad Ben-Yossef
1018ff27e85aSGilad Ben-Yossef *seq_size = idx;
1019ff27e85aSGilad Ben-Yossef }
1020ff27e85aSGilad Ben-Yossef
cc_set_xcbc_desc(struct aead_request * req,struct cc_hw_desc desc[],unsigned int * seq_size)1021ff27e85aSGilad Ben-Yossef static void cc_set_xcbc_desc(struct aead_request *req, struct cc_hw_desc desc[],
1022ff27e85aSGilad Ben-Yossef unsigned int *seq_size)
1023ff27e85aSGilad Ben-Yossef {
1024ff27e85aSGilad Ben-Yossef struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1025ff27e85aSGilad Ben-Yossef struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
1026ff27e85aSGilad Ben-Yossef unsigned int idx = *seq_size;
1027ff27e85aSGilad Ben-Yossef
1028ff27e85aSGilad Ben-Yossef /* Loading MAC state */
1029ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]);
1030ff27e85aSGilad Ben-Yossef set_din_const(&desc[idx], 0, CC_AES_BLOCK_SIZE);
1031ff27e85aSGilad Ben-Yossef set_setup_mode(&desc[idx], SETUP_LOAD_STATE0);
1032ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC);
1033ff27e85aSGilad Ben-Yossef set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT);
1034ff27e85aSGilad Ben-Yossef set_key_size_aes(&desc[idx], CC_AES_128_BIT_KEY_SIZE);
1035ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], S_DIN_to_HASH);
1036ff27e85aSGilad Ben-Yossef set_aes_not_hash_mode(&desc[idx]);
1037ff27e85aSGilad Ben-Yossef idx++;
1038ff27e85aSGilad Ben-Yossef
1039ff27e85aSGilad Ben-Yossef /* Setup XCBC MAC K1 */
1040ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]);
1041ff27e85aSGilad Ben-Yossef set_din_type(&desc[idx], DMA_DLLI,
1042ff27e85aSGilad Ben-Yossef ctx->auth_state.xcbc.xcbc_keys_dma_addr,
1043ff27e85aSGilad Ben-Yossef AES_KEYSIZE_128, NS_BIT);
1044ff27e85aSGilad Ben-Yossef set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
1045ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC);
1046ff27e85aSGilad Ben-Yossef set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT);
1047ff27e85aSGilad Ben-Yossef set_key_size_aes(&desc[idx], CC_AES_128_BIT_KEY_SIZE);
1048ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], S_DIN_to_HASH);
1049ff27e85aSGilad Ben-Yossef set_aes_not_hash_mode(&desc[idx]);
1050ff27e85aSGilad Ben-Yossef idx++;
1051ff27e85aSGilad Ben-Yossef
1052ff27e85aSGilad Ben-Yossef /* Setup XCBC MAC K2 */
1053ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]);
1054ff27e85aSGilad Ben-Yossef set_din_type(&desc[idx], DMA_DLLI,
1055ff27e85aSGilad Ben-Yossef (ctx->auth_state.xcbc.xcbc_keys_dma_addr +
1056ff27e85aSGilad Ben-Yossef AES_KEYSIZE_128), AES_KEYSIZE_128, NS_BIT);
1057ff27e85aSGilad Ben-Yossef set_setup_mode(&desc[idx], SETUP_LOAD_STATE1);
1058ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC);
1059ff27e85aSGilad Ben-Yossef set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT);
1060ff27e85aSGilad Ben-Yossef set_key_size_aes(&desc[idx], CC_AES_128_BIT_KEY_SIZE);
1061ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], S_DIN_to_HASH);
1062ff27e85aSGilad Ben-Yossef set_aes_not_hash_mode(&desc[idx]);
1063ff27e85aSGilad Ben-Yossef idx++;
1064ff27e85aSGilad Ben-Yossef
1065ff27e85aSGilad Ben-Yossef /* Setup XCBC MAC K3 */
1066ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]);
1067ff27e85aSGilad Ben-Yossef set_din_type(&desc[idx], DMA_DLLI,
1068ff27e85aSGilad Ben-Yossef (ctx->auth_state.xcbc.xcbc_keys_dma_addr +
1069ff27e85aSGilad Ben-Yossef 2 * AES_KEYSIZE_128), AES_KEYSIZE_128, NS_BIT);
1070ff27e85aSGilad Ben-Yossef set_setup_mode(&desc[idx], SETUP_LOAD_STATE2);
1071ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC);
1072ff27e85aSGilad Ben-Yossef set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT);
1073ff27e85aSGilad Ben-Yossef set_key_size_aes(&desc[idx], CC_AES_128_BIT_KEY_SIZE);
1074ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], S_DIN_to_HASH);
1075ff27e85aSGilad Ben-Yossef set_aes_not_hash_mode(&desc[idx]);
1076ff27e85aSGilad Ben-Yossef idx++;
1077ff27e85aSGilad Ben-Yossef
1078ff27e85aSGilad Ben-Yossef *seq_size = idx;
1079ff27e85aSGilad Ben-Yossef }
1080ff27e85aSGilad Ben-Yossef
cc_proc_header_desc(struct aead_request * req,struct cc_hw_desc desc[],unsigned int * seq_size)1081ff27e85aSGilad Ben-Yossef static void cc_proc_header_desc(struct aead_request *req,
1082ff27e85aSGilad Ben-Yossef struct cc_hw_desc desc[],
1083ff27e85aSGilad Ben-Yossef unsigned int *seq_size)
1084ff27e85aSGilad Ben-Yossef {
1085*07547fa7SHerbert Xu struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(req);
1086ff27e85aSGilad Ben-Yossef unsigned int idx = *seq_size;
1087da3cf67fSGilad Ben-Yossef
1088ff27e85aSGilad Ben-Yossef /* Hash associated data */
1089da3cf67fSGilad Ben-Yossef if (areq_ctx->assoclen > 0)
1090ff27e85aSGilad Ben-Yossef cc_set_assoc_desc(req, DIN_HASH, desc, &idx);
1091ff27e85aSGilad Ben-Yossef
1092ff27e85aSGilad Ben-Yossef /* Hash IV */
1093ff27e85aSGilad Ben-Yossef *seq_size = idx;
1094ff27e85aSGilad Ben-Yossef }
1095ff27e85aSGilad Ben-Yossef
cc_proc_scheme_desc(struct aead_request * req,struct cc_hw_desc desc[],unsigned int * seq_size)1096ff27e85aSGilad Ben-Yossef static void cc_proc_scheme_desc(struct aead_request *req,
1097ff27e85aSGilad Ben-Yossef struct cc_hw_desc desc[],
1098ff27e85aSGilad Ben-Yossef unsigned int *seq_size)
1099ff27e85aSGilad Ben-Yossef {
1100ff27e85aSGilad Ben-Yossef struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1101ff27e85aSGilad Ben-Yossef struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
1102ff27e85aSGilad Ben-Yossef struct cc_aead_handle *aead_handle = ctx->drvdata->aead_handle;
1103ff27e85aSGilad Ben-Yossef unsigned int hash_mode = (ctx->auth_mode == DRV_HASH_SHA1) ?
1104ff27e85aSGilad Ben-Yossef DRV_HASH_HW_SHA1 : DRV_HASH_HW_SHA256;
1105ff27e85aSGilad Ben-Yossef unsigned int digest_size = (ctx->auth_mode == DRV_HASH_SHA1) ?
1106ff27e85aSGilad Ben-Yossef CC_SHA1_DIGEST_SIZE : CC_SHA256_DIGEST_SIZE;
1107ff27e85aSGilad Ben-Yossef unsigned int idx = *seq_size;
1108ff27e85aSGilad Ben-Yossef
1109ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]);
1110ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], hash_mode);
1111ff27e85aSGilad Ben-Yossef set_dout_sram(&desc[idx], aead_handle->sram_workspace_addr,
1112f1e52fd0SYael Chemla ctx->hash_len);
1113ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], S_HASH_to_DOUT);
1114ff27e85aSGilad Ben-Yossef set_setup_mode(&desc[idx], SETUP_WRITE_STATE1);
1115ff27e85aSGilad Ben-Yossef set_cipher_do(&desc[idx], DO_PAD);
1116ff27e85aSGilad Ben-Yossef idx++;
1117ff27e85aSGilad Ben-Yossef
1118ff27e85aSGilad Ben-Yossef /* Get final ICV result */
1119ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]);
1120ff27e85aSGilad Ben-Yossef set_dout_sram(&desc[idx], aead_handle->sram_workspace_addr,
1121ff27e85aSGilad Ben-Yossef digest_size);
1122ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], S_HASH_to_DOUT);
1123ff27e85aSGilad Ben-Yossef set_setup_mode(&desc[idx], SETUP_WRITE_STATE0);
1124ff27e85aSGilad Ben-Yossef set_cipher_config0(&desc[idx], HASH_DIGEST_RESULT_LITTLE_ENDIAN);
1125ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], hash_mode);
1126ff27e85aSGilad Ben-Yossef idx++;
1127ff27e85aSGilad Ben-Yossef
1128ff27e85aSGilad Ben-Yossef /* Loading hash opad xor key state */
1129ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]);
1130ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], hash_mode);
1131ff27e85aSGilad Ben-Yossef set_din_type(&desc[idx], DMA_DLLI,
1132ff27e85aSGilad Ben-Yossef (ctx->auth_state.hmac.ipad_opad_dma_addr + digest_size),
1133ff27e85aSGilad Ben-Yossef digest_size, NS_BIT);
1134ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], S_DIN_to_HASH);
1135ff27e85aSGilad Ben-Yossef set_setup_mode(&desc[idx], SETUP_LOAD_STATE0);
1136ff27e85aSGilad Ben-Yossef idx++;
1137ff27e85aSGilad Ben-Yossef
1138ff27e85aSGilad Ben-Yossef /* Load init. digest len (64 bytes) */
1139ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]);
1140ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], hash_mode);
1141ff27e85aSGilad Ben-Yossef set_din_sram(&desc[idx], cc_digest_len_addr(ctx->drvdata, hash_mode),
1142f1e52fd0SYael Chemla ctx->hash_len);
1143ff27e85aSGilad Ben-Yossef set_cipher_config1(&desc[idx], HASH_PADDING_ENABLED);
1144ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], S_DIN_to_HASH);
1145ff27e85aSGilad Ben-Yossef set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
1146ff27e85aSGilad Ben-Yossef idx++;
1147ff27e85aSGilad Ben-Yossef
1148ff27e85aSGilad Ben-Yossef /* Perform HASH update */
1149ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]);
1150ff27e85aSGilad Ben-Yossef set_din_sram(&desc[idx], aead_handle->sram_workspace_addr,
1151ff27e85aSGilad Ben-Yossef digest_size);
1152ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], DIN_HASH);
1153ff27e85aSGilad Ben-Yossef idx++;
1154ff27e85aSGilad Ben-Yossef
1155ff27e85aSGilad Ben-Yossef *seq_size = idx;
1156ff27e85aSGilad Ben-Yossef }
1157ff27e85aSGilad Ben-Yossef
cc_mlli_to_sram(struct aead_request * req,struct cc_hw_desc desc[],unsigned int * seq_size)1158ff27e85aSGilad Ben-Yossef static void cc_mlli_to_sram(struct aead_request *req,
1159ff27e85aSGilad Ben-Yossef struct cc_hw_desc desc[], unsigned int *seq_size)
1160ff27e85aSGilad Ben-Yossef {
1161*07547fa7SHerbert Xu struct aead_req_ctx *req_ctx = aead_request_ctx_dma(req);
1162ff27e85aSGilad Ben-Yossef struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1163ff27e85aSGilad Ben-Yossef struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
1164ff27e85aSGilad Ben-Yossef struct device *dev = drvdata_to_dev(ctx->drvdata);
1165ff27e85aSGilad Ben-Yossef
1166d2d34fb5SGilad Ben-Yossef if ((req_ctx->assoc_buff_type == CC_DMA_BUF_MLLI ||
1167ff27e85aSGilad Ben-Yossef req_ctx->data_buff_type == CC_DMA_BUF_MLLI ||
1168d2d34fb5SGilad Ben-Yossef !req_ctx->is_single_pass) && req_ctx->mlli_params.mlli_len) {
1169ff27e85aSGilad Ben-Yossef dev_dbg(dev, "Copy-to-sram: mlli_dma=%08x, mlli_size=%u\n",
11701a895f1dSGeert Uytterhoeven ctx->drvdata->mlli_sram_addr,
1171ff27e85aSGilad Ben-Yossef req_ctx->mlli_params.mlli_len);
1172ff27e85aSGilad Ben-Yossef /* Copy MLLI table host-to-sram */
1173ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[*seq_size]);
1174ff27e85aSGilad Ben-Yossef set_din_type(&desc[*seq_size], DMA_DLLI,
1175ff27e85aSGilad Ben-Yossef req_ctx->mlli_params.mlli_dma_addr,
1176ff27e85aSGilad Ben-Yossef req_ctx->mlli_params.mlli_len, NS_BIT);
1177ff27e85aSGilad Ben-Yossef set_dout_sram(&desc[*seq_size],
1178ff27e85aSGilad Ben-Yossef ctx->drvdata->mlli_sram_addr,
1179ff27e85aSGilad Ben-Yossef req_ctx->mlli_params.mlli_len);
1180ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[*seq_size], BYPASS);
1181ff27e85aSGilad Ben-Yossef (*seq_size)++;
1182ff27e85aSGilad Ben-Yossef }
1183ff27e85aSGilad Ben-Yossef }
1184ff27e85aSGilad Ben-Yossef
cc_get_data_flow(enum drv_crypto_direction direct,enum cc_flow_mode setup_flow_mode,bool is_single_pass)1185ff27e85aSGilad Ben-Yossef static enum cc_flow_mode cc_get_data_flow(enum drv_crypto_direction direct,
1186ff27e85aSGilad Ben-Yossef enum cc_flow_mode setup_flow_mode,
1187ff27e85aSGilad Ben-Yossef bool is_single_pass)
1188ff27e85aSGilad Ben-Yossef {
1189ff27e85aSGilad Ben-Yossef enum cc_flow_mode data_flow_mode;
1190ff27e85aSGilad Ben-Yossef
1191ff27e85aSGilad Ben-Yossef if (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) {
1192ff27e85aSGilad Ben-Yossef if (setup_flow_mode == S_DIN_to_AES)
1193ff27e85aSGilad Ben-Yossef data_flow_mode = is_single_pass ?
1194ff27e85aSGilad Ben-Yossef AES_to_HASH_and_DOUT : DIN_AES_DOUT;
1195ff27e85aSGilad Ben-Yossef else
1196ff27e85aSGilad Ben-Yossef data_flow_mode = is_single_pass ?
1197ff27e85aSGilad Ben-Yossef DES_to_HASH_and_DOUT : DIN_DES_DOUT;
1198ff27e85aSGilad Ben-Yossef } else { /* Decrypt */
1199ff27e85aSGilad Ben-Yossef if (setup_flow_mode == S_DIN_to_AES)
1200ff27e85aSGilad Ben-Yossef data_flow_mode = is_single_pass ?
1201ff27e85aSGilad Ben-Yossef AES_and_HASH : DIN_AES_DOUT;
1202ff27e85aSGilad Ben-Yossef else
1203ff27e85aSGilad Ben-Yossef data_flow_mode = is_single_pass ?
1204ff27e85aSGilad Ben-Yossef DES_and_HASH : DIN_DES_DOUT;
1205ff27e85aSGilad Ben-Yossef }
1206ff27e85aSGilad Ben-Yossef
1207ff27e85aSGilad Ben-Yossef return data_flow_mode;
1208ff27e85aSGilad Ben-Yossef }
1209ff27e85aSGilad Ben-Yossef
cc_hmac_authenc(struct aead_request * req,struct cc_hw_desc desc[],unsigned int * seq_size)1210ff27e85aSGilad Ben-Yossef static void cc_hmac_authenc(struct aead_request *req, struct cc_hw_desc desc[],
1211ff27e85aSGilad Ben-Yossef unsigned int *seq_size)
1212ff27e85aSGilad Ben-Yossef {
1213ff27e85aSGilad Ben-Yossef struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1214ff27e85aSGilad Ben-Yossef struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
1215*07547fa7SHerbert Xu struct aead_req_ctx *req_ctx = aead_request_ctx_dma(req);
1216ff27e85aSGilad Ben-Yossef int direct = req_ctx->gen_ctx.op_type;
1217ff27e85aSGilad Ben-Yossef unsigned int data_flow_mode =
1218ff27e85aSGilad Ben-Yossef cc_get_data_flow(direct, ctx->flow_mode,
1219ff27e85aSGilad Ben-Yossef req_ctx->is_single_pass);
1220ff27e85aSGilad Ben-Yossef
1221ff27e85aSGilad Ben-Yossef if (req_ctx->is_single_pass) {
122292816ab6SGeert Uytterhoeven /*
1223ff27e85aSGilad Ben-Yossef * Single-pass flow
1224ff27e85aSGilad Ben-Yossef */
1225ff27e85aSGilad Ben-Yossef cc_set_hmac_desc(req, desc, seq_size);
1226ff27e85aSGilad Ben-Yossef cc_set_cipher_desc(req, desc, seq_size);
1227ff27e85aSGilad Ben-Yossef cc_proc_header_desc(req, desc, seq_size);
1228ff27e85aSGilad Ben-Yossef cc_proc_cipher_desc(req, data_flow_mode, desc, seq_size);
1229ff27e85aSGilad Ben-Yossef cc_proc_scheme_desc(req, desc, seq_size);
1230ff27e85aSGilad Ben-Yossef cc_proc_digest_desc(req, desc, seq_size);
1231ff27e85aSGilad Ben-Yossef return;
1232ff27e85aSGilad Ben-Yossef }
1233ff27e85aSGilad Ben-Yossef
123492816ab6SGeert Uytterhoeven /*
1235ff27e85aSGilad Ben-Yossef * Double-pass flow
1236ff27e85aSGilad Ben-Yossef * Fallback for unsupported single-pass modes,
1237ff27e85aSGilad Ben-Yossef * i.e. using assoc. data of non-word-multiple
1238ff27e85aSGilad Ben-Yossef */
1239ff27e85aSGilad Ben-Yossef if (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) {
1240ff27e85aSGilad Ben-Yossef /* encrypt first.. */
1241ff27e85aSGilad Ben-Yossef cc_proc_cipher(req, desc, seq_size, data_flow_mode);
1242ff27e85aSGilad Ben-Yossef /* authenc after..*/
1243ff27e85aSGilad Ben-Yossef cc_set_hmac_desc(req, desc, seq_size);
1244ff27e85aSGilad Ben-Yossef cc_proc_authen_desc(req, DIN_HASH, desc, seq_size, direct);
1245ff27e85aSGilad Ben-Yossef cc_proc_scheme_desc(req, desc, seq_size);
1246ff27e85aSGilad Ben-Yossef cc_proc_digest_desc(req, desc, seq_size);
1247ff27e85aSGilad Ben-Yossef
1248ff27e85aSGilad Ben-Yossef } else { /*DECRYPT*/
1249ff27e85aSGilad Ben-Yossef /* authenc first..*/
1250ff27e85aSGilad Ben-Yossef cc_set_hmac_desc(req, desc, seq_size);
1251ff27e85aSGilad Ben-Yossef cc_proc_authen_desc(req, DIN_HASH, desc, seq_size, direct);
1252ff27e85aSGilad Ben-Yossef cc_proc_scheme_desc(req, desc, seq_size);
1253ff27e85aSGilad Ben-Yossef /* decrypt after.. */
1254ff27e85aSGilad Ben-Yossef cc_proc_cipher(req, desc, seq_size, data_flow_mode);
1255ff27e85aSGilad Ben-Yossef /* read the digest result with setting the completion bit
1256ff27e85aSGilad Ben-Yossef * must be after the cipher operation
1257ff27e85aSGilad Ben-Yossef */
1258ff27e85aSGilad Ben-Yossef cc_proc_digest_desc(req, desc, seq_size);
1259ff27e85aSGilad Ben-Yossef }
1260ff27e85aSGilad Ben-Yossef }
1261ff27e85aSGilad Ben-Yossef
1262ff27e85aSGilad Ben-Yossef static void
cc_xcbc_authenc(struct aead_request * req,struct cc_hw_desc desc[],unsigned int * seq_size)1263ff27e85aSGilad Ben-Yossef cc_xcbc_authenc(struct aead_request *req, struct cc_hw_desc desc[],
1264ff27e85aSGilad Ben-Yossef unsigned int *seq_size)
1265ff27e85aSGilad Ben-Yossef {
1266ff27e85aSGilad Ben-Yossef struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1267ff27e85aSGilad Ben-Yossef struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
1268*07547fa7SHerbert Xu struct aead_req_ctx *req_ctx = aead_request_ctx_dma(req);
1269ff27e85aSGilad Ben-Yossef int direct = req_ctx->gen_ctx.op_type;
1270ff27e85aSGilad Ben-Yossef unsigned int data_flow_mode =
1271ff27e85aSGilad Ben-Yossef cc_get_data_flow(direct, ctx->flow_mode,
1272ff27e85aSGilad Ben-Yossef req_ctx->is_single_pass);
1273ff27e85aSGilad Ben-Yossef
1274ff27e85aSGilad Ben-Yossef if (req_ctx->is_single_pass) {
127592816ab6SGeert Uytterhoeven /*
1276ff27e85aSGilad Ben-Yossef * Single-pass flow
1277ff27e85aSGilad Ben-Yossef */
1278ff27e85aSGilad Ben-Yossef cc_set_xcbc_desc(req, desc, seq_size);
1279ff27e85aSGilad Ben-Yossef cc_set_cipher_desc(req, desc, seq_size);
1280ff27e85aSGilad Ben-Yossef cc_proc_header_desc(req, desc, seq_size);
1281ff27e85aSGilad Ben-Yossef cc_proc_cipher_desc(req, data_flow_mode, desc, seq_size);
1282ff27e85aSGilad Ben-Yossef cc_proc_digest_desc(req, desc, seq_size);
1283ff27e85aSGilad Ben-Yossef return;
1284ff27e85aSGilad Ben-Yossef }
1285ff27e85aSGilad Ben-Yossef
128692816ab6SGeert Uytterhoeven /*
1287ff27e85aSGilad Ben-Yossef * Double-pass flow
1288ff27e85aSGilad Ben-Yossef * Fallback for unsupported single-pass modes,
1289ff27e85aSGilad Ben-Yossef * i.e. using assoc. data of non-word-multiple
1290ff27e85aSGilad Ben-Yossef */
1291ff27e85aSGilad Ben-Yossef if (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) {
1292ff27e85aSGilad Ben-Yossef /* encrypt first.. */
1293ff27e85aSGilad Ben-Yossef cc_proc_cipher(req, desc, seq_size, data_flow_mode);
1294ff27e85aSGilad Ben-Yossef /* authenc after.. */
1295ff27e85aSGilad Ben-Yossef cc_set_xcbc_desc(req, desc, seq_size);
1296ff27e85aSGilad Ben-Yossef cc_proc_authen_desc(req, DIN_HASH, desc, seq_size, direct);
1297ff27e85aSGilad Ben-Yossef cc_proc_digest_desc(req, desc, seq_size);
1298ff27e85aSGilad Ben-Yossef } else { /*DECRYPT*/
1299ff27e85aSGilad Ben-Yossef /* authenc first.. */
1300ff27e85aSGilad Ben-Yossef cc_set_xcbc_desc(req, desc, seq_size);
1301ff27e85aSGilad Ben-Yossef cc_proc_authen_desc(req, DIN_HASH, desc, seq_size, direct);
1302ff27e85aSGilad Ben-Yossef /* decrypt after..*/
1303ff27e85aSGilad Ben-Yossef cc_proc_cipher(req, desc, seq_size, data_flow_mode);
1304ff27e85aSGilad Ben-Yossef /* read the digest result with setting the completion bit
1305ff27e85aSGilad Ben-Yossef * must be after the cipher operation
1306ff27e85aSGilad Ben-Yossef */
1307ff27e85aSGilad Ben-Yossef cc_proc_digest_desc(req, desc, seq_size);
1308ff27e85aSGilad Ben-Yossef }
1309ff27e85aSGilad Ben-Yossef }
1310ff27e85aSGilad Ben-Yossef
validate_data_size(struct cc_aead_ctx * ctx,enum drv_crypto_direction direct,struct aead_request * req)1311ff27e85aSGilad Ben-Yossef static int validate_data_size(struct cc_aead_ctx *ctx,
1312ff27e85aSGilad Ben-Yossef enum drv_crypto_direction direct,
1313ff27e85aSGilad Ben-Yossef struct aead_request *req)
1314ff27e85aSGilad Ben-Yossef {
1315*07547fa7SHerbert Xu struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(req);
1316ff27e85aSGilad Ben-Yossef struct device *dev = drvdata_to_dev(ctx->drvdata);
1317da3cf67fSGilad Ben-Yossef unsigned int assoclen = areq_ctx->assoclen;
1318ff27e85aSGilad Ben-Yossef unsigned int cipherlen = (direct == DRV_CRYPTO_DIRECTION_DECRYPT) ?
1319ff27e85aSGilad Ben-Yossef (req->cryptlen - ctx->authsize) : req->cryptlen;
1320ff27e85aSGilad Ben-Yossef
1321ff27e85aSGilad Ben-Yossef if (direct == DRV_CRYPTO_DIRECTION_DECRYPT &&
1322ff27e85aSGilad Ben-Yossef req->cryptlen < ctx->authsize)
1323ff27e85aSGilad Ben-Yossef goto data_size_err;
1324ff27e85aSGilad Ben-Yossef
1325ff27e85aSGilad Ben-Yossef areq_ctx->is_single_pass = true; /*defaulted to fast flow*/
1326ff27e85aSGilad Ben-Yossef
1327ff27e85aSGilad Ben-Yossef switch (ctx->flow_mode) {
1328ff27e85aSGilad Ben-Yossef case S_DIN_to_AES:
1329ff27e85aSGilad Ben-Yossef if (ctx->cipher_mode == DRV_CIPHER_CBC &&
1330ff27e85aSGilad Ben-Yossef !IS_ALIGNED(cipherlen, AES_BLOCK_SIZE))
1331ff27e85aSGilad Ben-Yossef goto data_size_err;
1332ff27e85aSGilad Ben-Yossef if (ctx->cipher_mode == DRV_CIPHER_CCM)
1333ff27e85aSGilad Ben-Yossef break;
1334ff27e85aSGilad Ben-Yossef if (ctx->cipher_mode == DRV_CIPHER_GCTR) {
1335ff27e85aSGilad Ben-Yossef if (areq_ctx->plaintext_authenticate_only)
1336ff27e85aSGilad Ben-Yossef areq_ctx->is_single_pass = false;
1337ff27e85aSGilad Ben-Yossef break;
1338ff27e85aSGilad Ben-Yossef }
1339ff27e85aSGilad Ben-Yossef
1340ff27e85aSGilad Ben-Yossef if (!IS_ALIGNED(assoclen, sizeof(u32)))
1341ff27e85aSGilad Ben-Yossef areq_ctx->is_single_pass = false;
1342ff27e85aSGilad Ben-Yossef
1343ff27e85aSGilad Ben-Yossef if (ctx->cipher_mode == DRV_CIPHER_CTR &&
1344ff27e85aSGilad Ben-Yossef !IS_ALIGNED(cipherlen, sizeof(u32)))
1345ff27e85aSGilad Ben-Yossef areq_ctx->is_single_pass = false;
1346ff27e85aSGilad Ben-Yossef
1347ff27e85aSGilad Ben-Yossef break;
1348ff27e85aSGilad Ben-Yossef case S_DIN_to_DES:
1349ff27e85aSGilad Ben-Yossef if (!IS_ALIGNED(cipherlen, DES_BLOCK_SIZE))
1350ff27e85aSGilad Ben-Yossef goto data_size_err;
1351ff27e85aSGilad Ben-Yossef if (!IS_ALIGNED(assoclen, DES_BLOCK_SIZE))
1352ff27e85aSGilad Ben-Yossef areq_ctx->is_single_pass = false;
1353ff27e85aSGilad Ben-Yossef break;
1354ff27e85aSGilad Ben-Yossef default:
1355ff27e85aSGilad Ben-Yossef dev_err(dev, "Unexpected flow mode (%d)\n", ctx->flow_mode);
1356ff27e85aSGilad Ben-Yossef goto data_size_err;
1357ff27e85aSGilad Ben-Yossef }
1358ff27e85aSGilad Ben-Yossef
1359ff27e85aSGilad Ben-Yossef return 0;
1360ff27e85aSGilad Ben-Yossef
1361ff27e85aSGilad Ben-Yossef data_size_err:
1362ff27e85aSGilad Ben-Yossef return -EINVAL;
1363ff27e85aSGilad Ben-Yossef }
1364ff27e85aSGilad Ben-Yossef
format_ccm_a0(u8 * pa0_buff,u32 header_size)1365ff27e85aSGilad Ben-Yossef static unsigned int format_ccm_a0(u8 *pa0_buff, u32 header_size)
1366ff27e85aSGilad Ben-Yossef {
1367ff27e85aSGilad Ben-Yossef unsigned int len = 0;
1368ff27e85aSGilad Ben-Yossef
1369ff27e85aSGilad Ben-Yossef if (header_size == 0)
1370ff27e85aSGilad Ben-Yossef return 0;
1371ff27e85aSGilad Ben-Yossef
1372ff27e85aSGilad Ben-Yossef if (header_size < ((1UL << 16) - (1UL << 8))) {
1373ff27e85aSGilad Ben-Yossef len = 2;
1374ff27e85aSGilad Ben-Yossef
1375ff27e85aSGilad Ben-Yossef pa0_buff[0] = (header_size >> 8) & 0xFF;
1376ff27e85aSGilad Ben-Yossef pa0_buff[1] = header_size & 0xFF;
1377ff27e85aSGilad Ben-Yossef } else {
1378ff27e85aSGilad Ben-Yossef len = 6;
1379ff27e85aSGilad Ben-Yossef
1380ff27e85aSGilad Ben-Yossef pa0_buff[0] = 0xFF;
1381ff27e85aSGilad Ben-Yossef pa0_buff[1] = 0xFE;
1382ff27e85aSGilad Ben-Yossef pa0_buff[2] = (header_size >> 24) & 0xFF;
1383ff27e85aSGilad Ben-Yossef pa0_buff[3] = (header_size >> 16) & 0xFF;
1384ff27e85aSGilad Ben-Yossef pa0_buff[4] = (header_size >> 8) & 0xFF;
1385ff27e85aSGilad Ben-Yossef pa0_buff[5] = header_size & 0xFF;
1386ff27e85aSGilad Ben-Yossef }
1387ff27e85aSGilad Ben-Yossef
1388ff27e85aSGilad Ben-Yossef return len;
1389ff27e85aSGilad Ben-Yossef }
1390ff27e85aSGilad Ben-Yossef
set_msg_len(u8 * block,unsigned int msglen,unsigned int csize)1391ff27e85aSGilad Ben-Yossef static int set_msg_len(u8 *block, unsigned int msglen, unsigned int csize)
1392ff27e85aSGilad Ben-Yossef {
1393ff27e85aSGilad Ben-Yossef __be32 data;
1394ff27e85aSGilad Ben-Yossef
1395ff27e85aSGilad Ben-Yossef memset(block, 0, csize);
1396ff27e85aSGilad Ben-Yossef block += csize;
1397ff27e85aSGilad Ben-Yossef
1398ff27e85aSGilad Ben-Yossef if (csize >= 4)
1399ff27e85aSGilad Ben-Yossef csize = 4;
1400ff27e85aSGilad Ben-Yossef else if (msglen > (1 << (8 * csize)))
1401ff27e85aSGilad Ben-Yossef return -EOVERFLOW;
1402ff27e85aSGilad Ben-Yossef
1403ff27e85aSGilad Ben-Yossef data = cpu_to_be32(msglen);
1404ff27e85aSGilad Ben-Yossef memcpy(block - csize, (u8 *)&data + 4 - csize, csize);
1405ff27e85aSGilad Ben-Yossef
1406ff27e85aSGilad Ben-Yossef return 0;
1407ff27e85aSGilad Ben-Yossef }
1408ff27e85aSGilad Ben-Yossef
cc_ccm(struct aead_request * req,struct cc_hw_desc desc[],unsigned int * seq_size)1409ff27e85aSGilad Ben-Yossef static int cc_ccm(struct aead_request *req, struct cc_hw_desc desc[],
1410ff27e85aSGilad Ben-Yossef unsigned int *seq_size)
1411ff27e85aSGilad Ben-Yossef {
1412ff27e85aSGilad Ben-Yossef struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1413ff27e85aSGilad Ben-Yossef struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
1414*07547fa7SHerbert Xu struct aead_req_ctx *req_ctx = aead_request_ctx_dma(req);
1415ff27e85aSGilad Ben-Yossef unsigned int idx = *seq_size;
1416ff27e85aSGilad Ben-Yossef unsigned int cipher_flow_mode;
1417ff27e85aSGilad Ben-Yossef dma_addr_t mac_result;
1418ff27e85aSGilad Ben-Yossef
1419ff27e85aSGilad Ben-Yossef if (req_ctx->gen_ctx.op_type == DRV_CRYPTO_DIRECTION_DECRYPT) {
1420ff27e85aSGilad Ben-Yossef cipher_flow_mode = AES_to_HASH_and_DOUT;
1421ff27e85aSGilad Ben-Yossef mac_result = req_ctx->mac_buf_dma_addr;
1422ff27e85aSGilad Ben-Yossef } else { /* Encrypt */
1423ff27e85aSGilad Ben-Yossef cipher_flow_mode = AES_and_HASH;
1424ff27e85aSGilad Ben-Yossef mac_result = req_ctx->icv_dma_addr;
1425ff27e85aSGilad Ben-Yossef }
1426ff27e85aSGilad Ben-Yossef
1427ff27e85aSGilad Ben-Yossef /* load key */
1428ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]);
1429ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], DRV_CIPHER_CTR);
1430ff27e85aSGilad Ben-Yossef set_din_type(&desc[idx], DMA_DLLI, ctx->enckey_dma_addr,
1431ff27e85aSGilad Ben-Yossef ((ctx->enc_keylen == 24) ? CC_AES_KEY_SIZE_MAX :
1432ff27e85aSGilad Ben-Yossef ctx->enc_keylen), NS_BIT);
1433ff27e85aSGilad Ben-Yossef set_key_size_aes(&desc[idx], ctx->enc_keylen);
1434ff27e85aSGilad Ben-Yossef set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
1435ff27e85aSGilad Ben-Yossef set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT);
1436ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], S_DIN_to_AES);
1437ff27e85aSGilad Ben-Yossef idx++;
1438ff27e85aSGilad Ben-Yossef
1439ff27e85aSGilad Ben-Yossef /* load ctr state */
1440ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]);
1441ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], DRV_CIPHER_CTR);
1442ff27e85aSGilad Ben-Yossef set_key_size_aes(&desc[idx], ctx->enc_keylen);
1443ff27e85aSGilad Ben-Yossef set_din_type(&desc[idx], DMA_DLLI,
1444ff27e85aSGilad Ben-Yossef req_ctx->gen_ctx.iv_dma_addr, AES_BLOCK_SIZE, NS_BIT);
1445ff27e85aSGilad Ben-Yossef set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT);
1446ff27e85aSGilad Ben-Yossef set_setup_mode(&desc[idx], SETUP_LOAD_STATE1);
1447ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], S_DIN_to_AES);
1448ff27e85aSGilad Ben-Yossef idx++;
1449ff27e85aSGilad Ben-Yossef
1450ff27e85aSGilad Ben-Yossef /* load MAC key */
1451ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]);
1452ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], DRV_CIPHER_CBC_MAC);
1453ff27e85aSGilad Ben-Yossef set_din_type(&desc[idx], DMA_DLLI, ctx->enckey_dma_addr,
1454ff27e85aSGilad Ben-Yossef ((ctx->enc_keylen == 24) ? CC_AES_KEY_SIZE_MAX :
1455ff27e85aSGilad Ben-Yossef ctx->enc_keylen), NS_BIT);
1456ff27e85aSGilad Ben-Yossef set_key_size_aes(&desc[idx], ctx->enc_keylen);
1457ff27e85aSGilad Ben-Yossef set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
1458ff27e85aSGilad Ben-Yossef set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT);
1459ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], S_DIN_to_HASH);
1460ff27e85aSGilad Ben-Yossef set_aes_not_hash_mode(&desc[idx]);
1461ff27e85aSGilad Ben-Yossef idx++;
1462ff27e85aSGilad Ben-Yossef
1463ff27e85aSGilad Ben-Yossef /* load MAC state */
1464ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]);
1465ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], DRV_CIPHER_CBC_MAC);
1466ff27e85aSGilad Ben-Yossef set_key_size_aes(&desc[idx], ctx->enc_keylen);
1467ff27e85aSGilad Ben-Yossef set_din_type(&desc[idx], DMA_DLLI, req_ctx->mac_buf_dma_addr,
1468ff27e85aSGilad Ben-Yossef AES_BLOCK_SIZE, NS_BIT);
1469ff27e85aSGilad Ben-Yossef set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT);
1470ff27e85aSGilad Ben-Yossef set_setup_mode(&desc[idx], SETUP_LOAD_STATE0);
1471ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], S_DIN_to_HASH);
1472ff27e85aSGilad Ben-Yossef set_aes_not_hash_mode(&desc[idx]);
1473ff27e85aSGilad Ben-Yossef idx++;
1474ff27e85aSGilad Ben-Yossef
1475ff27e85aSGilad Ben-Yossef /* process assoc data */
1476da3cf67fSGilad Ben-Yossef if (req_ctx->assoclen > 0) {
1477ff27e85aSGilad Ben-Yossef cc_set_assoc_desc(req, DIN_HASH, desc, &idx);
1478ff27e85aSGilad Ben-Yossef } else {
1479ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]);
1480ff27e85aSGilad Ben-Yossef set_din_type(&desc[idx], DMA_DLLI,
1481ff27e85aSGilad Ben-Yossef sg_dma_address(&req_ctx->ccm_adata_sg),
1482ff27e85aSGilad Ben-Yossef AES_BLOCK_SIZE + req_ctx->ccm_hdr_size, NS_BIT);
1483ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], DIN_HASH);
1484ff27e85aSGilad Ben-Yossef idx++;
1485ff27e85aSGilad Ben-Yossef }
1486ff27e85aSGilad Ben-Yossef
1487ff27e85aSGilad Ben-Yossef /* process the cipher */
1488ff27e85aSGilad Ben-Yossef if (req_ctx->cryptlen)
1489ff27e85aSGilad Ben-Yossef cc_proc_cipher_desc(req, cipher_flow_mode, desc, &idx);
1490ff27e85aSGilad Ben-Yossef
1491ff27e85aSGilad Ben-Yossef /* Read temporal MAC */
1492ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]);
1493ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], DRV_CIPHER_CBC_MAC);
1494ff27e85aSGilad Ben-Yossef set_dout_dlli(&desc[idx], req_ctx->mac_buf_dma_addr, ctx->authsize,
1495ff27e85aSGilad Ben-Yossef NS_BIT, 0);
1496ff27e85aSGilad Ben-Yossef set_setup_mode(&desc[idx], SETUP_WRITE_STATE0);
1497ff27e85aSGilad Ben-Yossef set_cipher_config0(&desc[idx], HASH_DIGEST_RESULT_LITTLE_ENDIAN);
1498ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], S_HASH_to_DOUT);
1499ff27e85aSGilad Ben-Yossef set_aes_not_hash_mode(&desc[idx]);
1500ff27e85aSGilad Ben-Yossef idx++;
1501ff27e85aSGilad Ben-Yossef
1502ff27e85aSGilad Ben-Yossef /* load AES-CTR state (for last MAC calculation)*/
1503ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]);
1504ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], DRV_CIPHER_CTR);
1505ff27e85aSGilad Ben-Yossef set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_ENCRYPT);
1506ff27e85aSGilad Ben-Yossef set_din_type(&desc[idx], DMA_DLLI, req_ctx->ccm_iv0_dma_addr,
1507ff27e85aSGilad Ben-Yossef AES_BLOCK_SIZE, NS_BIT);
1508ff27e85aSGilad Ben-Yossef set_key_size_aes(&desc[idx], ctx->enc_keylen);
1509ff27e85aSGilad Ben-Yossef set_setup_mode(&desc[idx], SETUP_LOAD_STATE1);
1510ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], S_DIN_to_AES);
1511ff27e85aSGilad Ben-Yossef idx++;
1512ff27e85aSGilad Ben-Yossef
1513ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]);
1514ff27e85aSGilad Ben-Yossef set_din_no_dma(&desc[idx], 0, 0xfffff0);
1515ff27e85aSGilad Ben-Yossef set_dout_no_dma(&desc[idx], 0, 0, 1);
1516ff27e85aSGilad Ben-Yossef idx++;
1517ff27e85aSGilad Ben-Yossef
1518ff27e85aSGilad Ben-Yossef /* encrypt the "T" value and store MAC in mac_state */
1519ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]);
1520ff27e85aSGilad Ben-Yossef set_din_type(&desc[idx], DMA_DLLI, req_ctx->mac_buf_dma_addr,
1521ff27e85aSGilad Ben-Yossef ctx->authsize, NS_BIT);
1522ff27e85aSGilad Ben-Yossef set_dout_dlli(&desc[idx], mac_result, ctx->authsize, NS_BIT, 1);
152327b3b22dSGilad Ben-Yossef set_queue_last_ind(ctx->drvdata, &desc[idx]);
1524ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], DIN_AES_DOUT);
1525ff27e85aSGilad Ben-Yossef idx++;
1526ff27e85aSGilad Ben-Yossef
1527ff27e85aSGilad Ben-Yossef *seq_size = idx;
1528ff27e85aSGilad Ben-Yossef return 0;
1529ff27e85aSGilad Ben-Yossef }
1530ff27e85aSGilad Ben-Yossef
config_ccm_adata(struct aead_request * req)1531ff27e85aSGilad Ben-Yossef static int config_ccm_adata(struct aead_request *req)
1532ff27e85aSGilad Ben-Yossef {
1533ff27e85aSGilad Ben-Yossef struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1534ff27e85aSGilad Ben-Yossef struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
1535ff27e85aSGilad Ben-Yossef struct device *dev = drvdata_to_dev(ctx->drvdata);
1536*07547fa7SHerbert Xu struct aead_req_ctx *req_ctx = aead_request_ctx_dma(req);
1537ff27e85aSGilad Ben-Yossef //unsigned int size_of_a = 0, rem_a_size = 0;
1538ff27e85aSGilad Ben-Yossef unsigned int lp = req->iv[0];
1539ff27e85aSGilad Ben-Yossef /* Note: The code assume that req->iv[0] already contains the value
1540ff27e85aSGilad Ben-Yossef * of L' of RFC3610
1541ff27e85aSGilad Ben-Yossef */
1542ff27e85aSGilad Ben-Yossef unsigned int l = lp + 1; /* This is L' of RFC 3610. */
1543ff27e85aSGilad Ben-Yossef unsigned int m = ctx->authsize; /* This is M' of RFC 3610. */
1544ff27e85aSGilad Ben-Yossef u8 *b0 = req_ctx->ccm_config + CCM_B0_OFFSET;
1545ff27e85aSGilad Ben-Yossef u8 *a0 = req_ctx->ccm_config + CCM_A0_OFFSET;
1546ff27e85aSGilad Ben-Yossef u8 *ctr_count_0 = req_ctx->ccm_config + CCM_CTR_COUNT_0_OFFSET;
1547ff27e85aSGilad Ben-Yossef unsigned int cryptlen = (req_ctx->gen_ctx.op_type ==
1548ff27e85aSGilad Ben-Yossef DRV_CRYPTO_DIRECTION_ENCRYPT) ?
1549ff27e85aSGilad Ben-Yossef req->cryptlen :
1550ff27e85aSGilad Ben-Yossef (req->cryptlen - ctx->authsize);
1551ff27e85aSGilad Ben-Yossef int rc;
1552ff27e85aSGilad Ben-Yossef
1553ff27e85aSGilad Ben-Yossef memset(req_ctx->mac_buf, 0, AES_BLOCK_SIZE);
1554ff27e85aSGilad Ben-Yossef memset(req_ctx->ccm_config, 0, AES_BLOCK_SIZE * 3);
1555ff27e85aSGilad Ben-Yossef
1556ff27e85aSGilad Ben-Yossef /* taken from crypto/ccm.c */
1557ff27e85aSGilad Ben-Yossef /* 2 <= L <= 8, so 1 <= L' <= 7. */
1558ff27e85aSGilad Ben-Yossef if (l < 2 || l > 8) {
1559c7b31c88SGilad Ben-Yossef dev_dbg(dev, "illegal iv value %X\n", req->iv[0]);
1560ff27e85aSGilad Ben-Yossef return -EINVAL;
1561ff27e85aSGilad Ben-Yossef }
1562ff27e85aSGilad Ben-Yossef memcpy(b0, req->iv, AES_BLOCK_SIZE);
1563ff27e85aSGilad Ben-Yossef
1564ff27e85aSGilad Ben-Yossef /* format control info per RFC 3610 and
1565ff27e85aSGilad Ben-Yossef * NIST Special Publication 800-38C
1566ff27e85aSGilad Ben-Yossef */
1567ff27e85aSGilad Ben-Yossef *b0 |= (8 * ((m - 2) / 2));
1568da3cf67fSGilad Ben-Yossef if (req_ctx->assoclen > 0)
1569ff27e85aSGilad Ben-Yossef *b0 |= 64; /* Enable bit 6 if Adata exists. */
1570ff27e85aSGilad Ben-Yossef
1571ff27e85aSGilad Ben-Yossef rc = set_msg_len(b0 + 16 - l, cryptlen, l); /* Write L'. */
1572ff27e85aSGilad Ben-Yossef if (rc) {
1573ff27e85aSGilad Ben-Yossef dev_err(dev, "message len overflow detected");
1574ff27e85aSGilad Ben-Yossef return rc;
1575ff27e85aSGilad Ben-Yossef }
1576ff27e85aSGilad Ben-Yossef /* END of "taken from crypto/ccm.c" */
1577ff27e85aSGilad Ben-Yossef
1578ff27e85aSGilad Ben-Yossef /* l(a) - size of associated data. */
1579da3cf67fSGilad Ben-Yossef req_ctx->ccm_hdr_size = format_ccm_a0(a0, req_ctx->assoclen);
1580ff27e85aSGilad Ben-Yossef
1581ff27e85aSGilad Ben-Yossef memset(req->iv + 15 - req->iv[0], 0, req->iv[0] + 1);
1582ff27e85aSGilad Ben-Yossef req->iv[15] = 1;
1583ff27e85aSGilad Ben-Yossef
1584ff27e85aSGilad Ben-Yossef memcpy(ctr_count_0, req->iv, AES_BLOCK_SIZE);
1585ff27e85aSGilad Ben-Yossef ctr_count_0[15] = 0;
1586ff27e85aSGilad Ben-Yossef
1587ff27e85aSGilad Ben-Yossef return 0;
1588ff27e85aSGilad Ben-Yossef }
1589ff27e85aSGilad Ben-Yossef
cc_proc_rfc4309_ccm(struct aead_request * req)1590ff27e85aSGilad Ben-Yossef static void cc_proc_rfc4309_ccm(struct aead_request *req)
1591ff27e85aSGilad Ben-Yossef {
1592ff27e85aSGilad Ben-Yossef struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1593ff27e85aSGilad Ben-Yossef struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
1594*07547fa7SHerbert Xu struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(req);
1595ff27e85aSGilad Ben-Yossef
1596ff27e85aSGilad Ben-Yossef /* L' */
1597ff27e85aSGilad Ben-Yossef memset(areq_ctx->ctr_iv, 0, AES_BLOCK_SIZE);
1598ff27e85aSGilad Ben-Yossef /* For RFC 4309, always use 4 bytes for message length
1599ff27e85aSGilad Ben-Yossef * (at most 2^32-1 bytes).
1600ff27e85aSGilad Ben-Yossef */
1601ff27e85aSGilad Ben-Yossef areq_ctx->ctr_iv[0] = 3;
1602ff27e85aSGilad Ben-Yossef
1603ff27e85aSGilad Ben-Yossef /* In RFC 4309 there is an 11-bytes nonce+IV part,
1604ff27e85aSGilad Ben-Yossef * that we build here.
1605ff27e85aSGilad Ben-Yossef */
1606ff27e85aSGilad Ben-Yossef memcpy(areq_ctx->ctr_iv + CCM_BLOCK_NONCE_OFFSET, ctx->ctr_nonce,
1607ff27e85aSGilad Ben-Yossef CCM_BLOCK_NONCE_SIZE);
1608ff27e85aSGilad Ben-Yossef memcpy(areq_ctx->ctr_iv + CCM_BLOCK_IV_OFFSET, req->iv,
1609ff27e85aSGilad Ben-Yossef CCM_BLOCK_IV_SIZE);
1610ff27e85aSGilad Ben-Yossef req->iv = areq_ctx->ctr_iv;
1611ff27e85aSGilad Ben-Yossef }
1612ff27e85aSGilad Ben-Yossef
cc_set_ghash_desc(struct aead_request * req,struct cc_hw_desc desc[],unsigned int * seq_size)1613ff27e85aSGilad Ben-Yossef static void cc_set_ghash_desc(struct aead_request *req,
1614ff27e85aSGilad Ben-Yossef struct cc_hw_desc desc[], unsigned int *seq_size)
1615ff27e85aSGilad Ben-Yossef {
1616ff27e85aSGilad Ben-Yossef struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1617ff27e85aSGilad Ben-Yossef struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
1618*07547fa7SHerbert Xu struct aead_req_ctx *req_ctx = aead_request_ctx_dma(req);
1619ff27e85aSGilad Ben-Yossef unsigned int idx = *seq_size;
1620ff27e85aSGilad Ben-Yossef
1621ff27e85aSGilad Ben-Yossef /* load key to AES*/
1622ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]);
1623ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], DRV_CIPHER_ECB);
1624ff27e85aSGilad Ben-Yossef set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_ENCRYPT);
1625ff27e85aSGilad Ben-Yossef set_din_type(&desc[idx], DMA_DLLI, ctx->enckey_dma_addr,
1626ff27e85aSGilad Ben-Yossef ctx->enc_keylen, NS_BIT);
1627ff27e85aSGilad Ben-Yossef set_key_size_aes(&desc[idx], ctx->enc_keylen);
1628ff27e85aSGilad Ben-Yossef set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
1629ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], S_DIN_to_AES);
1630ff27e85aSGilad Ben-Yossef idx++;
1631ff27e85aSGilad Ben-Yossef
1632ff27e85aSGilad Ben-Yossef /* process one zero block to generate hkey */
1633ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]);
1634ff27e85aSGilad Ben-Yossef set_din_const(&desc[idx], 0x0, AES_BLOCK_SIZE);
1635ff27e85aSGilad Ben-Yossef set_dout_dlli(&desc[idx], req_ctx->hkey_dma_addr, AES_BLOCK_SIZE,
1636ff27e85aSGilad Ben-Yossef NS_BIT, 0);
1637ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], DIN_AES_DOUT);
1638ff27e85aSGilad Ben-Yossef idx++;
1639ff27e85aSGilad Ben-Yossef
1640ff27e85aSGilad Ben-Yossef /* Memory Barrier */
1641ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]);
1642ff27e85aSGilad Ben-Yossef set_din_no_dma(&desc[idx], 0, 0xfffff0);
1643ff27e85aSGilad Ben-Yossef set_dout_no_dma(&desc[idx], 0, 0, 1);
1644ff27e85aSGilad Ben-Yossef idx++;
1645ff27e85aSGilad Ben-Yossef
1646ff27e85aSGilad Ben-Yossef /* Load GHASH subkey */
1647ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]);
1648ff27e85aSGilad Ben-Yossef set_din_type(&desc[idx], DMA_DLLI, req_ctx->hkey_dma_addr,
1649ff27e85aSGilad Ben-Yossef AES_BLOCK_SIZE, NS_BIT);
1650ff27e85aSGilad Ben-Yossef set_dout_no_dma(&desc[idx], 0, 0, 1);
1651ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], S_DIN_to_HASH);
1652ff27e85aSGilad Ben-Yossef set_aes_not_hash_mode(&desc[idx]);
1653ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], DRV_HASH_HW_GHASH);
1654ff27e85aSGilad Ben-Yossef set_cipher_config1(&desc[idx], HASH_PADDING_ENABLED);
1655ff27e85aSGilad Ben-Yossef set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
1656ff27e85aSGilad Ben-Yossef idx++;
1657ff27e85aSGilad Ben-Yossef
1658ff27e85aSGilad Ben-Yossef /* Configure Hash Engine to work with GHASH.
1659ff27e85aSGilad Ben-Yossef * Since it was not possible to extend HASH submodes to add GHASH,
1660ff27e85aSGilad Ben-Yossef * The following command is necessary in order to
1661ff27e85aSGilad Ben-Yossef * select GHASH (according to HW designers)
1662ff27e85aSGilad Ben-Yossef */
1663ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]);
1664ff27e85aSGilad Ben-Yossef set_din_no_dma(&desc[idx], 0, 0xfffff0);
1665ff27e85aSGilad Ben-Yossef set_dout_no_dma(&desc[idx], 0, 0, 1);
1666ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], S_DIN_to_HASH);
1667ff27e85aSGilad Ben-Yossef set_aes_not_hash_mode(&desc[idx]);
1668ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], DRV_HASH_HW_GHASH);
1669ff27e85aSGilad Ben-Yossef set_cipher_do(&desc[idx], 1); //1=AES_SK RKEK
1670ff27e85aSGilad Ben-Yossef set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_ENCRYPT);
1671ff27e85aSGilad Ben-Yossef set_cipher_config1(&desc[idx], HASH_PADDING_ENABLED);
1672ff27e85aSGilad Ben-Yossef set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
1673ff27e85aSGilad Ben-Yossef idx++;
1674ff27e85aSGilad Ben-Yossef
1675ff27e85aSGilad Ben-Yossef /* Load GHASH initial STATE (which is 0). (for any hash there is an
1676ff27e85aSGilad Ben-Yossef * initial state)
1677ff27e85aSGilad Ben-Yossef */
1678ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]);
1679ff27e85aSGilad Ben-Yossef set_din_const(&desc[idx], 0x0, AES_BLOCK_SIZE);
1680ff27e85aSGilad Ben-Yossef set_dout_no_dma(&desc[idx], 0, 0, 1);
1681ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], S_DIN_to_HASH);
1682ff27e85aSGilad Ben-Yossef set_aes_not_hash_mode(&desc[idx]);
1683ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], DRV_HASH_HW_GHASH);
1684ff27e85aSGilad Ben-Yossef set_cipher_config1(&desc[idx], HASH_PADDING_ENABLED);
1685ff27e85aSGilad Ben-Yossef set_setup_mode(&desc[idx], SETUP_LOAD_STATE0);
1686ff27e85aSGilad Ben-Yossef idx++;
1687ff27e85aSGilad Ben-Yossef
1688ff27e85aSGilad Ben-Yossef *seq_size = idx;
1689ff27e85aSGilad Ben-Yossef }
1690ff27e85aSGilad Ben-Yossef
cc_set_gctr_desc(struct aead_request * req,struct cc_hw_desc desc[],unsigned int * seq_size)1691ff27e85aSGilad Ben-Yossef static void cc_set_gctr_desc(struct aead_request *req, struct cc_hw_desc desc[],
1692ff27e85aSGilad Ben-Yossef unsigned int *seq_size)
1693ff27e85aSGilad Ben-Yossef {
1694ff27e85aSGilad Ben-Yossef struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1695ff27e85aSGilad Ben-Yossef struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
1696*07547fa7SHerbert Xu struct aead_req_ctx *req_ctx = aead_request_ctx_dma(req);
1697ff27e85aSGilad Ben-Yossef unsigned int idx = *seq_size;
1698ff27e85aSGilad Ben-Yossef
1699ff27e85aSGilad Ben-Yossef /* load key to AES*/
1700ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]);
1701ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], DRV_CIPHER_GCTR);
1702ff27e85aSGilad Ben-Yossef set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_ENCRYPT);
1703ff27e85aSGilad Ben-Yossef set_din_type(&desc[idx], DMA_DLLI, ctx->enckey_dma_addr,
1704ff27e85aSGilad Ben-Yossef ctx->enc_keylen, NS_BIT);
1705ff27e85aSGilad Ben-Yossef set_key_size_aes(&desc[idx], ctx->enc_keylen);
1706ff27e85aSGilad Ben-Yossef set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
1707ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], S_DIN_to_AES);
1708ff27e85aSGilad Ben-Yossef idx++;
1709ff27e85aSGilad Ben-Yossef
1710ff27e85aSGilad Ben-Yossef if (req_ctx->cryptlen && !req_ctx->plaintext_authenticate_only) {
1711ff27e85aSGilad Ben-Yossef /* load AES/CTR initial CTR value inc by 2*/
1712ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]);
1713ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], DRV_CIPHER_GCTR);
1714ff27e85aSGilad Ben-Yossef set_key_size_aes(&desc[idx], ctx->enc_keylen);
1715ff27e85aSGilad Ben-Yossef set_din_type(&desc[idx], DMA_DLLI,
1716ff27e85aSGilad Ben-Yossef req_ctx->gcm_iv_inc2_dma_addr, AES_BLOCK_SIZE,
1717ff27e85aSGilad Ben-Yossef NS_BIT);
1718ff27e85aSGilad Ben-Yossef set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_ENCRYPT);
1719ff27e85aSGilad Ben-Yossef set_setup_mode(&desc[idx], SETUP_LOAD_STATE1);
1720ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], S_DIN_to_AES);
1721ff27e85aSGilad Ben-Yossef idx++;
1722ff27e85aSGilad Ben-Yossef }
1723ff27e85aSGilad Ben-Yossef
1724ff27e85aSGilad Ben-Yossef *seq_size = idx;
1725ff27e85aSGilad Ben-Yossef }
1726ff27e85aSGilad Ben-Yossef
cc_proc_gcm_result(struct aead_request * req,struct cc_hw_desc desc[],unsigned int * seq_size)1727ff27e85aSGilad Ben-Yossef static void cc_proc_gcm_result(struct aead_request *req,
1728ff27e85aSGilad Ben-Yossef struct cc_hw_desc desc[],
1729ff27e85aSGilad Ben-Yossef unsigned int *seq_size)
1730ff27e85aSGilad Ben-Yossef {
1731ff27e85aSGilad Ben-Yossef struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1732ff27e85aSGilad Ben-Yossef struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
1733*07547fa7SHerbert Xu struct aead_req_ctx *req_ctx = aead_request_ctx_dma(req);
1734ff27e85aSGilad Ben-Yossef dma_addr_t mac_result;
1735ff27e85aSGilad Ben-Yossef unsigned int idx = *seq_size;
1736ff27e85aSGilad Ben-Yossef
1737ff27e85aSGilad Ben-Yossef if (req_ctx->gen_ctx.op_type == DRV_CRYPTO_DIRECTION_DECRYPT) {
1738ff27e85aSGilad Ben-Yossef mac_result = req_ctx->mac_buf_dma_addr;
1739ff27e85aSGilad Ben-Yossef } else { /* Encrypt */
1740ff27e85aSGilad Ben-Yossef mac_result = req_ctx->icv_dma_addr;
1741ff27e85aSGilad Ben-Yossef }
1742ff27e85aSGilad Ben-Yossef
1743ff27e85aSGilad Ben-Yossef /* process(ghash) gcm_block_len */
1744ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]);
1745ff27e85aSGilad Ben-Yossef set_din_type(&desc[idx], DMA_DLLI, req_ctx->gcm_block_len_dma_addr,
1746ff27e85aSGilad Ben-Yossef AES_BLOCK_SIZE, NS_BIT);
1747ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], DIN_HASH);
1748ff27e85aSGilad Ben-Yossef idx++;
1749ff27e85aSGilad Ben-Yossef
1750ff27e85aSGilad Ben-Yossef /* Store GHASH state after GHASH(Associated Data + Cipher +LenBlock) */
1751ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]);
1752ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], DRV_HASH_HW_GHASH);
1753ff27e85aSGilad Ben-Yossef set_din_no_dma(&desc[idx], 0, 0xfffff0);
1754ff27e85aSGilad Ben-Yossef set_dout_dlli(&desc[idx], req_ctx->mac_buf_dma_addr, AES_BLOCK_SIZE,
1755ff27e85aSGilad Ben-Yossef NS_BIT, 0);
1756ff27e85aSGilad Ben-Yossef set_setup_mode(&desc[idx], SETUP_WRITE_STATE0);
1757ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], S_HASH_to_DOUT);
1758ff27e85aSGilad Ben-Yossef set_aes_not_hash_mode(&desc[idx]);
1759ff27e85aSGilad Ben-Yossef
1760ff27e85aSGilad Ben-Yossef idx++;
1761ff27e85aSGilad Ben-Yossef
1762ff27e85aSGilad Ben-Yossef /* load AES/CTR initial CTR value inc by 1*/
1763ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]);
1764ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], DRV_CIPHER_GCTR);
1765ff27e85aSGilad Ben-Yossef set_key_size_aes(&desc[idx], ctx->enc_keylen);
1766ff27e85aSGilad Ben-Yossef set_din_type(&desc[idx], DMA_DLLI, req_ctx->gcm_iv_inc1_dma_addr,
1767ff27e85aSGilad Ben-Yossef AES_BLOCK_SIZE, NS_BIT);
1768ff27e85aSGilad Ben-Yossef set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_ENCRYPT);
1769ff27e85aSGilad Ben-Yossef set_setup_mode(&desc[idx], SETUP_LOAD_STATE1);
1770ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], S_DIN_to_AES);
1771ff27e85aSGilad Ben-Yossef idx++;
1772ff27e85aSGilad Ben-Yossef
1773ff27e85aSGilad Ben-Yossef /* Memory Barrier */
1774ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]);
1775ff27e85aSGilad Ben-Yossef set_din_no_dma(&desc[idx], 0, 0xfffff0);
1776ff27e85aSGilad Ben-Yossef set_dout_no_dma(&desc[idx], 0, 0, 1);
1777ff27e85aSGilad Ben-Yossef idx++;
1778ff27e85aSGilad Ben-Yossef
1779ff27e85aSGilad Ben-Yossef /* process GCTR on stored GHASH and store MAC in mac_state*/
1780ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]);
1781ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], DRV_CIPHER_GCTR);
1782ff27e85aSGilad Ben-Yossef set_din_type(&desc[idx], DMA_DLLI, req_ctx->mac_buf_dma_addr,
1783ff27e85aSGilad Ben-Yossef AES_BLOCK_SIZE, NS_BIT);
1784ff27e85aSGilad Ben-Yossef set_dout_dlli(&desc[idx], mac_result, ctx->authsize, NS_BIT, 1);
178527b3b22dSGilad Ben-Yossef set_queue_last_ind(ctx->drvdata, &desc[idx]);
1786ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], DIN_AES_DOUT);
1787ff27e85aSGilad Ben-Yossef idx++;
1788ff27e85aSGilad Ben-Yossef
1789ff27e85aSGilad Ben-Yossef *seq_size = idx;
1790ff27e85aSGilad Ben-Yossef }
1791ff27e85aSGilad Ben-Yossef
cc_gcm(struct aead_request * req,struct cc_hw_desc desc[],unsigned int * seq_size)1792ff27e85aSGilad Ben-Yossef static int cc_gcm(struct aead_request *req, struct cc_hw_desc desc[],
1793ff27e85aSGilad Ben-Yossef unsigned int *seq_size)
1794ff27e85aSGilad Ben-Yossef {
1795*07547fa7SHerbert Xu struct aead_req_ctx *req_ctx = aead_request_ctx_dma(req);
1796ff27e85aSGilad Ben-Yossef unsigned int cipher_flow_mode;
1797ff27e85aSGilad Ben-Yossef
1798ff27e85aSGilad Ben-Yossef //in RFC4543 no data to encrypt. just copy data from src to dest.
1799ff27e85aSGilad Ben-Yossef if (req_ctx->plaintext_authenticate_only) {
1800ff27e85aSGilad Ben-Yossef cc_proc_cipher_desc(req, BYPASS, desc, seq_size);
1801ff27e85aSGilad Ben-Yossef cc_set_ghash_desc(req, desc, seq_size);
1802ff27e85aSGilad Ben-Yossef /* process(ghash) assoc data */
1803ff27e85aSGilad Ben-Yossef cc_set_assoc_desc(req, DIN_HASH, desc, seq_size);
1804ff27e85aSGilad Ben-Yossef cc_set_gctr_desc(req, desc, seq_size);
1805ff27e85aSGilad Ben-Yossef cc_proc_gcm_result(req, desc, seq_size);
1806ff27e85aSGilad Ben-Yossef return 0;
1807ff27e85aSGilad Ben-Yossef }
1808ff27e85aSGilad Ben-Yossef
180969cd3e16SGilad Ben-Yossef if (req_ctx->gen_ctx.op_type == DRV_CRYPTO_DIRECTION_DECRYPT) {
181069cd3e16SGilad Ben-Yossef cipher_flow_mode = AES_and_HASH;
181169cd3e16SGilad Ben-Yossef } else { /* Encrypt */
181269cd3e16SGilad Ben-Yossef cipher_flow_mode = AES_to_HASH_and_DOUT;
181369cd3e16SGilad Ben-Yossef }
181469cd3e16SGilad Ben-Yossef
1815ff27e85aSGilad Ben-Yossef // for gcm and rfc4106.
1816ff27e85aSGilad Ben-Yossef cc_set_ghash_desc(req, desc, seq_size);
1817ff27e85aSGilad Ben-Yossef /* process(ghash) assoc data */
1818da3cf67fSGilad Ben-Yossef if (req_ctx->assoclen > 0)
1819ff27e85aSGilad Ben-Yossef cc_set_assoc_desc(req, DIN_HASH, desc, seq_size);
1820ff27e85aSGilad Ben-Yossef cc_set_gctr_desc(req, desc, seq_size);
1821ff27e85aSGilad Ben-Yossef /* process(gctr+ghash) */
1822ff27e85aSGilad Ben-Yossef if (req_ctx->cryptlen)
1823ff27e85aSGilad Ben-Yossef cc_proc_cipher_desc(req, cipher_flow_mode, desc, seq_size);
1824ff27e85aSGilad Ben-Yossef cc_proc_gcm_result(req, desc, seq_size);
1825ff27e85aSGilad Ben-Yossef
1826ff27e85aSGilad Ben-Yossef return 0;
1827ff27e85aSGilad Ben-Yossef }
1828ff27e85aSGilad Ben-Yossef
config_gcm_context(struct aead_request * req)1829ff27e85aSGilad Ben-Yossef static int config_gcm_context(struct aead_request *req)
1830ff27e85aSGilad Ben-Yossef {
1831ff27e85aSGilad Ben-Yossef struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1832ff27e85aSGilad Ben-Yossef struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
1833*07547fa7SHerbert Xu struct aead_req_ctx *req_ctx = aead_request_ctx_dma(req);
1834ff27e85aSGilad Ben-Yossef struct device *dev = drvdata_to_dev(ctx->drvdata);
1835ff27e85aSGilad Ben-Yossef
1836ff27e85aSGilad Ben-Yossef unsigned int cryptlen = (req_ctx->gen_ctx.op_type ==
1837ff27e85aSGilad Ben-Yossef DRV_CRYPTO_DIRECTION_ENCRYPT) ?
1838ff27e85aSGilad Ben-Yossef req->cryptlen :
1839ff27e85aSGilad Ben-Yossef (req->cryptlen - ctx->authsize);
1840ff27e85aSGilad Ben-Yossef __be32 counter = cpu_to_be32(2);
1841ff27e85aSGilad Ben-Yossef
1842da3cf67fSGilad Ben-Yossef dev_dbg(dev, "%s() cryptlen = %d, req_ctx->assoclen = %d ctx->authsize = %d\n",
1843da3cf67fSGilad Ben-Yossef __func__, cryptlen, req_ctx->assoclen, ctx->authsize);
1844ff27e85aSGilad Ben-Yossef
1845ff27e85aSGilad Ben-Yossef memset(req_ctx->hkey, 0, AES_BLOCK_SIZE);
1846ff27e85aSGilad Ben-Yossef
1847ff27e85aSGilad Ben-Yossef memset(req_ctx->mac_buf, 0, AES_BLOCK_SIZE);
1848ff27e85aSGilad Ben-Yossef
1849ff27e85aSGilad Ben-Yossef memcpy(req->iv + 12, &counter, 4);
1850ff27e85aSGilad Ben-Yossef memcpy(req_ctx->gcm_iv_inc2, req->iv, 16);
1851ff27e85aSGilad Ben-Yossef
1852ff27e85aSGilad Ben-Yossef counter = cpu_to_be32(1);
1853ff27e85aSGilad Ben-Yossef memcpy(req->iv + 12, &counter, 4);
1854ff27e85aSGilad Ben-Yossef memcpy(req_ctx->gcm_iv_inc1, req->iv, 16);
1855ff27e85aSGilad Ben-Yossef
1856ff27e85aSGilad Ben-Yossef if (!req_ctx->plaintext_authenticate_only) {
1857ff27e85aSGilad Ben-Yossef __be64 temp64;
1858ff27e85aSGilad Ben-Yossef
1859da3cf67fSGilad Ben-Yossef temp64 = cpu_to_be64(req_ctx->assoclen * 8);
1860ff27e85aSGilad Ben-Yossef memcpy(&req_ctx->gcm_len_block.len_a, &temp64, sizeof(temp64));
1861ff27e85aSGilad Ben-Yossef temp64 = cpu_to_be64(cryptlen * 8);
1862ff27e85aSGilad Ben-Yossef memcpy(&req_ctx->gcm_len_block.len_c, &temp64, 8);
1863ff27e85aSGilad Ben-Yossef } else {
1864ff27e85aSGilad Ben-Yossef /* rfc4543=> all data(AAD,IV,Plain) are considered additional
1865ff27e85aSGilad Ben-Yossef * data that is nothing is encrypted.
1866ff27e85aSGilad Ben-Yossef */
1867ff27e85aSGilad Ben-Yossef __be64 temp64;
1868ff27e85aSGilad Ben-Yossef
18690eae14a0SGilad Ben-Yossef temp64 = cpu_to_be64((req_ctx->assoclen + cryptlen) * 8);
1870ff27e85aSGilad Ben-Yossef memcpy(&req_ctx->gcm_len_block.len_a, &temp64, sizeof(temp64));
1871ff27e85aSGilad Ben-Yossef temp64 = 0;
1872ff27e85aSGilad Ben-Yossef memcpy(&req_ctx->gcm_len_block.len_c, &temp64, 8);
1873ff27e85aSGilad Ben-Yossef }
1874ff27e85aSGilad Ben-Yossef
1875ff27e85aSGilad Ben-Yossef return 0;
1876ff27e85aSGilad Ben-Yossef }
1877ff27e85aSGilad Ben-Yossef
cc_proc_rfc4_gcm(struct aead_request * req)1878ff27e85aSGilad Ben-Yossef static void cc_proc_rfc4_gcm(struct aead_request *req)
1879ff27e85aSGilad Ben-Yossef {
1880ff27e85aSGilad Ben-Yossef struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1881ff27e85aSGilad Ben-Yossef struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
1882*07547fa7SHerbert Xu struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(req);
1883ff27e85aSGilad Ben-Yossef
1884ff27e85aSGilad Ben-Yossef memcpy(areq_ctx->ctr_iv + GCM_BLOCK_RFC4_NONCE_OFFSET,
1885ff27e85aSGilad Ben-Yossef ctx->ctr_nonce, GCM_BLOCK_RFC4_NONCE_SIZE);
1886ff27e85aSGilad Ben-Yossef memcpy(areq_ctx->ctr_iv + GCM_BLOCK_RFC4_IV_OFFSET, req->iv,
1887ff27e85aSGilad Ben-Yossef GCM_BLOCK_RFC4_IV_SIZE);
1888ff27e85aSGilad Ben-Yossef req->iv = areq_ctx->ctr_iv;
1889ff27e85aSGilad Ben-Yossef }
1890ff27e85aSGilad Ben-Yossef
cc_proc_aead(struct aead_request * req,enum drv_crypto_direction direct)1891ff27e85aSGilad Ben-Yossef static int cc_proc_aead(struct aead_request *req,
1892ff27e85aSGilad Ben-Yossef enum drv_crypto_direction direct)
1893ff27e85aSGilad Ben-Yossef {
1894ff27e85aSGilad Ben-Yossef int rc = 0;
1895ff27e85aSGilad Ben-Yossef int seq_len = 0;
1896ff27e85aSGilad Ben-Yossef struct cc_hw_desc desc[MAX_AEAD_PROCESS_SEQ];
1897ff27e85aSGilad Ben-Yossef struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1898ff27e85aSGilad Ben-Yossef struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
1899*07547fa7SHerbert Xu struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(req);
1900ff27e85aSGilad Ben-Yossef struct device *dev = drvdata_to_dev(ctx->drvdata);
1901ff27e85aSGilad Ben-Yossef struct cc_crypto_req cc_req = {};
1902ff27e85aSGilad Ben-Yossef
1903ff27e85aSGilad Ben-Yossef dev_dbg(dev, "%s context=%p req=%p iv=%p src=%p src_ofs=%d dst=%p dst_ofs=%d cryptolen=%d\n",
1904ff27e85aSGilad Ben-Yossef ((direct == DRV_CRYPTO_DIRECTION_ENCRYPT) ? "Enc" : "Dec"),
1905ff27e85aSGilad Ben-Yossef ctx, req, req->iv, sg_virt(req->src), req->src->offset,
1906ff27e85aSGilad Ben-Yossef sg_virt(req->dst), req->dst->offset, req->cryptlen);
1907ff27e85aSGilad Ben-Yossef
1908ff27e85aSGilad Ben-Yossef /* STAT_PHASE_0: Init and sanity checks */
1909ff27e85aSGilad Ben-Yossef
1910ff27e85aSGilad Ben-Yossef /* Check data length according to mode */
1911ff27e85aSGilad Ben-Yossef if (validate_data_size(ctx, direct, req)) {
1912ff27e85aSGilad Ben-Yossef dev_err(dev, "Unsupported crypt/assoc len %d/%d.\n",
1913da3cf67fSGilad Ben-Yossef req->cryptlen, areq_ctx->assoclen);
1914ff27e85aSGilad Ben-Yossef return -EINVAL;
1915ff27e85aSGilad Ben-Yossef }
1916ff27e85aSGilad Ben-Yossef
1917ff27e85aSGilad Ben-Yossef /* Setup request structure */
1918f4274eecSGeert Uytterhoeven cc_req.user_cb = cc_aead_complete;
1919f4274eecSGeert Uytterhoeven cc_req.user_arg = req;
1920ff27e85aSGilad Ben-Yossef
1921ff27e85aSGilad Ben-Yossef /* Setup request context */
1922ff27e85aSGilad Ben-Yossef areq_ctx->gen_ctx.op_type = direct;
1923ff27e85aSGilad Ben-Yossef areq_ctx->req_authsize = ctx->authsize;
1924ff27e85aSGilad Ben-Yossef areq_ctx->cipher_mode = ctx->cipher_mode;
1925ff27e85aSGilad Ben-Yossef
1926ff27e85aSGilad Ben-Yossef /* STAT_PHASE_1: Map buffers */
1927ff27e85aSGilad Ben-Yossef
1928ff27e85aSGilad Ben-Yossef if (ctx->cipher_mode == DRV_CIPHER_CTR) {
1929ff27e85aSGilad Ben-Yossef /* Build CTR IV - Copy nonce from last 4 bytes in
1930ff27e85aSGilad Ben-Yossef * CTR key to first 4 bytes in CTR IV
1931ff27e85aSGilad Ben-Yossef */
1932ff27e85aSGilad Ben-Yossef memcpy(areq_ctx->ctr_iv, ctx->ctr_nonce,
1933ff27e85aSGilad Ben-Yossef CTR_RFC3686_NONCE_SIZE);
1934e6e6600cSGilad Ben-Yossef memcpy(areq_ctx->ctr_iv + CTR_RFC3686_NONCE_SIZE, req->iv,
1935e6e6600cSGilad Ben-Yossef CTR_RFC3686_IV_SIZE);
1936ff27e85aSGilad Ben-Yossef /* Initialize counter portion of counter block */
1937ff27e85aSGilad Ben-Yossef *(__be32 *)(areq_ctx->ctr_iv + CTR_RFC3686_NONCE_SIZE +
1938ff27e85aSGilad Ben-Yossef CTR_RFC3686_IV_SIZE) = cpu_to_be32(1);
1939ff27e85aSGilad Ben-Yossef
1940ff27e85aSGilad Ben-Yossef /* Replace with counter iv */
1941ff27e85aSGilad Ben-Yossef req->iv = areq_ctx->ctr_iv;
1942ff27e85aSGilad Ben-Yossef areq_ctx->hw_iv_size = CTR_RFC3686_BLOCK_SIZE;
1943ff27e85aSGilad Ben-Yossef } else if ((ctx->cipher_mode == DRV_CIPHER_CCM) ||
1944ff27e85aSGilad Ben-Yossef (ctx->cipher_mode == DRV_CIPHER_GCTR)) {
1945ff27e85aSGilad Ben-Yossef areq_ctx->hw_iv_size = AES_BLOCK_SIZE;
1946ff27e85aSGilad Ben-Yossef if (areq_ctx->ctr_iv != req->iv) {
1947ff27e85aSGilad Ben-Yossef memcpy(areq_ctx->ctr_iv, req->iv,
1948ff27e85aSGilad Ben-Yossef crypto_aead_ivsize(tfm));
1949ff27e85aSGilad Ben-Yossef req->iv = areq_ctx->ctr_iv;
1950ff27e85aSGilad Ben-Yossef }
1951ff27e85aSGilad Ben-Yossef } else {
1952ff27e85aSGilad Ben-Yossef areq_ctx->hw_iv_size = crypto_aead_ivsize(tfm);
1953ff27e85aSGilad Ben-Yossef }
1954ff27e85aSGilad Ben-Yossef
1955ff27e85aSGilad Ben-Yossef if (ctx->cipher_mode == DRV_CIPHER_CCM) {
1956ff27e85aSGilad Ben-Yossef rc = config_ccm_adata(req);
1957ff27e85aSGilad Ben-Yossef if (rc) {
1958ff27e85aSGilad Ben-Yossef dev_dbg(dev, "config_ccm_adata() returned with a failure %d!",
1959ff27e85aSGilad Ben-Yossef rc);
1960ff27e85aSGilad Ben-Yossef goto exit;
1961ff27e85aSGilad Ben-Yossef }
1962ff27e85aSGilad Ben-Yossef } else {
1963ff27e85aSGilad Ben-Yossef areq_ctx->ccm_hdr_size = ccm_header_size_null;
1964ff27e85aSGilad Ben-Yossef }
1965ff27e85aSGilad Ben-Yossef
1966ff27e85aSGilad Ben-Yossef if (ctx->cipher_mode == DRV_CIPHER_GCTR) {
1967ff27e85aSGilad Ben-Yossef rc = config_gcm_context(req);
1968ff27e85aSGilad Ben-Yossef if (rc) {
1969ff27e85aSGilad Ben-Yossef dev_dbg(dev, "config_gcm_context() returned with a failure %d!",
1970ff27e85aSGilad Ben-Yossef rc);
1971ff27e85aSGilad Ben-Yossef goto exit;
1972ff27e85aSGilad Ben-Yossef }
1973ff27e85aSGilad Ben-Yossef }
1974ff27e85aSGilad Ben-Yossef
1975ff27e85aSGilad Ben-Yossef rc = cc_map_aead_request(ctx->drvdata, req);
1976ff27e85aSGilad Ben-Yossef if (rc) {
1977ff27e85aSGilad Ben-Yossef dev_err(dev, "map_request() failed\n");
1978ff27e85aSGilad Ben-Yossef goto exit;
1979ff27e85aSGilad Ben-Yossef }
1980ff27e85aSGilad Ben-Yossef
1981ff27e85aSGilad Ben-Yossef /* STAT_PHASE_2: Create sequence */
1982ff27e85aSGilad Ben-Yossef
1983ff27e85aSGilad Ben-Yossef /* Load MLLI tables to SRAM if necessary */
1984ff27e85aSGilad Ben-Yossef cc_mlli_to_sram(req, desc, &seq_len);
1985ff27e85aSGilad Ben-Yossef
1986ff27e85aSGilad Ben-Yossef switch (ctx->auth_mode) {
1987ff27e85aSGilad Ben-Yossef case DRV_HASH_SHA1:
1988ff27e85aSGilad Ben-Yossef case DRV_HASH_SHA256:
1989ff27e85aSGilad Ben-Yossef cc_hmac_authenc(req, desc, &seq_len);
1990ff27e85aSGilad Ben-Yossef break;
1991ff27e85aSGilad Ben-Yossef case DRV_HASH_XCBC_MAC:
1992ff27e85aSGilad Ben-Yossef cc_xcbc_authenc(req, desc, &seq_len);
1993ff27e85aSGilad Ben-Yossef break;
1994ff27e85aSGilad Ben-Yossef case DRV_HASH_NULL:
1995ff27e85aSGilad Ben-Yossef if (ctx->cipher_mode == DRV_CIPHER_CCM)
1996ff27e85aSGilad Ben-Yossef cc_ccm(req, desc, &seq_len);
1997ff27e85aSGilad Ben-Yossef if (ctx->cipher_mode == DRV_CIPHER_GCTR)
1998ff27e85aSGilad Ben-Yossef cc_gcm(req, desc, &seq_len);
1999ff27e85aSGilad Ben-Yossef break;
2000ff27e85aSGilad Ben-Yossef default:
2001ff27e85aSGilad Ben-Yossef dev_err(dev, "Unsupported authenc (%d)\n", ctx->auth_mode);
2002ff27e85aSGilad Ben-Yossef cc_unmap_aead_request(dev, req);
2003ff27e85aSGilad Ben-Yossef rc = -ENOTSUPP;
2004ff27e85aSGilad Ben-Yossef goto exit;
2005ff27e85aSGilad Ben-Yossef }
2006ff27e85aSGilad Ben-Yossef
2007ff27e85aSGilad Ben-Yossef /* STAT_PHASE_3: Lock HW and push sequence */
2008ff27e85aSGilad Ben-Yossef
2009ff27e85aSGilad Ben-Yossef rc = cc_send_request(ctx->drvdata, &cc_req, desc, seq_len, &req->base);
2010ff27e85aSGilad Ben-Yossef
2011ff27e85aSGilad Ben-Yossef if (rc != -EINPROGRESS && rc != -EBUSY) {
2012ff27e85aSGilad Ben-Yossef dev_err(dev, "send_request() failed (rc=%d)\n", rc);
2013ff27e85aSGilad Ben-Yossef cc_unmap_aead_request(dev, req);
2014ff27e85aSGilad Ben-Yossef }
2015ff27e85aSGilad Ben-Yossef
2016ff27e85aSGilad Ben-Yossef exit:
2017ff27e85aSGilad Ben-Yossef return rc;
2018ff27e85aSGilad Ben-Yossef }
2019ff27e85aSGilad Ben-Yossef
cc_aead_encrypt(struct aead_request * req)2020ff27e85aSGilad Ben-Yossef static int cc_aead_encrypt(struct aead_request *req)
2021ff27e85aSGilad Ben-Yossef {
2022*07547fa7SHerbert Xu struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(req);
2023ff27e85aSGilad Ben-Yossef int rc;
2024ff27e85aSGilad Ben-Yossef
20259f31eb6eSGilad Ben-Yossef memset(areq_ctx, 0, sizeof(*areq_ctx));
20269f31eb6eSGilad Ben-Yossef
2027ff27e85aSGilad Ben-Yossef /* No generated IV required */
2028ff27e85aSGilad Ben-Yossef areq_ctx->backup_iv = req->iv;
2029da3cf67fSGilad Ben-Yossef areq_ctx->assoclen = req->assoclen;
2030ff27e85aSGilad Ben-Yossef
2031ff27e85aSGilad Ben-Yossef rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_ENCRYPT);
2032ff27e85aSGilad Ben-Yossef if (rc != -EINPROGRESS && rc != -EBUSY)
2033ff27e85aSGilad Ben-Yossef req->iv = areq_ctx->backup_iv;
2034ff27e85aSGilad Ben-Yossef
2035ff27e85aSGilad Ben-Yossef return rc;
2036ff27e85aSGilad Ben-Yossef }
2037ff27e85aSGilad Ben-Yossef
cc_rfc4309_ccm_encrypt(struct aead_request * req)2038ff27e85aSGilad Ben-Yossef static int cc_rfc4309_ccm_encrypt(struct aead_request *req)
2039ff27e85aSGilad Ben-Yossef {
2040ff27e85aSGilad Ben-Yossef /* Very similar to cc_aead_encrypt() above. */
2041ff27e85aSGilad Ben-Yossef
2042*07547fa7SHerbert Xu struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(req);
2043b66c1876SGilad Ben-Yossef int rc;
2044ff27e85aSGilad Ben-Yossef
2045b66c1876SGilad Ben-Yossef rc = crypto_ipsec_check_assoclen(req->assoclen);
2046b66c1876SGilad Ben-Yossef if (rc)
2047ff27e85aSGilad Ben-Yossef goto out;
2048ff27e85aSGilad Ben-Yossef
20499f31eb6eSGilad Ben-Yossef memset(areq_ctx, 0, sizeof(*areq_ctx));
20509f31eb6eSGilad Ben-Yossef
2051ff27e85aSGilad Ben-Yossef /* No generated IV required */
2052ff27e85aSGilad Ben-Yossef areq_ctx->backup_iv = req->iv;
20530eae14a0SGilad Ben-Yossef areq_ctx->assoclen = req->assoclen - CCM_BLOCK_IV_SIZE;
2054ff27e85aSGilad Ben-Yossef
2055ff27e85aSGilad Ben-Yossef cc_proc_rfc4309_ccm(req);
2056ff27e85aSGilad Ben-Yossef
2057ff27e85aSGilad Ben-Yossef rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_ENCRYPT);
2058ff27e85aSGilad Ben-Yossef if (rc != -EINPROGRESS && rc != -EBUSY)
2059ff27e85aSGilad Ben-Yossef req->iv = areq_ctx->backup_iv;
2060ff27e85aSGilad Ben-Yossef out:
2061ff27e85aSGilad Ben-Yossef return rc;
2062ff27e85aSGilad Ben-Yossef }
2063ff27e85aSGilad Ben-Yossef
cc_aead_decrypt(struct aead_request * req)2064ff27e85aSGilad Ben-Yossef static int cc_aead_decrypt(struct aead_request *req)
2065ff27e85aSGilad Ben-Yossef {
2066*07547fa7SHerbert Xu struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(req);
2067ff27e85aSGilad Ben-Yossef int rc;
2068ff27e85aSGilad Ben-Yossef
20699f31eb6eSGilad Ben-Yossef memset(areq_ctx, 0, sizeof(*areq_ctx));
20709f31eb6eSGilad Ben-Yossef
2071ff27e85aSGilad Ben-Yossef /* No generated IV required */
2072ff27e85aSGilad Ben-Yossef areq_ctx->backup_iv = req->iv;
2073da3cf67fSGilad Ben-Yossef areq_ctx->assoclen = req->assoclen;
2074ff27e85aSGilad Ben-Yossef
2075ff27e85aSGilad Ben-Yossef rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_DECRYPT);
2076ff27e85aSGilad Ben-Yossef if (rc != -EINPROGRESS && rc != -EBUSY)
2077ff27e85aSGilad Ben-Yossef req->iv = areq_ctx->backup_iv;
2078ff27e85aSGilad Ben-Yossef
2079ff27e85aSGilad Ben-Yossef return rc;
2080ff27e85aSGilad Ben-Yossef }
2081ff27e85aSGilad Ben-Yossef
cc_rfc4309_ccm_decrypt(struct aead_request * req)2082ff27e85aSGilad Ben-Yossef static int cc_rfc4309_ccm_decrypt(struct aead_request *req)
2083ff27e85aSGilad Ben-Yossef {
2084*07547fa7SHerbert Xu struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(req);
2085b66c1876SGilad Ben-Yossef int rc;
2086ff27e85aSGilad Ben-Yossef
2087b66c1876SGilad Ben-Yossef rc = crypto_ipsec_check_assoclen(req->assoclen);
2088b66c1876SGilad Ben-Yossef if (rc)
2089ff27e85aSGilad Ben-Yossef goto out;
2090ff27e85aSGilad Ben-Yossef
20919f31eb6eSGilad Ben-Yossef memset(areq_ctx, 0, sizeof(*areq_ctx));
20929f31eb6eSGilad Ben-Yossef
2093ff27e85aSGilad Ben-Yossef /* No generated IV required */
2094ff27e85aSGilad Ben-Yossef areq_ctx->backup_iv = req->iv;
20950eae14a0SGilad Ben-Yossef areq_ctx->assoclen = req->assoclen - CCM_BLOCK_IV_SIZE;
2096ff27e85aSGilad Ben-Yossef
2097ff27e85aSGilad Ben-Yossef cc_proc_rfc4309_ccm(req);
2098ff27e85aSGilad Ben-Yossef
2099ff27e85aSGilad Ben-Yossef rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_DECRYPT);
2100ff27e85aSGilad Ben-Yossef if (rc != -EINPROGRESS && rc != -EBUSY)
2101ff27e85aSGilad Ben-Yossef req->iv = areq_ctx->backup_iv;
2102ff27e85aSGilad Ben-Yossef
2103ff27e85aSGilad Ben-Yossef out:
2104ff27e85aSGilad Ben-Yossef return rc;
2105ff27e85aSGilad Ben-Yossef }
2106ff27e85aSGilad Ben-Yossef
cc_rfc4106_gcm_setkey(struct crypto_aead * tfm,const u8 * key,unsigned int keylen)2107ff27e85aSGilad Ben-Yossef static int cc_rfc4106_gcm_setkey(struct crypto_aead *tfm, const u8 *key,
2108ff27e85aSGilad Ben-Yossef unsigned int keylen)
2109ff27e85aSGilad Ben-Yossef {
2110ff27e85aSGilad Ben-Yossef struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
2111ff27e85aSGilad Ben-Yossef struct device *dev = drvdata_to_dev(ctx->drvdata);
2112ff27e85aSGilad Ben-Yossef
2113ff27e85aSGilad Ben-Yossef dev_dbg(dev, "%s() keylen %d, key %p\n", __func__, keylen, key);
2114ff27e85aSGilad Ben-Yossef
2115ff27e85aSGilad Ben-Yossef if (keylen < 4)
2116ff27e85aSGilad Ben-Yossef return -EINVAL;
2117ff27e85aSGilad Ben-Yossef
2118ff27e85aSGilad Ben-Yossef keylen -= 4;
2119ff27e85aSGilad Ben-Yossef memcpy(ctx->ctr_nonce, key + keylen, 4);
2120ff27e85aSGilad Ben-Yossef
2121ff27e85aSGilad Ben-Yossef return cc_aead_setkey(tfm, key, keylen);
2122ff27e85aSGilad Ben-Yossef }
2123ff27e85aSGilad Ben-Yossef
cc_rfc4543_gcm_setkey(struct crypto_aead * tfm,const u8 * key,unsigned int keylen)2124ff27e85aSGilad Ben-Yossef static int cc_rfc4543_gcm_setkey(struct crypto_aead *tfm, const u8 *key,
2125ff27e85aSGilad Ben-Yossef unsigned int keylen)
2126ff27e85aSGilad Ben-Yossef {
2127ff27e85aSGilad Ben-Yossef struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
2128ff27e85aSGilad Ben-Yossef struct device *dev = drvdata_to_dev(ctx->drvdata);
2129ff27e85aSGilad Ben-Yossef
2130ff27e85aSGilad Ben-Yossef dev_dbg(dev, "%s() keylen %d, key %p\n", __func__, keylen, key);
2131ff27e85aSGilad Ben-Yossef
2132ff27e85aSGilad Ben-Yossef if (keylen < 4)
2133ff27e85aSGilad Ben-Yossef return -EINVAL;
2134ff27e85aSGilad Ben-Yossef
2135ff27e85aSGilad Ben-Yossef keylen -= 4;
2136ff27e85aSGilad Ben-Yossef memcpy(ctx->ctr_nonce, key + keylen, 4);
2137ff27e85aSGilad Ben-Yossef
2138ff27e85aSGilad Ben-Yossef return cc_aead_setkey(tfm, key, keylen);
2139ff27e85aSGilad Ben-Yossef }
2140ff27e85aSGilad Ben-Yossef
cc_gcm_setauthsize(struct crypto_aead * authenc,unsigned int authsize)2141ff27e85aSGilad Ben-Yossef static int cc_gcm_setauthsize(struct crypto_aead *authenc,
2142ff27e85aSGilad Ben-Yossef unsigned int authsize)
2143ff27e85aSGilad Ben-Yossef {
2144ff27e85aSGilad Ben-Yossef switch (authsize) {
2145ff27e85aSGilad Ben-Yossef case 4:
2146ff27e85aSGilad Ben-Yossef case 8:
2147ff27e85aSGilad Ben-Yossef case 12:
2148ff27e85aSGilad Ben-Yossef case 13:
2149ff27e85aSGilad Ben-Yossef case 14:
2150ff27e85aSGilad Ben-Yossef case 15:
2151ff27e85aSGilad Ben-Yossef case 16:
2152ff27e85aSGilad Ben-Yossef break;
2153ff27e85aSGilad Ben-Yossef default:
2154ff27e85aSGilad Ben-Yossef return -EINVAL;
2155ff27e85aSGilad Ben-Yossef }
2156ff27e85aSGilad Ben-Yossef
2157ff27e85aSGilad Ben-Yossef return cc_aead_setauthsize(authenc, authsize);
2158ff27e85aSGilad Ben-Yossef }
2159ff27e85aSGilad Ben-Yossef
cc_rfc4106_gcm_setauthsize(struct crypto_aead * authenc,unsigned int authsize)2160ff27e85aSGilad Ben-Yossef static int cc_rfc4106_gcm_setauthsize(struct crypto_aead *authenc,
2161ff27e85aSGilad Ben-Yossef unsigned int authsize)
2162ff27e85aSGilad Ben-Yossef {
2163ff27e85aSGilad Ben-Yossef struct cc_aead_ctx *ctx = crypto_aead_ctx(authenc);
2164ff27e85aSGilad Ben-Yossef struct device *dev = drvdata_to_dev(ctx->drvdata);
2165ff27e85aSGilad Ben-Yossef
2166ff27e85aSGilad Ben-Yossef dev_dbg(dev, "authsize %d\n", authsize);
2167ff27e85aSGilad Ben-Yossef
2168ff27e85aSGilad Ben-Yossef switch (authsize) {
2169ff27e85aSGilad Ben-Yossef case 8:
2170ff27e85aSGilad Ben-Yossef case 12:
2171ff27e85aSGilad Ben-Yossef case 16:
2172ff27e85aSGilad Ben-Yossef break;
2173ff27e85aSGilad Ben-Yossef default:
2174ff27e85aSGilad Ben-Yossef return -EINVAL;
2175ff27e85aSGilad Ben-Yossef }
2176ff27e85aSGilad Ben-Yossef
2177ff27e85aSGilad Ben-Yossef return cc_aead_setauthsize(authenc, authsize);
2178ff27e85aSGilad Ben-Yossef }
2179ff27e85aSGilad Ben-Yossef
cc_rfc4543_gcm_setauthsize(struct crypto_aead * authenc,unsigned int authsize)2180ff27e85aSGilad Ben-Yossef static int cc_rfc4543_gcm_setauthsize(struct crypto_aead *authenc,
2181ff27e85aSGilad Ben-Yossef unsigned int authsize)
2182ff27e85aSGilad Ben-Yossef {
2183ff27e85aSGilad Ben-Yossef struct cc_aead_ctx *ctx = crypto_aead_ctx(authenc);
2184ff27e85aSGilad Ben-Yossef struct device *dev = drvdata_to_dev(ctx->drvdata);
2185ff27e85aSGilad Ben-Yossef
2186ff27e85aSGilad Ben-Yossef dev_dbg(dev, "authsize %d\n", authsize);
2187ff27e85aSGilad Ben-Yossef
2188ff27e85aSGilad Ben-Yossef if (authsize != 16)
2189ff27e85aSGilad Ben-Yossef return -EINVAL;
2190ff27e85aSGilad Ben-Yossef
2191ff27e85aSGilad Ben-Yossef return cc_aead_setauthsize(authenc, authsize);
2192ff27e85aSGilad Ben-Yossef }
2193ff27e85aSGilad Ben-Yossef
cc_rfc4106_gcm_encrypt(struct aead_request * req)2194ff27e85aSGilad Ben-Yossef static int cc_rfc4106_gcm_encrypt(struct aead_request *req)
2195ff27e85aSGilad Ben-Yossef {
2196*07547fa7SHerbert Xu struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(req);
2197b66c1876SGilad Ben-Yossef int rc;
2198ff27e85aSGilad Ben-Yossef
2199b66c1876SGilad Ben-Yossef rc = crypto_ipsec_check_assoclen(req->assoclen);
2200b66c1876SGilad Ben-Yossef if (rc)
2201ff27e85aSGilad Ben-Yossef goto out;
2202ff27e85aSGilad Ben-Yossef
22039f31eb6eSGilad Ben-Yossef memset(areq_ctx, 0, sizeof(*areq_ctx));
22049f31eb6eSGilad Ben-Yossef
2205ff27e85aSGilad Ben-Yossef /* No generated IV required */
2206ff27e85aSGilad Ben-Yossef areq_ctx->backup_iv = req->iv;
22070eae14a0SGilad Ben-Yossef areq_ctx->assoclen = req->assoclen - GCM_BLOCK_RFC4_IV_SIZE;
2208ff27e85aSGilad Ben-Yossef
2209ff27e85aSGilad Ben-Yossef cc_proc_rfc4_gcm(req);
2210ff27e85aSGilad Ben-Yossef
2211ff27e85aSGilad Ben-Yossef rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_ENCRYPT);
2212ff27e85aSGilad Ben-Yossef if (rc != -EINPROGRESS && rc != -EBUSY)
2213ff27e85aSGilad Ben-Yossef req->iv = areq_ctx->backup_iv;
2214ff27e85aSGilad Ben-Yossef out:
2215ff27e85aSGilad Ben-Yossef return rc;
2216ff27e85aSGilad Ben-Yossef }
2217ff27e85aSGilad Ben-Yossef
cc_rfc4543_gcm_encrypt(struct aead_request * req)2218ff27e85aSGilad Ben-Yossef static int cc_rfc4543_gcm_encrypt(struct aead_request *req)
2219ff27e85aSGilad Ben-Yossef {
2220*07547fa7SHerbert Xu struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(req);
2221b66c1876SGilad Ben-Yossef int rc;
2222b93ecf42SIuliana Prodan
2223b66c1876SGilad Ben-Yossef rc = crypto_ipsec_check_assoclen(req->assoclen);
2224b66c1876SGilad Ben-Yossef if (rc)
2225b93ecf42SIuliana Prodan goto out;
2226ff27e85aSGilad Ben-Yossef
22279f31eb6eSGilad Ben-Yossef memset(areq_ctx, 0, sizeof(*areq_ctx));
22289f31eb6eSGilad Ben-Yossef
2229ff27e85aSGilad Ben-Yossef //plaintext is not encryped with rfc4543
2230ff27e85aSGilad Ben-Yossef areq_ctx->plaintext_authenticate_only = true;
2231ff27e85aSGilad Ben-Yossef
2232ff27e85aSGilad Ben-Yossef /* No generated IV required */
2233ff27e85aSGilad Ben-Yossef areq_ctx->backup_iv = req->iv;
2234da3cf67fSGilad Ben-Yossef areq_ctx->assoclen = req->assoclen;
2235ff27e85aSGilad Ben-Yossef
2236ff27e85aSGilad Ben-Yossef cc_proc_rfc4_gcm(req);
2237ff27e85aSGilad Ben-Yossef
2238ff27e85aSGilad Ben-Yossef rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_ENCRYPT);
2239ff27e85aSGilad Ben-Yossef if (rc != -EINPROGRESS && rc != -EBUSY)
2240ff27e85aSGilad Ben-Yossef req->iv = areq_ctx->backup_iv;
2241b93ecf42SIuliana Prodan out:
2242ff27e85aSGilad Ben-Yossef return rc;
2243ff27e85aSGilad Ben-Yossef }
2244ff27e85aSGilad Ben-Yossef
cc_rfc4106_gcm_decrypt(struct aead_request * req)2245ff27e85aSGilad Ben-Yossef static int cc_rfc4106_gcm_decrypt(struct aead_request *req)
2246ff27e85aSGilad Ben-Yossef {
2247*07547fa7SHerbert Xu struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(req);
2248b66c1876SGilad Ben-Yossef int rc;
2249ff27e85aSGilad Ben-Yossef
2250b66c1876SGilad Ben-Yossef rc = crypto_ipsec_check_assoclen(req->assoclen);
2251b66c1876SGilad Ben-Yossef if (rc)
2252ff27e85aSGilad Ben-Yossef goto out;
2253ff27e85aSGilad Ben-Yossef
22549f31eb6eSGilad Ben-Yossef memset(areq_ctx, 0, sizeof(*areq_ctx));
22559f31eb6eSGilad Ben-Yossef
2256ff27e85aSGilad Ben-Yossef /* No generated IV required */
2257ff27e85aSGilad Ben-Yossef areq_ctx->backup_iv = req->iv;
22580eae14a0SGilad Ben-Yossef areq_ctx->assoclen = req->assoclen - GCM_BLOCK_RFC4_IV_SIZE;
2259ff27e85aSGilad Ben-Yossef
2260ff27e85aSGilad Ben-Yossef cc_proc_rfc4_gcm(req);
2261ff27e85aSGilad Ben-Yossef
2262ff27e85aSGilad Ben-Yossef rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_DECRYPT);
2263ff27e85aSGilad Ben-Yossef if (rc != -EINPROGRESS && rc != -EBUSY)
2264ff27e85aSGilad Ben-Yossef req->iv = areq_ctx->backup_iv;
2265ff27e85aSGilad Ben-Yossef out:
2266ff27e85aSGilad Ben-Yossef return rc;
2267ff27e85aSGilad Ben-Yossef }
2268ff27e85aSGilad Ben-Yossef
cc_rfc4543_gcm_decrypt(struct aead_request * req)2269ff27e85aSGilad Ben-Yossef static int cc_rfc4543_gcm_decrypt(struct aead_request *req)
2270ff27e85aSGilad Ben-Yossef {
2271*07547fa7SHerbert Xu struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(req);
2272b66c1876SGilad Ben-Yossef int rc;
2273b93ecf42SIuliana Prodan
2274b66c1876SGilad Ben-Yossef rc = crypto_ipsec_check_assoclen(req->assoclen);
2275b66c1876SGilad Ben-Yossef if (rc)
2276b93ecf42SIuliana Prodan goto out;
2277ff27e85aSGilad Ben-Yossef
22789f31eb6eSGilad Ben-Yossef memset(areq_ctx, 0, sizeof(*areq_ctx));
22799f31eb6eSGilad Ben-Yossef
2280ff27e85aSGilad Ben-Yossef //plaintext is not decryped with rfc4543
2281ff27e85aSGilad Ben-Yossef areq_ctx->plaintext_authenticate_only = true;
2282ff27e85aSGilad Ben-Yossef
2283ff27e85aSGilad Ben-Yossef /* No generated IV required */
2284ff27e85aSGilad Ben-Yossef areq_ctx->backup_iv = req->iv;
2285da3cf67fSGilad Ben-Yossef areq_ctx->assoclen = req->assoclen;
2286ff27e85aSGilad Ben-Yossef
2287ff27e85aSGilad Ben-Yossef cc_proc_rfc4_gcm(req);
2288ff27e85aSGilad Ben-Yossef
2289ff27e85aSGilad Ben-Yossef rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_DECRYPT);
2290ff27e85aSGilad Ben-Yossef if (rc != -EINPROGRESS && rc != -EBUSY)
2291ff27e85aSGilad Ben-Yossef req->iv = areq_ctx->backup_iv;
2292b93ecf42SIuliana Prodan out:
2293ff27e85aSGilad Ben-Yossef return rc;
2294ff27e85aSGilad Ben-Yossef }
2295ff27e85aSGilad Ben-Yossef
2296ff27e85aSGilad Ben-Yossef /* aead alg */
2297ff27e85aSGilad Ben-Yossef static struct cc_alg_template aead_algs[] = {
2298ff27e85aSGilad Ben-Yossef {
2299ff27e85aSGilad Ben-Yossef .name = "authenc(hmac(sha1),cbc(aes))",
2300ff27e85aSGilad Ben-Yossef .driver_name = "authenc-hmac-sha1-cbc-aes-ccree",
2301ff27e85aSGilad Ben-Yossef .blocksize = AES_BLOCK_SIZE,
2302ff27e85aSGilad Ben-Yossef .template_aead = {
2303ff27e85aSGilad Ben-Yossef .setkey = cc_aead_setkey,
2304ff27e85aSGilad Ben-Yossef .setauthsize = cc_aead_setauthsize,
2305ff27e85aSGilad Ben-Yossef .encrypt = cc_aead_encrypt,
2306ff27e85aSGilad Ben-Yossef .decrypt = cc_aead_decrypt,
2307ff27e85aSGilad Ben-Yossef .init = cc_aead_init,
2308ff27e85aSGilad Ben-Yossef .exit = cc_aead_exit,
2309ff27e85aSGilad Ben-Yossef .ivsize = AES_BLOCK_SIZE,
2310ff27e85aSGilad Ben-Yossef .maxauthsize = SHA1_DIGEST_SIZE,
2311ff27e85aSGilad Ben-Yossef },
2312ff27e85aSGilad Ben-Yossef .cipher_mode = DRV_CIPHER_CBC,
2313ff27e85aSGilad Ben-Yossef .flow_mode = S_DIN_to_AES,
2314ff27e85aSGilad Ben-Yossef .auth_mode = DRV_HASH_SHA1,
231527b3b22dSGilad Ben-Yossef .min_hw_rev = CC_HW_REV_630,
23161c876a90SGilad Ben-Yossef .std_body = CC_STD_NIST,
2317ff27e85aSGilad Ben-Yossef },
2318ff27e85aSGilad Ben-Yossef {
2319ff27e85aSGilad Ben-Yossef .name = "authenc(hmac(sha1),cbc(des3_ede))",
2320ff27e85aSGilad Ben-Yossef .driver_name = "authenc-hmac-sha1-cbc-des3-ccree",
2321ff27e85aSGilad Ben-Yossef .blocksize = DES3_EDE_BLOCK_SIZE,
2322ff27e85aSGilad Ben-Yossef .template_aead = {
23239fbfcefcSHerbert Xu .setkey = cc_des3_aead_setkey,
2324ff27e85aSGilad Ben-Yossef .setauthsize = cc_aead_setauthsize,
2325ff27e85aSGilad Ben-Yossef .encrypt = cc_aead_encrypt,
2326ff27e85aSGilad Ben-Yossef .decrypt = cc_aead_decrypt,
2327ff27e85aSGilad Ben-Yossef .init = cc_aead_init,
2328ff27e85aSGilad Ben-Yossef .exit = cc_aead_exit,
2329ff27e85aSGilad Ben-Yossef .ivsize = DES3_EDE_BLOCK_SIZE,
2330ff27e85aSGilad Ben-Yossef .maxauthsize = SHA1_DIGEST_SIZE,
2331ff27e85aSGilad Ben-Yossef },
2332ff27e85aSGilad Ben-Yossef .cipher_mode = DRV_CIPHER_CBC,
2333ff27e85aSGilad Ben-Yossef .flow_mode = S_DIN_to_DES,
2334ff27e85aSGilad Ben-Yossef .auth_mode = DRV_HASH_SHA1,
233527b3b22dSGilad Ben-Yossef .min_hw_rev = CC_HW_REV_630,
23361c876a90SGilad Ben-Yossef .std_body = CC_STD_NIST,
2337ff27e85aSGilad Ben-Yossef },
2338ff27e85aSGilad Ben-Yossef {
2339ff27e85aSGilad Ben-Yossef .name = "authenc(hmac(sha256),cbc(aes))",
2340ff27e85aSGilad Ben-Yossef .driver_name = "authenc-hmac-sha256-cbc-aes-ccree",
2341ff27e85aSGilad Ben-Yossef .blocksize = AES_BLOCK_SIZE,
2342ff27e85aSGilad Ben-Yossef .template_aead = {
2343ff27e85aSGilad Ben-Yossef .setkey = cc_aead_setkey,
2344ff27e85aSGilad Ben-Yossef .setauthsize = cc_aead_setauthsize,
2345ff27e85aSGilad Ben-Yossef .encrypt = cc_aead_encrypt,
2346ff27e85aSGilad Ben-Yossef .decrypt = cc_aead_decrypt,
2347ff27e85aSGilad Ben-Yossef .init = cc_aead_init,
2348ff27e85aSGilad Ben-Yossef .exit = cc_aead_exit,
2349ff27e85aSGilad Ben-Yossef .ivsize = AES_BLOCK_SIZE,
2350ff27e85aSGilad Ben-Yossef .maxauthsize = SHA256_DIGEST_SIZE,
2351ff27e85aSGilad Ben-Yossef },
2352ff27e85aSGilad Ben-Yossef .cipher_mode = DRV_CIPHER_CBC,
2353ff27e85aSGilad Ben-Yossef .flow_mode = S_DIN_to_AES,
2354ff27e85aSGilad Ben-Yossef .auth_mode = DRV_HASH_SHA256,
235527b3b22dSGilad Ben-Yossef .min_hw_rev = CC_HW_REV_630,
23561c876a90SGilad Ben-Yossef .std_body = CC_STD_NIST,
2357ff27e85aSGilad Ben-Yossef },
2358ff27e85aSGilad Ben-Yossef {
2359ff27e85aSGilad Ben-Yossef .name = "authenc(hmac(sha256),cbc(des3_ede))",
2360ff27e85aSGilad Ben-Yossef .driver_name = "authenc-hmac-sha256-cbc-des3-ccree",
2361ff27e85aSGilad Ben-Yossef .blocksize = DES3_EDE_BLOCK_SIZE,
2362ff27e85aSGilad Ben-Yossef .template_aead = {
23639fbfcefcSHerbert Xu .setkey = cc_des3_aead_setkey,
2364ff27e85aSGilad Ben-Yossef .setauthsize = cc_aead_setauthsize,
2365ff27e85aSGilad Ben-Yossef .encrypt = cc_aead_encrypt,
2366ff27e85aSGilad Ben-Yossef .decrypt = cc_aead_decrypt,
2367ff27e85aSGilad Ben-Yossef .init = cc_aead_init,
2368ff27e85aSGilad Ben-Yossef .exit = cc_aead_exit,
2369ff27e85aSGilad Ben-Yossef .ivsize = DES3_EDE_BLOCK_SIZE,
2370ff27e85aSGilad Ben-Yossef .maxauthsize = SHA256_DIGEST_SIZE,
2371ff27e85aSGilad Ben-Yossef },
2372ff27e85aSGilad Ben-Yossef .cipher_mode = DRV_CIPHER_CBC,
2373ff27e85aSGilad Ben-Yossef .flow_mode = S_DIN_to_DES,
2374ff27e85aSGilad Ben-Yossef .auth_mode = DRV_HASH_SHA256,
237527b3b22dSGilad Ben-Yossef .min_hw_rev = CC_HW_REV_630,
23761c876a90SGilad Ben-Yossef .std_body = CC_STD_NIST,
2377ff27e85aSGilad Ben-Yossef },
2378ff27e85aSGilad Ben-Yossef {
2379ff27e85aSGilad Ben-Yossef .name = "authenc(xcbc(aes),cbc(aes))",
2380ff27e85aSGilad Ben-Yossef .driver_name = "authenc-xcbc-aes-cbc-aes-ccree",
2381ff27e85aSGilad Ben-Yossef .blocksize = AES_BLOCK_SIZE,
2382ff27e85aSGilad Ben-Yossef .template_aead = {
2383ff27e85aSGilad Ben-Yossef .setkey = cc_aead_setkey,
2384ff27e85aSGilad Ben-Yossef .setauthsize = cc_aead_setauthsize,
2385ff27e85aSGilad Ben-Yossef .encrypt = cc_aead_encrypt,
2386ff27e85aSGilad Ben-Yossef .decrypt = cc_aead_decrypt,
2387ff27e85aSGilad Ben-Yossef .init = cc_aead_init,
2388ff27e85aSGilad Ben-Yossef .exit = cc_aead_exit,
2389ff27e85aSGilad Ben-Yossef .ivsize = AES_BLOCK_SIZE,
2390ff27e85aSGilad Ben-Yossef .maxauthsize = AES_BLOCK_SIZE,
2391ff27e85aSGilad Ben-Yossef },
2392ff27e85aSGilad Ben-Yossef .cipher_mode = DRV_CIPHER_CBC,
2393ff27e85aSGilad Ben-Yossef .flow_mode = S_DIN_to_AES,
2394ff27e85aSGilad Ben-Yossef .auth_mode = DRV_HASH_XCBC_MAC,
239527b3b22dSGilad Ben-Yossef .min_hw_rev = CC_HW_REV_630,
23961c876a90SGilad Ben-Yossef .std_body = CC_STD_NIST,
2397ff27e85aSGilad Ben-Yossef },
2398ff27e85aSGilad Ben-Yossef {
2399ff27e85aSGilad Ben-Yossef .name = "authenc(hmac(sha1),rfc3686(ctr(aes)))",
2400ff27e85aSGilad Ben-Yossef .driver_name = "authenc-hmac-sha1-rfc3686-ctr-aes-ccree",
2401ff27e85aSGilad Ben-Yossef .blocksize = 1,
2402ff27e85aSGilad Ben-Yossef .template_aead = {
2403ff27e85aSGilad Ben-Yossef .setkey = cc_aead_setkey,
2404ff27e85aSGilad Ben-Yossef .setauthsize = cc_aead_setauthsize,
2405ff27e85aSGilad Ben-Yossef .encrypt = cc_aead_encrypt,
2406ff27e85aSGilad Ben-Yossef .decrypt = cc_aead_decrypt,
2407ff27e85aSGilad Ben-Yossef .init = cc_aead_init,
2408ff27e85aSGilad Ben-Yossef .exit = cc_aead_exit,
2409ff27e85aSGilad Ben-Yossef .ivsize = CTR_RFC3686_IV_SIZE,
2410ff27e85aSGilad Ben-Yossef .maxauthsize = SHA1_DIGEST_SIZE,
2411ff27e85aSGilad Ben-Yossef },
2412ff27e85aSGilad Ben-Yossef .cipher_mode = DRV_CIPHER_CTR,
2413ff27e85aSGilad Ben-Yossef .flow_mode = S_DIN_to_AES,
2414ff27e85aSGilad Ben-Yossef .auth_mode = DRV_HASH_SHA1,
241527b3b22dSGilad Ben-Yossef .min_hw_rev = CC_HW_REV_630,
24161c876a90SGilad Ben-Yossef .std_body = CC_STD_NIST,
2417ff27e85aSGilad Ben-Yossef },
2418ff27e85aSGilad Ben-Yossef {
2419ff27e85aSGilad Ben-Yossef .name = "authenc(hmac(sha256),rfc3686(ctr(aes)))",
2420ff27e85aSGilad Ben-Yossef .driver_name = "authenc-hmac-sha256-rfc3686-ctr-aes-ccree",
2421ff27e85aSGilad Ben-Yossef .blocksize = 1,
2422ff27e85aSGilad Ben-Yossef .template_aead = {
2423ff27e85aSGilad Ben-Yossef .setkey = cc_aead_setkey,
2424ff27e85aSGilad Ben-Yossef .setauthsize = cc_aead_setauthsize,
2425ff27e85aSGilad Ben-Yossef .encrypt = cc_aead_encrypt,
2426ff27e85aSGilad Ben-Yossef .decrypt = cc_aead_decrypt,
2427ff27e85aSGilad Ben-Yossef .init = cc_aead_init,
2428ff27e85aSGilad Ben-Yossef .exit = cc_aead_exit,
2429ff27e85aSGilad Ben-Yossef .ivsize = CTR_RFC3686_IV_SIZE,
2430ff27e85aSGilad Ben-Yossef .maxauthsize = SHA256_DIGEST_SIZE,
2431ff27e85aSGilad Ben-Yossef },
2432ff27e85aSGilad Ben-Yossef .cipher_mode = DRV_CIPHER_CTR,
2433ff27e85aSGilad Ben-Yossef .flow_mode = S_DIN_to_AES,
2434ff27e85aSGilad Ben-Yossef .auth_mode = DRV_HASH_SHA256,
243527b3b22dSGilad Ben-Yossef .min_hw_rev = CC_HW_REV_630,
24361c876a90SGilad Ben-Yossef .std_body = CC_STD_NIST,
2437ff27e85aSGilad Ben-Yossef },
2438ff27e85aSGilad Ben-Yossef {
2439ff27e85aSGilad Ben-Yossef .name = "authenc(xcbc(aes),rfc3686(ctr(aes)))",
2440ff27e85aSGilad Ben-Yossef .driver_name = "authenc-xcbc-aes-rfc3686-ctr-aes-ccree",
2441ff27e85aSGilad Ben-Yossef .blocksize = 1,
2442ff27e85aSGilad Ben-Yossef .template_aead = {
2443ff27e85aSGilad Ben-Yossef .setkey = cc_aead_setkey,
2444ff27e85aSGilad Ben-Yossef .setauthsize = cc_aead_setauthsize,
2445ff27e85aSGilad Ben-Yossef .encrypt = cc_aead_encrypt,
2446ff27e85aSGilad Ben-Yossef .decrypt = cc_aead_decrypt,
2447ff27e85aSGilad Ben-Yossef .init = cc_aead_init,
2448ff27e85aSGilad Ben-Yossef .exit = cc_aead_exit,
2449ff27e85aSGilad Ben-Yossef .ivsize = CTR_RFC3686_IV_SIZE,
2450ff27e85aSGilad Ben-Yossef .maxauthsize = AES_BLOCK_SIZE,
2451ff27e85aSGilad Ben-Yossef },
2452ff27e85aSGilad Ben-Yossef .cipher_mode = DRV_CIPHER_CTR,
2453ff27e85aSGilad Ben-Yossef .flow_mode = S_DIN_to_AES,
2454ff27e85aSGilad Ben-Yossef .auth_mode = DRV_HASH_XCBC_MAC,
245527b3b22dSGilad Ben-Yossef .min_hw_rev = CC_HW_REV_630,
24561c876a90SGilad Ben-Yossef .std_body = CC_STD_NIST,
2457ff27e85aSGilad Ben-Yossef },
2458ff27e85aSGilad Ben-Yossef {
2459ff27e85aSGilad Ben-Yossef .name = "ccm(aes)",
2460ff27e85aSGilad Ben-Yossef .driver_name = "ccm-aes-ccree",
2461ff27e85aSGilad Ben-Yossef .blocksize = 1,
2462ff27e85aSGilad Ben-Yossef .template_aead = {
2463ff27e85aSGilad Ben-Yossef .setkey = cc_aead_setkey,
2464ff27e85aSGilad Ben-Yossef .setauthsize = cc_ccm_setauthsize,
2465ff27e85aSGilad Ben-Yossef .encrypt = cc_aead_encrypt,
2466ff27e85aSGilad Ben-Yossef .decrypt = cc_aead_decrypt,
2467ff27e85aSGilad Ben-Yossef .init = cc_aead_init,
2468ff27e85aSGilad Ben-Yossef .exit = cc_aead_exit,
2469ff27e85aSGilad Ben-Yossef .ivsize = AES_BLOCK_SIZE,
2470ff27e85aSGilad Ben-Yossef .maxauthsize = AES_BLOCK_SIZE,
2471ff27e85aSGilad Ben-Yossef },
2472ff27e85aSGilad Ben-Yossef .cipher_mode = DRV_CIPHER_CCM,
2473ff27e85aSGilad Ben-Yossef .flow_mode = S_DIN_to_AES,
2474ff27e85aSGilad Ben-Yossef .auth_mode = DRV_HASH_NULL,
247527b3b22dSGilad Ben-Yossef .min_hw_rev = CC_HW_REV_630,
24761c876a90SGilad Ben-Yossef .std_body = CC_STD_NIST,
2477ff27e85aSGilad Ben-Yossef },
2478ff27e85aSGilad Ben-Yossef {
2479ff27e85aSGilad Ben-Yossef .name = "rfc4309(ccm(aes))",
2480ff27e85aSGilad Ben-Yossef .driver_name = "rfc4309-ccm-aes-ccree",
2481ff27e85aSGilad Ben-Yossef .blocksize = 1,
2482ff27e85aSGilad Ben-Yossef .template_aead = {
2483ff27e85aSGilad Ben-Yossef .setkey = cc_rfc4309_ccm_setkey,
2484ff27e85aSGilad Ben-Yossef .setauthsize = cc_rfc4309_ccm_setauthsize,
2485ff27e85aSGilad Ben-Yossef .encrypt = cc_rfc4309_ccm_encrypt,
2486ff27e85aSGilad Ben-Yossef .decrypt = cc_rfc4309_ccm_decrypt,
2487ff27e85aSGilad Ben-Yossef .init = cc_aead_init,
2488ff27e85aSGilad Ben-Yossef .exit = cc_aead_exit,
2489ff27e85aSGilad Ben-Yossef .ivsize = CCM_BLOCK_IV_SIZE,
2490ff27e85aSGilad Ben-Yossef .maxauthsize = AES_BLOCK_SIZE,
2491ff27e85aSGilad Ben-Yossef },
2492ff27e85aSGilad Ben-Yossef .cipher_mode = DRV_CIPHER_CCM,
2493ff27e85aSGilad Ben-Yossef .flow_mode = S_DIN_to_AES,
2494ff27e85aSGilad Ben-Yossef .auth_mode = DRV_HASH_NULL,
249527b3b22dSGilad Ben-Yossef .min_hw_rev = CC_HW_REV_630,
24961c876a90SGilad Ben-Yossef .std_body = CC_STD_NIST,
2497ff27e85aSGilad Ben-Yossef },
2498ff27e85aSGilad Ben-Yossef {
2499ff27e85aSGilad Ben-Yossef .name = "gcm(aes)",
2500ff27e85aSGilad Ben-Yossef .driver_name = "gcm-aes-ccree",
2501ff27e85aSGilad Ben-Yossef .blocksize = 1,
2502ff27e85aSGilad Ben-Yossef .template_aead = {
2503ff27e85aSGilad Ben-Yossef .setkey = cc_aead_setkey,
2504ff27e85aSGilad Ben-Yossef .setauthsize = cc_gcm_setauthsize,
2505ff27e85aSGilad Ben-Yossef .encrypt = cc_aead_encrypt,
2506ff27e85aSGilad Ben-Yossef .decrypt = cc_aead_decrypt,
2507ff27e85aSGilad Ben-Yossef .init = cc_aead_init,
2508ff27e85aSGilad Ben-Yossef .exit = cc_aead_exit,
2509ff27e85aSGilad Ben-Yossef .ivsize = 12,
2510ff27e85aSGilad Ben-Yossef .maxauthsize = AES_BLOCK_SIZE,
2511ff27e85aSGilad Ben-Yossef },
2512ff27e85aSGilad Ben-Yossef .cipher_mode = DRV_CIPHER_GCTR,
2513ff27e85aSGilad Ben-Yossef .flow_mode = S_DIN_to_AES,
2514ff27e85aSGilad Ben-Yossef .auth_mode = DRV_HASH_NULL,
251527b3b22dSGilad Ben-Yossef .min_hw_rev = CC_HW_REV_630,
25161c876a90SGilad Ben-Yossef .std_body = CC_STD_NIST,
2517ff27e85aSGilad Ben-Yossef },
2518ff27e85aSGilad Ben-Yossef {
2519ff27e85aSGilad Ben-Yossef .name = "rfc4106(gcm(aes))",
2520ff27e85aSGilad Ben-Yossef .driver_name = "rfc4106-gcm-aes-ccree",
2521ff27e85aSGilad Ben-Yossef .blocksize = 1,
2522ff27e85aSGilad Ben-Yossef .template_aead = {
2523ff27e85aSGilad Ben-Yossef .setkey = cc_rfc4106_gcm_setkey,
2524ff27e85aSGilad Ben-Yossef .setauthsize = cc_rfc4106_gcm_setauthsize,
2525ff27e85aSGilad Ben-Yossef .encrypt = cc_rfc4106_gcm_encrypt,
2526ff27e85aSGilad Ben-Yossef .decrypt = cc_rfc4106_gcm_decrypt,
2527ff27e85aSGilad Ben-Yossef .init = cc_aead_init,
2528ff27e85aSGilad Ben-Yossef .exit = cc_aead_exit,
2529ff27e85aSGilad Ben-Yossef .ivsize = GCM_BLOCK_RFC4_IV_SIZE,
2530ff27e85aSGilad Ben-Yossef .maxauthsize = AES_BLOCK_SIZE,
2531ff27e85aSGilad Ben-Yossef },
2532ff27e85aSGilad Ben-Yossef .cipher_mode = DRV_CIPHER_GCTR,
2533ff27e85aSGilad Ben-Yossef .flow_mode = S_DIN_to_AES,
2534ff27e85aSGilad Ben-Yossef .auth_mode = DRV_HASH_NULL,
253527b3b22dSGilad Ben-Yossef .min_hw_rev = CC_HW_REV_630,
25361c876a90SGilad Ben-Yossef .std_body = CC_STD_NIST,
2537ff27e85aSGilad Ben-Yossef },
2538ff27e85aSGilad Ben-Yossef {
2539ff27e85aSGilad Ben-Yossef .name = "rfc4543(gcm(aes))",
2540ff27e85aSGilad Ben-Yossef .driver_name = "rfc4543-gcm-aes-ccree",
2541ff27e85aSGilad Ben-Yossef .blocksize = 1,
2542ff27e85aSGilad Ben-Yossef .template_aead = {
2543ff27e85aSGilad Ben-Yossef .setkey = cc_rfc4543_gcm_setkey,
2544ff27e85aSGilad Ben-Yossef .setauthsize = cc_rfc4543_gcm_setauthsize,
2545ff27e85aSGilad Ben-Yossef .encrypt = cc_rfc4543_gcm_encrypt,
2546ff27e85aSGilad Ben-Yossef .decrypt = cc_rfc4543_gcm_decrypt,
2547ff27e85aSGilad Ben-Yossef .init = cc_aead_init,
2548ff27e85aSGilad Ben-Yossef .exit = cc_aead_exit,
2549ff27e85aSGilad Ben-Yossef .ivsize = GCM_BLOCK_RFC4_IV_SIZE,
2550ff27e85aSGilad Ben-Yossef .maxauthsize = AES_BLOCK_SIZE,
2551ff27e85aSGilad Ben-Yossef },
2552ff27e85aSGilad Ben-Yossef .cipher_mode = DRV_CIPHER_GCTR,
2553ff27e85aSGilad Ben-Yossef .flow_mode = S_DIN_to_AES,
2554ff27e85aSGilad Ben-Yossef .auth_mode = DRV_HASH_NULL,
255527b3b22dSGilad Ben-Yossef .min_hw_rev = CC_HW_REV_630,
25561c876a90SGilad Ben-Yossef .std_body = CC_STD_NIST,
2557ff27e85aSGilad Ben-Yossef },
2558ff27e85aSGilad Ben-Yossef };
2559ff27e85aSGilad Ben-Yossef
cc_create_aead_alg(struct cc_alg_template * tmpl,struct device * dev)2560ff27e85aSGilad Ben-Yossef static struct cc_crypto_alg *cc_create_aead_alg(struct cc_alg_template *tmpl,
2561ff27e85aSGilad Ben-Yossef struct device *dev)
2562ff27e85aSGilad Ben-Yossef {
2563ff27e85aSGilad Ben-Yossef struct cc_crypto_alg *t_alg;
2564ff27e85aSGilad Ben-Yossef struct aead_alg *alg;
2565ff27e85aSGilad Ben-Yossef
2566ff4d719aSGeert Uytterhoeven t_alg = devm_kzalloc(dev, sizeof(*t_alg), GFP_KERNEL);
2567ff27e85aSGilad Ben-Yossef if (!t_alg)
2568ff27e85aSGilad Ben-Yossef return ERR_PTR(-ENOMEM);
2569ff27e85aSGilad Ben-Yossef
2570ff27e85aSGilad Ben-Yossef alg = &tmpl->template_aead;
2571ff27e85aSGilad Ben-Yossef
2572ff27e85aSGilad Ben-Yossef snprintf(alg->base.cra_name, CRYPTO_MAX_ALG_NAME, "%s", tmpl->name);
2573ff27e85aSGilad Ben-Yossef snprintf(alg->base.cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s",
2574ff27e85aSGilad Ben-Yossef tmpl->driver_name);
2575ff27e85aSGilad Ben-Yossef alg->base.cra_module = THIS_MODULE;
2576ff27e85aSGilad Ben-Yossef alg->base.cra_priority = CC_CRA_PRIO;
2577ff27e85aSGilad Ben-Yossef
2578ff27e85aSGilad Ben-Yossef alg->base.cra_ctxsize = sizeof(struct cc_aead_ctx);
257976c9e53eSGilad Ben-Yossef alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY;
258021f802ccSGilad Ben-Yossef alg->base.cra_blocksize = tmpl->blocksize;
2581ff27e85aSGilad Ben-Yossef alg->init = cc_aead_init;
2582ff27e85aSGilad Ben-Yossef alg->exit = cc_aead_exit;
2583ff27e85aSGilad Ben-Yossef
2584ff27e85aSGilad Ben-Yossef t_alg->aead_alg = *alg;
2585ff27e85aSGilad Ben-Yossef
2586ff27e85aSGilad Ben-Yossef t_alg->cipher_mode = tmpl->cipher_mode;
2587ff27e85aSGilad Ben-Yossef t_alg->flow_mode = tmpl->flow_mode;
2588ff27e85aSGilad Ben-Yossef t_alg->auth_mode = tmpl->auth_mode;
2589ff27e85aSGilad Ben-Yossef
2590ff27e85aSGilad Ben-Yossef return t_alg;
2591ff27e85aSGilad Ben-Yossef }
2592ff27e85aSGilad Ben-Yossef
cc_aead_free(struct cc_drvdata * drvdata)2593ff27e85aSGilad Ben-Yossef int cc_aead_free(struct cc_drvdata *drvdata)
2594ff27e85aSGilad Ben-Yossef {
2595ff27e85aSGilad Ben-Yossef struct cc_crypto_alg *t_alg, *n;
2596ff4d719aSGeert Uytterhoeven struct cc_aead_handle *aead_handle = drvdata->aead_handle;
2597ff27e85aSGilad Ben-Yossef
2598ff27e85aSGilad Ben-Yossef /* Remove registered algs */
2599ff4d719aSGeert Uytterhoeven list_for_each_entry_safe(t_alg, n, &aead_handle->aead_list, entry) {
2600ff27e85aSGilad Ben-Yossef crypto_unregister_aead(&t_alg->aead_alg);
2601ff27e85aSGilad Ben-Yossef list_del(&t_alg->entry);
2602ff27e85aSGilad Ben-Yossef }
2603ff27e85aSGilad Ben-Yossef
2604ff27e85aSGilad Ben-Yossef return 0;
2605ff27e85aSGilad Ben-Yossef }
2606ff27e85aSGilad Ben-Yossef
cc_aead_alloc(struct cc_drvdata * drvdata)2607ff27e85aSGilad Ben-Yossef int cc_aead_alloc(struct cc_drvdata *drvdata)
2608ff27e85aSGilad Ben-Yossef {
2609ff27e85aSGilad Ben-Yossef struct cc_aead_handle *aead_handle;
2610ff27e85aSGilad Ben-Yossef struct cc_crypto_alg *t_alg;
2611ff27e85aSGilad Ben-Yossef int rc = -ENOMEM;
2612ff27e85aSGilad Ben-Yossef int alg;
2613ff27e85aSGilad Ben-Yossef struct device *dev = drvdata_to_dev(drvdata);
2614ff27e85aSGilad Ben-Yossef
2615ff4d719aSGeert Uytterhoeven aead_handle = devm_kmalloc(dev, sizeof(*aead_handle), GFP_KERNEL);
2616ff27e85aSGilad Ben-Yossef if (!aead_handle) {
2617ff27e85aSGilad Ben-Yossef rc = -ENOMEM;
2618ff27e85aSGilad Ben-Yossef goto fail0;
2619ff27e85aSGilad Ben-Yossef }
2620ff27e85aSGilad Ben-Yossef
2621ff27e85aSGilad Ben-Yossef INIT_LIST_HEAD(&aead_handle->aead_list);
2622ff27e85aSGilad Ben-Yossef drvdata->aead_handle = aead_handle;
2623ff27e85aSGilad Ben-Yossef
2624ff27e85aSGilad Ben-Yossef aead_handle->sram_workspace_addr = cc_sram_alloc(drvdata,
2625ff27e85aSGilad Ben-Yossef MAX_HMAC_DIGEST_SIZE);
2626ff27e85aSGilad Ben-Yossef
2627ff27e85aSGilad Ben-Yossef if (aead_handle->sram_workspace_addr == NULL_SRAM_ADDR) {
2628ff27e85aSGilad Ben-Yossef rc = -ENOMEM;
2629ff27e85aSGilad Ben-Yossef goto fail1;
2630ff27e85aSGilad Ben-Yossef }
2631ff27e85aSGilad Ben-Yossef
2632ff27e85aSGilad Ben-Yossef /* Linux crypto */
2633ff27e85aSGilad Ben-Yossef for (alg = 0; alg < ARRAY_SIZE(aead_algs); alg++) {
26341c876a90SGilad Ben-Yossef if ((aead_algs[alg].min_hw_rev > drvdata->hw_rev) ||
26351c876a90SGilad Ben-Yossef !(drvdata->std_bodies & aead_algs[alg].std_body))
263627b3b22dSGilad Ben-Yossef continue;
263727b3b22dSGilad Ben-Yossef
2638ff27e85aSGilad Ben-Yossef t_alg = cc_create_aead_alg(&aead_algs[alg], dev);
2639ff27e85aSGilad Ben-Yossef if (IS_ERR(t_alg)) {
2640ff27e85aSGilad Ben-Yossef rc = PTR_ERR(t_alg);
2641ff27e85aSGilad Ben-Yossef dev_err(dev, "%s alg allocation failed\n",
2642ff27e85aSGilad Ben-Yossef aead_algs[alg].driver_name);
2643ff27e85aSGilad Ben-Yossef goto fail1;
2644ff27e85aSGilad Ben-Yossef }
2645ff27e85aSGilad Ben-Yossef t_alg->drvdata = drvdata;
2646ff27e85aSGilad Ben-Yossef rc = crypto_register_aead(&t_alg->aead_alg);
2647ff27e85aSGilad Ben-Yossef if (rc) {
2648ff27e85aSGilad Ben-Yossef dev_err(dev, "%s alg registration failed\n",
2649ff27e85aSGilad Ben-Yossef t_alg->aead_alg.base.cra_driver_name);
2650ff4d719aSGeert Uytterhoeven goto fail1;
2651ff4d719aSGeert Uytterhoeven }
2652ff4d719aSGeert Uytterhoeven
2653ff27e85aSGilad Ben-Yossef list_add_tail(&t_alg->entry, &aead_handle->aead_list);
2654ff27e85aSGilad Ben-Yossef dev_dbg(dev, "Registered %s\n",
2655ff27e85aSGilad Ben-Yossef t_alg->aead_alg.base.cra_driver_name);
2656ff27e85aSGilad Ben-Yossef }
2657ff27e85aSGilad Ben-Yossef
2658ff27e85aSGilad Ben-Yossef return 0;
2659ff27e85aSGilad Ben-Yossef
2660ff27e85aSGilad Ben-Yossef fail1:
2661ff27e85aSGilad Ben-Yossef cc_aead_free(drvdata);
2662ff27e85aSGilad Ben-Yossef fail0:
2663ff27e85aSGilad Ben-Yossef return rc;
2664ff27e85aSGilad Ben-Yossef }
2665