1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Intel IXP4xx NPE-C crypto driver
4 *
5 * Copyright (C) 2008 Christian Hohnstaedt <chohnstaedt@innominate.com>
6 */
7
8 #include <linux/platform_device.h>
9 #include <linux/dma-mapping.h>
10 #include <linux/dmapool.h>
11 #include <linux/crypto.h>
12 #include <linux/kernel.h>
13 #include <linux/rtnetlink.h>
14 #include <linux/interrupt.h>
15 #include <linux/spinlock.h>
16 #include <linux/gfp.h>
17 #include <linux/module.h>
18 #include <linux/of.h>
19
20 #include <crypto/ctr.h>
21 #include <crypto/internal/des.h>
22 #include <crypto/aes.h>
23 #include <crypto/hmac.h>
24 #include <crypto/sha1.h>
25 #include <crypto/algapi.h>
26 #include <crypto/internal/aead.h>
27 #include <crypto/internal/skcipher.h>
28 #include <crypto/authenc.h>
29 #include <crypto/scatterwalk.h>
30
31 #include <linux/soc/ixp4xx/npe.h>
32 #include <linux/soc/ixp4xx/qmgr.h>
33
34 /* Intermittent includes, delete this after v5.14-rc1 */
35 #include <linux/soc/ixp4xx/cpu.h>
36
37 #define MAX_KEYLEN 32
38
39 /* hash: cfgword + 2 * digestlen; crypt: keylen + cfgword */
40 #define NPE_CTX_LEN 80
41 #define AES_BLOCK128 16
42
43 #define NPE_OP_HASH_VERIFY 0x01
44 #define NPE_OP_CCM_ENABLE 0x04
45 #define NPE_OP_CRYPT_ENABLE 0x08
46 #define NPE_OP_HASH_ENABLE 0x10
47 #define NPE_OP_NOT_IN_PLACE 0x20
48 #define NPE_OP_HMAC_DISABLE 0x40
49 #define NPE_OP_CRYPT_ENCRYPT 0x80
50
51 #define NPE_OP_CCM_GEN_MIC 0xcc
52 #define NPE_OP_HASH_GEN_ICV 0x50
53 #define NPE_OP_ENC_GEN_KEY 0xc9
54
55 #define MOD_ECB 0x0000
56 #define MOD_CTR 0x1000
57 #define MOD_CBC_ENC 0x2000
58 #define MOD_CBC_DEC 0x3000
59 #define MOD_CCM_ENC 0x4000
60 #define MOD_CCM_DEC 0x5000
61
62 #define KEYLEN_128 4
63 #define KEYLEN_192 6
64 #define KEYLEN_256 8
65
66 #define CIPH_DECR 0x0000
67 #define CIPH_ENCR 0x0400
68
69 #define MOD_DES 0x0000
70 #define MOD_TDEA2 0x0100
71 #define MOD_3DES 0x0200
72 #define MOD_AES 0x0800
73 #define MOD_AES128 (0x0800 | KEYLEN_128)
74 #define MOD_AES192 (0x0900 | KEYLEN_192)
75 #define MOD_AES256 (0x0a00 | KEYLEN_256)
76
77 #define MAX_IVLEN 16
78 #define NPE_QLEN 16
79 /* Space for registering when the first
80 * NPE_QLEN crypt_ctl are busy */
81 #define NPE_QLEN_TOTAL 64
82
83 #define CTL_FLAG_UNUSED 0x0000
84 #define CTL_FLAG_USED 0x1000
85 #define CTL_FLAG_PERFORM_ABLK 0x0001
86 #define CTL_FLAG_GEN_ICV 0x0002
87 #define CTL_FLAG_GEN_REVAES 0x0004
88 #define CTL_FLAG_PERFORM_AEAD 0x0008
89 #define CTL_FLAG_MASK 0x000f
90
91 #define HMAC_PAD_BLOCKLEN SHA1_BLOCK_SIZE
92
93 #define MD5_DIGEST_SIZE 16
94
95 struct buffer_desc {
96 u32 phys_next;
97 #ifdef __ARMEB__
98 u16 buf_len;
99 u16 pkt_len;
100 #else
101 u16 pkt_len;
102 u16 buf_len;
103 #endif
104 dma_addr_t phys_addr;
105 u32 __reserved[4];
106 struct buffer_desc *next;
107 enum dma_data_direction dir;
108 };
109
110 struct crypt_ctl {
111 #ifdef __ARMEB__
112 u8 mode; /* NPE_OP_* operation mode */
113 u8 init_len;
114 u16 reserved;
115 #else
116 u16 reserved;
117 u8 init_len;
118 u8 mode; /* NPE_OP_* operation mode */
119 #endif
120 u8 iv[MAX_IVLEN]; /* IV for CBC mode or CTR IV for CTR mode */
121 u32 icv_rev_aes; /* icv or rev aes */
122 u32 src_buf;
123 u32 dst_buf;
124 #ifdef __ARMEB__
125 u16 auth_offs; /* Authentication start offset */
126 u16 auth_len; /* Authentication data length */
127 u16 crypt_offs; /* Cryption start offset */
128 u16 crypt_len; /* Cryption data length */
129 #else
130 u16 auth_len; /* Authentication data length */
131 u16 auth_offs; /* Authentication start offset */
132 u16 crypt_len; /* Cryption data length */
133 u16 crypt_offs; /* Cryption start offset */
134 #endif
135 u32 aadAddr; /* Additional Auth Data Addr for CCM mode */
136 u32 crypto_ctx; /* NPE Crypto Param structure address */
137
138 /* Used by Host: 4*4 bytes*/
139 unsigned int ctl_flags;
140 union {
141 struct skcipher_request *ablk_req;
142 struct aead_request *aead_req;
143 struct crypto_tfm *tfm;
144 } data;
145 struct buffer_desc *regist_buf;
146 u8 *regist_ptr;
147 };
148
149 struct ablk_ctx {
150 struct buffer_desc *src;
151 struct buffer_desc *dst;
152 u8 iv[MAX_IVLEN];
153 bool encrypt;
154 struct skcipher_request fallback_req; // keep at the end
155 };
156
157 struct aead_ctx {
158 struct buffer_desc *src;
159 struct buffer_desc *dst;
160 struct scatterlist ivlist;
161 /* used when the hmac is not on one sg entry */
162 u8 *hmac_virt;
163 int encrypt;
164 };
165
166 struct ix_hash_algo {
167 u32 cfgword;
168 unsigned char *icv;
169 };
170
171 struct ix_sa_dir {
172 unsigned char *npe_ctx;
173 dma_addr_t npe_ctx_phys;
174 int npe_ctx_idx;
175 u8 npe_mode;
176 };
177
178 struct ixp_ctx {
179 struct ix_sa_dir encrypt;
180 struct ix_sa_dir decrypt;
181 int authkey_len;
182 u8 authkey[MAX_KEYLEN];
183 int enckey_len;
184 u8 enckey[MAX_KEYLEN];
185 u8 salt[MAX_IVLEN];
186 u8 nonce[CTR_RFC3686_NONCE_SIZE];
187 unsigned int salted;
188 atomic_t configuring;
189 struct completion completion;
190 struct crypto_skcipher *fallback_tfm;
191 };
192
193 struct ixp_alg {
194 struct skcipher_alg crypto;
195 const struct ix_hash_algo *hash;
196 u32 cfg_enc;
197 u32 cfg_dec;
198
199 int registered;
200 };
201
202 struct ixp_aead_alg {
203 struct aead_alg crypto;
204 const struct ix_hash_algo *hash;
205 u32 cfg_enc;
206 u32 cfg_dec;
207
208 int registered;
209 };
210
211 static const struct ix_hash_algo hash_alg_md5 = {
212 .cfgword = 0xAA010004,
213 .icv = "\x01\x23\x45\x67\x89\xAB\xCD\xEF"
214 "\xFE\xDC\xBA\x98\x76\x54\x32\x10",
215 };
216
217 static const struct ix_hash_algo hash_alg_sha1 = {
218 .cfgword = 0x00000005,
219 .icv = "\x67\x45\x23\x01\xEF\xCD\xAB\x89\x98\xBA"
220 "\xDC\xFE\x10\x32\x54\x76\xC3\xD2\xE1\xF0",
221 };
222
223 static struct npe *npe_c;
224
225 static unsigned int send_qid;
226 static unsigned int recv_qid;
227 static struct dma_pool *buffer_pool;
228 static struct dma_pool *ctx_pool;
229
230 static struct crypt_ctl *crypt_virt;
231 static dma_addr_t crypt_phys;
232
233 static int support_aes = 1;
234
235 static struct platform_device *pdev;
236
crypt_virt2phys(struct crypt_ctl * virt)237 static inline dma_addr_t crypt_virt2phys(struct crypt_ctl *virt)
238 {
239 return crypt_phys + (virt - crypt_virt) * sizeof(struct crypt_ctl);
240 }
241
crypt_phys2virt(dma_addr_t phys)242 static inline struct crypt_ctl *crypt_phys2virt(dma_addr_t phys)
243 {
244 return crypt_virt + (phys - crypt_phys) / sizeof(struct crypt_ctl);
245 }
246
cipher_cfg_enc(struct crypto_tfm * tfm)247 static inline u32 cipher_cfg_enc(struct crypto_tfm *tfm)
248 {
249 return container_of(tfm->__crt_alg, struct ixp_alg, crypto.base)->cfg_enc;
250 }
251
cipher_cfg_dec(struct crypto_tfm * tfm)252 static inline u32 cipher_cfg_dec(struct crypto_tfm *tfm)
253 {
254 return container_of(tfm->__crt_alg, struct ixp_alg, crypto.base)->cfg_dec;
255 }
256
ix_hash(struct crypto_tfm * tfm)257 static inline const struct ix_hash_algo *ix_hash(struct crypto_tfm *tfm)
258 {
259 return container_of(tfm->__crt_alg, struct ixp_alg, crypto.base)->hash;
260 }
261
setup_crypt_desc(void)262 static int setup_crypt_desc(void)
263 {
264 struct device *dev = &pdev->dev;
265
266 BUILD_BUG_ON(!(IS_ENABLED(CONFIG_COMPILE_TEST) &&
267 IS_ENABLED(CONFIG_64BIT)) &&
268 sizeof(struct crypt_ctl) != 64);
269 crypt_virt = dma_alloc_coherent(dev,
270 NPE_QLEN * sizeof(struct crypt_ctl),
271 &crypt_phys, GFP_ATOMIC);
272 if (!crypt_virt)
273 return -ENOMEM;
274 return 0;
275 }
276
277 static DEFINE_SPINLOCK(desc_lock);
get_crypt_desc(void)278 static struct crypt_ctl *get_crypt_desc(void)
279 {
280 int i;
281 static int idx;
282 unsigned long flags;
283
284 spin_lock_irqsave(&desc_lock, flags);
285
286 if (unlikely(!crypt_virt))
287 setup_crypt_desc();
288 if (unlikely(!crypt_virt)) {
289 spin_unlock_irqrestore(&desc_lock, flags);
290 return NULL;
291 }
292 i = idx;
293 if (crypt_virt[i].ctl_flags == CTL_FLAG_UNUSED) {
294 if (++idx >= NPE_QLEN)
295 idx = 0;
296 crypt_virt[i].ctl_flags = CTL_FLAG_USED;
297 spin_unlock_irqrestore(&desc_lock, flags);
298 return crypt_virt + i;
299 } else {
300 spin_unlock_irqrestore(&desc_lock, flags);
301 return NULL;
302 }
303 }
304
305 static DEFINE_SPINLOCK(emerg_lock);
get_crypt_desc_emerg(void)306 static struct crypt_ctl *get_crypt_desc_emerg(void)
307 {
308 int i;
309 static int idx = NPE_QLEN;
310 struct crypt_ctl *desc;
311 unsigned long flags;
312
313 desc = get_crypt_desc();
314 if (desc)
315 return desc;
316 if (unlikely(!crypt_virt))
317 return NULL;
318
319 spin_lock_irqsave(&emerg_lock, flags);
320 i = idx;
321 if (crypt_virt[i].ctl_flags == CTL_FLAG_UNUSED) {
322 if (++idx >= NPE_QLEN_TOTAL)
323 idx = NPE_QLEN;
324 crypt_virt[i].ctl_flags = CTL_FLAG_USED;
325 spin_unlock_irqrestore(&emerg_lock, flags);
326 return crypt_virt + i;
327 } else {
328 spin_unlock_irqrestore(&emerg_lock, flags);
329 return NULL;
330 }
331 }
332
free_buf_chain(struct device * dev,struct buffer_desc * buf,dma_addr_t phys)333 static void free_buf_chain(struct device *dev, struct buffer_desc *buf,
334 dma_addr_t phys)
335 {
336 while (buf) {
337 struct buffer_desc *buf1;
338 u32 phys1;
339
340 buf1 = buf->next;
341 phys1 = buf->phys_next;
342 dma_unmap_single(dev, buf->phys_addr, buf->buf_len, buf->dir);
343 dma_pool_free(buffer_pool, buf, phys);
344 buf = buf1;
345 phys = phys1;
346 }
347 }
348
349 static struct tasklet_struct crypto_done_tasklet;
350
finish_scattered_hmac(struct crypt_ctl * crypt)351 static void finish_scattered_hmac(struct crypt_ctl *crypt)
352 {
353 struct aead_request *req = crypt->data.aead_req;
354 struct aead_ctx *req_ctx = aead_request_ctx(req);
355 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
356 int authsize = crypto_aead_authsize(tfm);
357 int decryptlen = req->assoclen + req->cryptlen - authsize;
358
359 if (req_ctx->encrypt) {
360 scatterwalk_map_and_copy(req_ctx->hmac_virt, req->dst,
361 decryptlen, authsize, 1);
362 }
363 dma_pool_free(buffer_pool, req_ctx->hmac_virt, crypt->icv_rev_aes);
364 }
365
one_packet(dma_addr_t phys)366 static void one_packet(dma_addr_t phys)
367 {
368 struct device *dev = &pdev->dev;
369 struct crypt_ctl *crypt;
370 struct ixp_ctx *ctx;
371 int failed;
372
373 failed = phys & 0x1 ? -EBADMSG : 0;
374 phys &= ~0x3;
375 crypt = crypt_phys2virt(phys);
376
377 switch (crypt->ctl_flags & CTL_FLAG_MASK) {
378 case CTL_FLAG_PERFORM_AEAD: {
379 struct aead_request *req = crypt->data.aead_req;
380 struct aead_ctx *req_ctx = aead_request_ctx(req);
381
382 free_buf_chain(dev, req_ctx->src, crypt->src_buf);
383 free_buf_chain(dev, req_ctx->dst, crypt->dst_buf);
384 if (req_ctx->hmac_virt)
385 finish_scattered_hmac(crypt);
386
387 aead_request_complete(req, failed);
388 break;
389 }
390 case CTL_FLAG_PERFORM_ABLK: {
391 struct skcipher_request *req = crypt->data.ablk_req;
392 struct ablk_ctx *req_ctx = skcipher_request_ctx(req);
393 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
394 unsigned int ivsize = crypto_skcipher_ivsize(tfm);
395 unsigned int offset;
396
397 if (ivsize > 0) {
398 offset = req->cryptlen - ivsize;
399 if (req_ctx->encrypt) {
400 scatterwalk_map_and_copy(req->iv, req->dst,
401 offset, ivsize, 0);
402 } else {
403 memcpy(req->iv, req_ctx->iv, ivsize);
404 memzero_explicit(req_ctx->iv, ivsize);
405 }
406 }
407
408 if (req_ctx->dst)
409 free_buf_chain(dev, req_ctx->dst, crypt->dst_buf);
410
411 free_buf_chain(dev, req_ctx->src, crypt->src_buf);
412 skcipher_request_complete(req, failed);
413 break;
414 }
415 case CTL_FLAG_GEN_ICV:
416 ctx = crypto_tfm_ctx(crypt->data.tfm);
417 dma_pool_free(ctx_pool, crypt->regist_ptr,
418 crypt->regist_buf->phys_addr);
419 dma_pool_free(buffer_pool, crypt->regist_buf, crypt->src_buf);
420 if (atomic_dec_and_test(&ctx->configuring))
421 complete(&ctx->completion);
422 break;
423 case CTL_FLAG_GEN_REVAES:
424 ctx = crypto_tfm_ctx(crypt->data.tfm);
425 *(__be32 *)ctx->decrypt.npe_ctx &= cpu_to_be32(~CIPH_ENCR);
426 if (atomic_dec_and_test(&ctx->configuring))
427 complete(&ctx->completion);
428 break;
429 default:
430 BUG();
431 }
432 crypt->ctl_flags = CTL_FLAG_UNUSED;
433 }
434
irqhandler(void * _unused)435 static void irqhandler(void *_unused)
436 {
437 tasklet_schedule(&crypto_done_tasklet);
438 }
439
crypto_done_action(unsigned long arg)440 static void crypto_done_action(unsigned long arg)
441 {
442 int i;
443
444 for (i = 0; i < 4; i++) {
445 dma_addr_t phys = qmgr_get_entry(recv_qid);
446 if (!phys)
447 return;
448 one_packet(phys);
449 }
450 tasklet_schedule(&crypto_done_tasklet);
451 }
452
init_ixp_crypto(struct device * dev)453 static int init_ixp_crypto(struct device *dev)
454 {
455 struct device_node *np = dev->of_node;
456 u32 msg[2] = { 0, 0 };
457 int ret = -ENODEV;
458 u32 npe_id;
459
460 dev_info(dev, "probing...\n");
461
462 /* Locate the NPE and queue manager to use from device tree */
463 if (IS_ENABLED(CONFIG_OF) && np) {
464 struct of_phandle_args queue_spec;
465 struct of_phandle_args npe_spec;
466
467 ret = of_parse_phandle_with_fixed_args(np, "intel,npe-handle",
468 1, 0, &npe_spec);
469 if (ret) {
470 dev_err(dev, "no NPE engine specified\n");
471 return -ENODEV;
472 }
473 npe_id = npe_spec.args[0];
474 of_node_put(npe_spec.np);
475
476 ret = of_parse_phandle_with_fixed_args(np, "queue-rx", 1, 0,
477 &queue_spec);
478 if (ret) {
479 dev_err(dev, "no rx queue phandle\n");
480 return -ENODEV;
481 }
482 recv_qid = queue_spec.args[0];
483 of_node_put(queue_spec.np);
484
485 ret = of_parse_phandle_with_fixed_args(np, "queue-txready", 1, 0,
486 &queue_spec);
487 if (ret) {
488 dev_err(dev, "no txready queue phandle\n");
489 return -ENODEV;
490 }
491 send_qid = queue_spec.args[0];
492 of_node_put(queue_spec.np);
493 } else {
494 /*
495 * Hardcoded engine when using platform data, this goes away
496 * when we switch to using DT only.
497 */
498 npe_id = 2;
499 send_qid = 29;
500 recv_qid = 30;
501 }
502
503 npe_c = npe_request(npe_id);
504 if (!npe_c)
505 return ret;
506
507 if (!npe_running(npe_c)) {
508 ret = npe_load_firmware(npe_c, npe_name(npe_c), dev);
509 if (ret)
510 goto npe_release;
511 if (npe_recv_message(npe_c, msg, "STATUS_MSG"))
512 goto npe_error;
513 } else {
514 if (npe_send_message(npe_c, msg, "STATUS_MSG"))
515 goto npe_error;
516
517 if (npe_recv_message(npe_c, msg, "STATUS_MSG"))
518 goto npe_error;
519 }
520
521 switch ((msg[1] >> 16) & 0xff) {
522 case 3:
523 dev_warn(dev, "Firmware of %s lacks AES support\n", npe_name(npe_c));
524 support_aes = 0;
525 break;
526 case 4:
527 case 5:
528 support_aes = 1;
529 break;
530 default:
531 dev_err(dev, "Firmware of %s lacks crypto support\n", npe_name(npe_c));
532 ret = -ENODEV;
533 goto npe_release;
534 }
535 /* buffer_pool will also be used to sometimes store the hmac,
536 * so assure it is large enough
537 */
538 BUILD_BUG_ON(SHA1_DIGEST_SIZE > sizeof(struct buffer_desc));
539 buffer_pool = dma_pool_create("buffer", dev, sizeof(struct buffer_desc),
540 32, 0);
541 ret = -ENOMEM;
542 if (!buffer_pool)
543 goto err;
544
545 ctx_pool = dma_pool_create("context", dev, NPE_CTX_LEN, 16, 0);
546 if (!ctx_pool)
547 goto err;
548
549 ret = qmgr_request_queue(send_qid, NPE_QLEN_TOTAL, 0, 0,
550 "ixp_crypto:out", NULL);
551 if (ret)
552 goto err;
553 ret = qmgr_request_queue(recv_qid, NPE_QLEN, 0, 0,
554 "ixp_crypto:in", NULL);
555 if (ret) {
556 qmgr_release_queue(send_qid);
557 goto err;
558 }
559 qmgr_set_irq(recv_qid, QUEUE_IRQ_SRC_NOT_EMPTY, irqhandler, NULL);
560 tasklet_init(&crypto_done_tasklet, crypto_done_action, 0);
561
562 qmgr_enable_irq(recv_qid);
563 return 0;
564
565 npe_error:
566 dev_err(dev, "%s not responding\n", npe_name(npe_c));
567 ret = -EIO;
568 err:
569 dma_pool_destroy(ctx_pool);
570 dma_pool_destroy(buffer_pool);
571 npe_release:
572 npe_release(npe_c);
573 return ret;
574 }
575
release_ixp_crypto(struct device * dev)576 static void release_ixp_crypto(struct device *dev)
577 {
578 qmgr_disable_irq(recv_qid);
579 tasklet_kill(&crypto_done_tasklet);
580
581 qmgr_release_queue(send_qid);
582 qmgr_release_queue(recv_qid);
583
584 dma_pool_destroy(ctx_pool);
585 dma_pool_destroy(buffer_pool);
586
587 npe_release(npe_c);
588
589 if (crypt_virt)
590 dma_free_coherent(dev, NPE_QLEN * sizeof(struct crypt_ctl),
591 crypt_virt, crypt_phys);
592 }
593
reset_sa_dir(struct ix_sa_dir * dir)594 static void reset_sa_dir(struct ix_sa_dir *dir)
595 {
596 memset(dir->npe_ctx, 0, NPE_CTX_LEN);
597 dir->npe_ctx_idx = 0;
598 dir->npe_mode = 0;
599 }
600
init_sa_dir(struct ix_sa_dir * dir)601 static int init_sa_dir(struct ix_sa_dir *dir)
602 {
603 dir->npe_ctx = dma_pool_alloc(ctx_pool, GFP_KERNEL, &dir->npe_ctx_phys);
604 if (!dir->npe_ctx)
605 return -ENOMEM;
606
607 reset_sa_dir(dir);
608 return 0;
609 }
610
free_sa_dir(struct ix_sa_dir * dir)611 static void free_sa_dir(struct ix_sa_dir *dir)
612 {
613 memset(dir->npe_ctx, 0, NPE_CTX_LEN);
614 dma_pool_free(ctx_pool, dir->npe_ctx, dir->npe_ctx_phys);
615 }
616
init_tfm(struct crypto_tfm * tfm)617 static int init_tfm(struct crypto_tfm *tfm)
618 {
619 struct ixp_ctx *ctx = crypto_tfm_ctx(tfm);
620 int ret;
621
622 atomic_set(&ctx->configuring, 0);
623 ret = init_sa_dir(&ctx->encrypt);
624 if (ret)
625 return ret;
626 ret = init_sa_dir(&ctx->decrypt);
627 if (ret)
628 free_sa_dir(&ctx->encrypt);
629
630 return ret;
631 }
632
init_tfm_ablk(struct crypto_skcipher * tfm)633 static int init_tfm_ablk(struct crypto_skcipher *tfm)
634 {
635 struct crypto_tfm *ctfm = crypto_skcipher_tfm(tfm);
636 struct ixp_ctx *ctx = crypto_tfm_ctx(ctfm);
637 const char *name = crypto_tfm_alg_name(ctfm);
638
639 ctx->fallback_tfm = crypto_alloc_skcipher(name, 0, CRYPTO_ALG_NEED_FALLBACK);
640 if (IS_ERR(ctx->fallback_tfm)) {
641 pr_err("ERROR: Cannot allocate fallback for %s %ld\n",
642 name, PTR_ERR(ctx->fallback_tfm));
643 return PTR_ERR(ctx->fallback_tfm);
644 }
645
646 pr_info("Fallback for %s is %s\n",
647 crypto_tfm_alg_driver_name(&tfm->base),
648 crypto_tfm_alg_driver_name(crypto_skcipher_tfm(ctx->fallback_tfm))
649 );
650
651 crypto_skcipher_set_reqsize(tfm, sizeof(struct ablk_ctx) + crypto_skcipher_reqsize(ctx->fallback_tfm));
652 return init_tfm(crypto_skcipher_tfm(tfm));
653 }
654
init_tfm_aead(struct crypto_aead * tfm)655 static int init_tfm_aead(struct crypto_aead *tfm)
656 {
657 crypto_aead_set_reqsize(tfm, sizeof(struct aead_ctx));
658 return init_tfm(crypto_aead_tfm(tfm));
659 }
660
exit_tfm(struct crypto_tfm * tfm)661 static void exit_tfm(struct crypto_tfm *tfm)
662 {
663 struct ixp_ctx *ctx = crypto_tfm_ctx(tfm);
664
665 free_sa_dir(&ctx->encrypt);
666 free_sa_dir(&ctx->decrypt);
667 }
668
exit_tfm_ablk(struct crypto_skcipher * tfm)669 static void exit_tfm_ablk(struct crypto_skcipher *tfm)
670 {
671 struct crypto_tfm *ctfm = crypto_skcipher_tfm(tfm);
672 struct ixp_ctx *ctx = crypto_tfm_ctx(ctfm);
673
674 crypto_free_skcipher(ctx->fallback_tfm);
675 exit_tfm(crypto_skcipher_tfm(tfm));
676 }
677
exit_tfm_aead(struct crypto_aead * tfm)678 static void exit_tfm_aead(struct crypto_aead *tfm)
679 {
680 exit_tfm(crypto_aead_tfm(tfm));
681 }
682
register_chain_var(struct crypto_tfm * tfm,u8 xpad,u32 target,int init_len,u32 ctx_addr,const u8 * key,int key_len)683 static int register_chain_var(struct crypto_tfm *tfm, u8 xpad, u32 target,
684 int init_len, u32 ctx_addr, const u8 *key,
685 int key_len)
686 {
687 struct ixp_ctx *ctx = crypto_tfm_ctx(tfm);
688 struct crypt_ctl *crypt;
689 struct buffer_desc *buf;
690 int i;
691 u8 *pad;
692 dma_addr_t pad_phys, buf_phys;
693
694 BUILD_BUG_ON(NPE_CTX_LEN < HMAC_PAD_BLOCKLEN);
695 pad = dma_pool_alloc(ctx_pool, GFP_KERNEL, &pad_phys);
696 if (!pad)
697 return -ENOMEM;
698 buf = dma_pool_alloc(buffer_pool, GFP_KERNEL, &buf_phys);
699 if (!buf) {
700 dma_pool_free(ctx_pool, pad, pad_phys);
701 return -ENOMEM;
702 }
703 crypt = get_crypt_desc_emerg();
704 if (!crypt) {
705 dma_pool_free(ctx_pool, pad, pad_phys);
706 dma_pool_free(buffer_pool, buf, buf_phys);
707 return -EAGAIN;
708 }
709
710 memcpy(pad, key, key_len);
711 memset(pad + key_len, 0, HMAC_PAD_BLOCKLEN - key_len);
712 for (i = 0; i < HMAC_PAD_BLOCKLEN; i++)
713 pad[i] ^= xpad;
714
715 crypt->data.tfm = tfm;
716 crypt->regist_ptr = pad;
717 crypt->regist_buf = buf;
718
719 crypt->auth_offs = 0;
720 crypt->auth_len = HMAC_PAD_BLOCKLEN;
721 crypt->crypto_ctx = ctx_addr;
722 crypt->src_buf = buf_phys;
723 crypt->icv_rev_aes = target;
724 crypt->mode = NPE_OP_HASH_GEN_ICV;
725 crypt->init_len = init_len;
726 crypt->ctl_flags |= CTL_FLAG_GEN_ICV;
727
728 buf->next = NULL;
729 buf->buf_len = HMAC_PAD_BLOCKLEN;
730 buf->pkt_len = 0;
731 buf->phys_addr = pad_phys;
732
733 atomic_inc(&ctx->configuring);
734 qmgr_put_entry(send_qid, crypt_virt2phys(crypt));
735 BUG_ON(qmgr_stat_overflow(send_qid));
736 return 0;
737 }
738
setup_auth(struct crypto_tfm * tfm,int encrypt,unsigned int authsize,const u8 * key,int key_len,unsigned int digest_len)739 static int setup_auth(struct crypto_tfm *tfm, int encrypt, unsigned int authsize,
740 const u8 *key, int key_len, unsigned int digest_len)
741 {
742 u32 itarget, otarget, npe_ctx_addr;
743 unsigned char *cinfo;
744 int init_len, ret = 0;
745 u32 cfgword;
746 struct ix_sa_dir *dir;
747 struct ixp_ctx *ctx = crypto_tfm_ctx(tfm);
748 const struct ix_hash_algo *algo;
749
750 dir = encrypt ? &ctx->encrypt : &ctx->decrypt;
751 cinfo = dir->npe_ctx + dir->npe_ctx_idx;
752 algo = ix_hash(tfm);
753
754 /* write cfg word to cryptinfo */
755 cfgword = algo->cfgword | (authsize << 6); /* (authsize/4) << 8 */
756 #ifndef __ARMEB__
757 cfgword ^= 0xAA000000; /* change the "byte swap" flags */
758 #endif
759 *(__be32 *)cinfo = cpu_to_be32(cfgword);
760 cinfo += sizeof(cfgword);
761
762 /* write ICV to cryptinfo */
763 memcpy(cinfo, algo->icv, digest_len);
764 cinfo += digest_len;
765
766 itarget = dir->npe_ctx_phys + dir->npe_ctx_idx
767 + sizeof(algo->cfgword);
768 otarget = itarget + digest_len;
769 init_len = cinfo - (dir->npe_ctx + dir->npe_ctx_idx);
770 npe_ctx_addr = dir->npe_ctx_phys + dir->npe_ctx_idx;
771
772 dir->npe_ctx_idx += init_len;
773 dir->npe_mode |= NPE_OP_HASH_ENABLE;
774
775 if (!encrypt)
776 dir->npe_mode |= NPE_OP_HASH_VERIFY;
777
778 ret = register_chain_var(tfm, HMAC_OPAD_VALUE, otarget,
779 init_len, npe_ctx_addr, key, key_len);
780 if (ret)
781 return ret;
782 return register_chain_var(tfm, HMAC_IPAD_VALUE, itarget,
783 init_len, npe_ctx_addr, key, key_len);
784 }
785
gen_rev_aes_key(struct crypto_tfm * tfm)786 static int gen_rev_aes_key(struct crypto_tfm *tfm)
787 {
788 struct crypt_ctl *crypt;
789 struct ixp_ctx *ctx = crypto_tfm_ctx(tfm);
790 struct ix_sa_dir *dir = &ctx->decrypt;
791
792 crypt = get_crypt_desc_emerg();
793 if (!crypt)
794 return -EAGAIN;
795
796 *(__be32 *)dir->npe_ctx |= cpu_to_be32(CIPH_ENCR);
797
798 crypt->data.tfm = tfm;
799 crypt->crypt_offs = 0;
800 crypt->crypt_len = AES_BLOCK128;
801 crypt->src_buf = 0;
802 crypt->crypto_ctx = dir->npe_ctx_phys;
803 crypt->icv_rev_aes = dir->npe_ctx_phys + sizeof(u32);
804 crypt->mode = NPE_OP_ENC_GEN_KEY;
805 crypt->init_len = dir->npe_ctx_idx;
806 crypt->ctl_flags |= CTL_FLAG_GEN_REVAES;
807
808 atomic_inc(&ctx->configuring);
809 qmgr_put_entry(send_qid, crypt_virt2phys(crypt));
810 BUG_ON(qmgr_stat_overflow(send_qid));
811 return 0;
812 }
813
setup_cipher(struct crypto_tfm * tfm,int encrypt,const u8 * key,int key_len)814 static int setup_cipher(struct crypto_tfm *tfm, int encrypt, const u8 *key,
815 int key_len)
816 {
817 u8 *cinfo;
818 u32 cipher_cfg;
819 u32 keylen_cfg = 0;
820 struct ix_sa_dir *dir;
821 struct ixp_ctx *ctx = crypto_tfm_ctx(tfm);
822 int err;
823
824 dir = encrypt ? &ctx->encrypt : &ctx->decrypt;
825 cinfo = dir->npe_ctx;
826
827 if (encrypt) {
828 cipher_cfg = cipher_cfg_enc(tfm);
829 dir->npe_mode |= NPE_OP_CRYPT_ENCRYPT;
830 } else {
831 cipher_cfg = cipher_cfg_dec(tfm);
832 }
833 if (cipher_cfg & MOD_AES) {
834 switch (key_len) {
835 case 16:
836 keylen_cfg = MOD_AES128;
837 break;
838 case 24:
839 keylen_cfg = MOD_AES192;
840 break;
841 case 32:
842 keylen_cfg = MOD_AES256;
843 break;
844 default:
845 return -EINVAL;
846 }
847 cipher_cfg |= keylen_cfg;
848 } else {
849 err = crypto_des_verify_key(tfm, key);
850 if (err)
851 return err;
852 }
853 /* write cfg word to cryptinfo */
854 *(__be32 *)cinfo = cpu_to_be32(cipher_cfg);
855 cinfo += sizeof(cipher_cfg);
856
857 /* write cipher key to cryptinfo */
858 memcpy(cinfo, key, key_len);
859 /* NPE wants keylen set to DES3_EDE_KEY_SIZE even for single DES */
860 if (key_len < DES3_EDE_KEY_SIZE && !(cipher_cfg & MOD_AES)) {
861 memset(cinfo + key_len, 0, DES3_EDE_KEY_SIZE - key_len);
862 key_len = DES3_EDE_KEY_SIZE;
863 }
864 dir->npe_ctx_idx = sizeof(cipher_cfg) + key_len;
865 dir->npe_mode |= NPE_OP_CRYPT_ENABLE;
866 if ((cipher_cfg & MOD_AES) && !encrypt)
867 return gen_rev_aes_key(tfm);
868
869 return 0;
870 }
871
chainup_buffers(struct device * dev,struct scatterlist * sg,unsigned int nbytes,struct buffer_desc * buf,gfp_t flags,enum dma_data_direction dir)872 static struct buffer_desc *chainup_buffers(struct device *dev,
873 struct scatterlist *sg, unsigned int nbytes,
874 struct buffer_desc *buf, gfp_t flags,
875 enum dma_data_direction dir)
876 {
877 for (; nbytes > 0; sg = sg_next(sg)) {
878 unsigned int len = min(nbytes, sg->length);
879 struct buffer_desc *next_buf;
880 dma_addr_t next_buf_phys;
881 void *ptr;
882
883 nbytes -= len;
884 ptr = sg_virt(sg);
885 next_buf = dma_pool_alloc(buffer_pool, flags, &next_buf_phys);
886 if (!next_buf) {
887 buf = NULL;
888 break;
889 }
890 sg_dma_address(sg) = dma_map_single(dev, ptr, len, dir);
891 buf->next = next_buf;
892 buf->phys_next = next_buf_phys;
893 buf = next_buf;
894
895 buf->phys_addr = sg_dma_address(sg);
896 buf->buf_len = len;
897 buf->dir = dir;
898 }
899 buf->next = NULL;
900 buf->phys_next = 0;
901 return buf;
902 }
903
ablk_setkey(struct crypto_skcipher * tfm,const u8 * key,unsigned int key_len)904 static int ablk_setkey(struct crypto_skcipher *tfm, const u8 *key,
905 unsigned int key_len)
906 {
907 struct ixp_ctx *ctx = crypto_skcipher_ctx(tfm);
908 int ret;
909
910 init_completion(&ctx->completion);
911 atomic_inc(&ctx->configuring);
912
913 reset_sa_dir(&ctx->encrypt);
914 reset_sa_dir(&ctx->decrypt);
915
916 ctx->encrypt.npe_mode = NPE_OP_HMAC_DISABLE;
917 ctx->decrypt.npe_mode = NPE_OP_HMAC_DISABLE;
918
919 ret = setup_cipher(&tfm->base, 0, key, key_len);
920 if (ret)
921 goto out;
922 ret = setup_cipher(&tfm->base, 1, key, key_len);
923 out:
924 if (!atomic_dec_and_test(&ctx->configuring))
925 wait_for_completion(&ctx->completion);
926 if (ret)
927 return ret;
928 crypto_skcipher_clear_flags(ctx->fallback_tfm, CRYPTO_TFM_REQ_MASK);
929 crypto_skcipher_set_flags(ctx->fallback_tfm, tfm->base.crt_flags & CRYPTO_TFM_REQ_MASK);
930
931 return crypto_skcipher_setkey(ctx->fallback_tfm, key, key_len);
932 }
933
ablk_des3_setkey(struct crypto_skcipher * tfm,const u8 * key,unsigned int key_len)934 static int ablk_des3_setkey(struct crypto_skcipher *tfm, const u8 *key,
935 unsigned int key_len)
936 {
937 return verify_skcipher_des3_key(tfm, key) ?:
938 ablk_setkey(tfm, key, key_len);
939 }
940
ablk_rfc3686_setkey(struct crypto_skcipher * tfm,const u8 * key,unsigned int key_len)941 static int ablk_rfc3686_setkey(struct crypto_skcipher *tfm, const u8 *key,
942 unsigned int key_len)
943 {
944 struct ixp_ctx *ctx = crypto_skcipher_ctx(tfm);
945
946 /* the nonce is stored in bytes at end of key */
947 if (key_len < CTR_RFC3686_NONCE_SIZE)
948 return -EINVAL;
949
950 memcpy(ctx->nonce, key + (key_len - CTR_RFC3686_NONCE_SIZE),
951 CTR_RFC3686_NONCE_SIZE);
952
953 key_len -= CTR_RFC3686_NONCE_SIZE;
954 return ablk_setkey(tfm, key, key_len);
955 }
956
ixp4xx_cipher_fallback(struct skcipher_request * areq,int encrypt)957 static int ixp4xx_cipher_fallback(struct skcipher_request *areq, int encrypt)
958 {
959 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq);
960 struct ixp_ctx *op = crypto_skcipher_ctx(tfm);
961 struct ablk_ctx *rctx = skcipher_request_ctx(areq);
962 int err;
963
964 skcipher_request_set_tfm(&rctx->fallback_req, op->fallback_tfm);
965 skcipher_request_set_callback(&rctx->fallback_req, areq->base.flags,
966 areq->base.complete, areq->base.data);
967 skcipher_request_set_crypt(&rctx->fallback_req, areq->src, areq->dst,
968 areq->cryptlen, areq->iv);
969 if (encrypt)
970 err = crypto_skcipher_encrypt(&rctx->fallback_req);
971 else
972 err = crypto_skcipher_decrypt(&rctx->fallback_req);
973 return err;
974 }
975
ablk_perform(struct skcipher_request * req,int encrypt)976 static int ablk_perform(struct skcipher_request *req, int encrypt)
977 {
978 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
979 struct ixp_ctx *ctx = crypto_skcipher_ctx(tfm);
980 unsigned int ivsize = crypto_skcipher_ivsize(tfm);
981 struct ix_sa_dir *dir;
982 struct crypt_ctl *crypt;
983 unsigned int nbytes = req->cryptlen;
984 enum dma_data_direction src_direction = DMA_BIDIRECTIONAL;
985 struct ablk_ctx *req_ctx = skcipher_request_ctx(req);
986 struct buffer_desc src_hook;
987 struct device *dev = &pdev->dev;
988 unsigned int offset;
989 gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ?
990 GFP_KERNEL : GFP_ATOMIC;
991
992 if (sg_nents(req->src) > 1 || sg_nents(req->dst) > 1)
993 return ixp4xx_cipher_fallback(req, encrypt);
994
995 if (qmgr_stat_full(send_qid))
996 return -EAGAIN;
997 if (atomic_read(&ctx->configuring))
998 return -EAGAIN;
999
1000 dir = encrypt ? &ctx->encrypt : &ctx->decrypt;
1001 req_ctx->encrypt = encrypt;
1002
1003 crypt = get_crypt_desc();
1004 if (!crypt)
1005 return -ENOMEM;
1006
1007 crypt->data.ablk_req = req;
1008 crypt->crypto_ctx = dir->npe_ctx_phys;
1009 crypt->mode = dir->npe_mode;
1010 crypt->init_len = dir->npe_ctx_idx;
1011
1012 crypt->crypt_offs = 0;
1013 crypt->crypt_len = nbytes;
1014
1015 BUG_ON(ivsize && !req->iv);
1016 memcpy(crypt->iv, req->iv, ivsize);
1017 if (ivsize > 0 && !encrypt) {
1018 offset = req->cryptlen - ivsize;
1019 scatterwalk_map_and_copy(req_ctx->iv, req->src, offset, ivsize, 0);
1020 }
1021 if (req->src != req->dst) {
1022 struct buffer_desc dst_hook;
1023
1024 crypt->mode |= NPE_OP_NOT_IN_PLACE;
1025 /* This was never tested by Intel
1026 * for more than one dst buffer, I think. */
1027 req_ctx->dst = NULL;
1028 if (!chainup_buffers(dev, req->dst, nbytes, &dst_hook,
1029 flags, DMA_FROM_DEVICE))
1030 goto free_buf_dest;
1031 src_direction = DMA_TO_DEVICE;
1032 req_ctx->dst = dst_hook.next;
1033 crypt->dst_buf = dst_hook.phys_next;
1034 } else {
1035 req_ctx->dst = NULL;
1036 }
1037 req_ctx->src = NULL;
1038 if (!chainup_buffers(dev, req->src, nbytes, &src_hook, flags,
1039 src_direction))
1040 goto free_buf_src;
1041
1042 req_ctx->src = src_hook.next;
1043 crypt->src_buf = src_hook.phys_next;
1044 crypt->ctl_flags |= CTL_FLAG_PERFORM_ABLK;
1045 qmgr_put_entry(send_qid, crypt_virt2phys(crypt));
1046 BUG_ON(qmgr_stat_overflow(send_qid));
1047 return -EINPROGRESS;
1048
1049 free_buf_src:
1050 free_buf_chain(dev, req_ctx->src, crypt->src_buf);
1051 free_buf_dest:
1052 if (req->src != req->dst)
1053 free_buf_chain(dev, req_ctx->dst, crypt->dst_buf);
1054
1055 crypt->ctl_flags = CTL_FLAG_UNUSED;
1056 return -ENOMEM;
1057 }
1058
ablk_encrypt(struct skcipher_request * req)1059 static int ablk_encrypt(struct skcipher_request *req)
1060 {
1061 return ablk_perform(req, 1);
1062 }
1063
ablk_decrypt(struct skcipher_request * req)1064 static int ablk_decrypt(struct skcipher_request *req)
1065 {
1066 return ablk_perform(req, 0);
1067 }
1068
ablk_rfc3686_crypt(struct skcipher_request * req)1069 static int ablk_rfc3686_crypt(struct skcipher_request *req)
1070 {
1071 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
1072 struct ixp_ctx *ctx = crypto_skcipher_ctx(tfm);
1073 u8 iv[CTR_RFC3686_BLOCK_SIZE];
1074 u8 *info = req->iv;
1075 int ret;
1076
1077 /* set up counter block */
1078 memcpy(iv, ctx->nonce, CTR_RFC3686_NONCE_SIZE);
1079 memcpy(iv + CTR_RFC3686_NONCE_SIZE, info, CTR_RFC3686_IV_SIZE);
1080
1081 /* initialize counter portion of counter block */
1082 *(__be32 *)(iv + CTR_RFC3686_NONCE_SIZE + CTR_RFC3686_IV_SIZE) =
1083 cpu_to_be32(1);
1084
1085 req->iv = iv;
1086 ret = ablk_perform(req, 1);
1087 req->iv = info;
1088 return ret;
1089 }
1090
aead_perform(struct aead_request * req,int encrypt,int cryptoffset,int eff_cryptlen,u8 * iv)1091 static int aead_perform(struct aead_request *req, int encrypt,
1092 int cryptoffset, int eff_cryptlen, u8 *iv)
1093 {
1094 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1095 struct ixp_ctx *ctx = crypto_aead_ctx(tfm);
1096 unsigned int ivsize = crypto_aead_ivsize(tfm);
1097 unsigned int authsize = crypto_aead_authsize(tfm);
1098 struct ix_sa_dir *dir;
1099 struct crypt_ctl *crypt;
1100 unsigned int cryptlen;
1101 struct buffer_desc *buf, src_hook;
1102 struct aead_ctx *req_ctx = aead_request_ctx(req);
1103 struct device *dev = &pdev->dev;
1104 gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ?
1105 GFP_KERNEL : GFP_ATOMIC;
1106 enum dma_data_direction src_direction = DMA_BIDIRECTIONAL;
1107 unsigned int lastlen;
1108
1109 if (qmgr_stat_full(send_qid))
1110 return -EAGAIN;
1111 if (atomic_read(&ctx->configuring))
1112 return -EAGAIN;
1113
1114 if (encrypt) {
1115 dir = &ctx->encrypt;
1116 cryptlen = req->cryptlen;
1117 } else {
1118 dir = &ctx->decrypt;
1119 /* req->cryptlen includes the authsize when decrypting */
1120 cryptlen = req->cryptlen - authsize;
1121 eff_cryptlen -= authsize;
1122 }
1123 crypt = get_crypt_desc();
1124 if (!crypt)
1125 return -ENOMEM;
1126
1127 crypt->data.aead_req = req;
1128 crypt->crypto_ctx = dir->npe_ctx_phys;
1129 crypt->mode = dir->npe_mode;
1130 crypt->init_len = dir->npe_ctx_idx;
1131
1132 crypt->crypt_offs = cryptoffset;
1133 crypt->crypt_len = eff_cryptlen;
1134
1135 crypt->auth_offs = 0;
1136 crypt->auth_len = req->assoclen + cryptlen;
1137 BUG_ON(ivsize && !req->iv);
1138 memcpy(crypt->iv, req->iv, ivsize);
1139
1140 buf = chainup_buffers(dev, req->src, crypt->auth_len,
1141 &src_hook, flags, src_direction);
1142 req_ctx->src = src_hook.next;
1143 crypt->src_buf = src_hook.phys_next;
1144 if (!buf)
1145 goto free_buf_src;
1146
1147 lastlen = buf->buf_len;
1148 if (lastlen >= authsize)
1149 crypt->icv_rev_aes = buf->phys_addr +
1150 buf->buf_len - authsize;
1151
1152 req_ctx->dst = NULL;
1153
1154 if (req->src != req->dst) {
1155 struct buffer_desc dst_hook;
1156
1157 crypt->mode |= NPE_OP_NOT_IN_PLACE;
1158 src_direction = DMA_TO_DEVICE;
1159
1160 buf = chainup_buffers(dev, req->dst, crypt->auth_len,
1161 &dst_hook, flags, DMA_FROM_DEVICE);
1162 req_ctx->dst = dst_hook.next;
1163 crypt->dst_buf = dst_hook.phys_next;
1164
1165 if (!buf)
1166 goto free_buf_dst;
1167
1168 if (encrypt) {
1169 lastlen = buf->buf_len;
1170 if (lastlen >= authsize)
1171 crypt->icv_rev_aes = buf->phys_addr +
1172 buf->buf_len - authsize;
1173 }
1174 }
1175
1176 if (unlikely(lastlen < authsize)) {
1177 dma_addr_t dma;
1178 /* The 12 hmac bytes are scattered,
1179 * we need to copy them into a safe buffer */
1180 req_ctx->hmac_virt = dma_pool_alloc(buffer_pool, flags, &dma);
1181 if (unlikely(!req_ctx->hmac_virt))
1182 goto free_buf_dst;
1183 crypt->icv_rev_aes = dma;
1184 if (!encrypt) {
1185 scatterwalk_map_and_copy(req_ctx->hmac_virt,
1186 req->src, cryptlen, authsize, 0);
1187 }
1188 req_ctx->encrypt = encrypt;
1189 } else {
1190 req_ctx->hmac_virt = NULL;
1191 }
1192
1193 crypt->ctl_flags |= CTL_FLAG_PERFORM_AEAD;
1194 qmgr_put_entry(send_qid, crypt_virt2phys(crypt));
1195 BUG_ON(qmgr_stat_overflow(send_qid));
1196 return -EINPROGRESS;
1197
1198 free_buf_dst:
1199 free_buf_chain(dev, req_ctx->dst, crypt->dst_buf);
1200 free_buf_src:
1201 free_buf_chain(dev, req_ctx->src, crypt->src_buf);
1202 crypt->ctl_flags = CTL_FLAG_UNUSED;
1203 return -ENOMEM;
1204 }
1205
aead_setup(struct crypto_aead * tfm,unsigned int authsize)1206 static int aead_setup(struct crypto_aead *tfm, unsigned int authsize)
1207 {
1208 struct ixp_ctx *ctx = crypto_aead_ctx(tfm);
1209 unsigned int digest_len = crypto_aead_maxauthsize(tfm);
1210 int ret;
1211
1212 if (!ctx->enckey_len && !ctx->authkey_len)
1213 return 0;
1214 init_completion(&ctx->completion);
1215 atomic_inc(&ctx->configuring);
1216
1217 reset_sa_dir(&ctx->encrypt);
1218 reset_sa_dir(&ctx->decrypt);
1219
1220 ret = setup_cipher(&tfm->base, 0, ctx->enckey, ctx->enckey_len);
1221 if (ret)
1222 goto out;
1223 ret = setup_cipher(&tfm->base, 1, ctx->enckey, ctx->enckey_len);
1224 if (ret)
1225 goto out;
1226 ret = setup_auth(&tfm->base, 0, authsize, ctx->authkey,
1227 ctx->authkey_len, digest_len);
1228 if (ret)
1229 goto out;
1230 ret = setup_auth(&tfm->base, 1, authsize, ctx->authkey,
1231 ctx->authkey_len, digest_len);
1232 out:
1233 if (!atomic_dec_and_test(&ctx->configuring))
1234 wait_for_completion(&ctx->completion);
1235 return ret;
1236 }
1237
aead_setauthsize(struct crypto_aead * tfm,unsigned int authsize)1238 static int aead_setauthsize(struct crypto_aead *tfm, unsigned int authsize)
1239 {
1240 int max = crypto_aead_maxauthsize(tfm) >> 2;
1241
1242 if ((authsize >> 2) < 1 || (authsize >> 2) > max || (authsize & 3))
1243 return -EINVAL;
1244 return aead_setup(tfm, authsize);
1245 }
1246
aead_setkey(struct crypto_aead * tfm,const u8 * key,unsigned int keylen)1247 static int aead_setkey(struct crypto_aead *tfm, const u8 *key,
1248 unsigned int keylen)
1249 {
1250 struct ixp_ctx *ctx = crypto_aead_ctx(tfm);
1251 struct crypto_authenc_keys keys;
1252
1253 if (crypto_authenc_extractkeys(&keys, key, keylen) != 0)
1254 goto badkey;
1255
1256 if (keys.authkeylen > sizeof(ctx->authkey))
1257 goto badkey;
1258
1259 if (keys.enckeylen > sizeof(ctx->enckey))
1260 goto badkey;
1261
1262 memcpy(ctx->authkey, keys.authkey, keys.authkeylen);
1263 memcpy(ctx->enckey, keys.enckey, keys.enckeylen);
1264 ctx->authkey_len = keys.authkeylen;
1265 ctx->enckey_len = keys.enckeylen;
1266
1267 memzero_explicit(&keys, sizeof(keys));
1268 return aead_setup(tfm, crypto_aead_authsize(tfm));
1269 badkey:
1270 memzero_explicit(&keys, sizeof(keys));
1271 return -EINVAL;
1272 }
1273
des3_aead_setkey(struct crypto_aead * tfm,const u8 * key,unsigned int keylen)1274 static int des3_aead_setkey(struct crypto_aead *tfm, const u8 *key,
1275 unsigned int keylen)
1276 {
1277 struct ixp_ctx *ctx = crypto_aead_ctx(tfm);
1278 struct crypto_authenc_keys keys;
1279 int err;
1280
1281 err = crypto_authenc_extractkeys(&keys, key, keylen);
1282 if (unlikely(err))
1283 goto badkey;
1284
1285 err = -EINVAL;
1286 if (keys.authkeylen > sizeof(ctx->authkey))
1287 goto badkey;
1288
1289 err = verify_aead_des3_key(tfm, keys.enckey, keys.enckeylen);
1290 if (err)
1291 goto badkey;
1292
1293 memcpy(ctx->authkey, keys.authkey, keys.authkeylen);
1294 memcpy(ctx->enckey, keys.enckey, keys.enckeylen);
1295 ctx->authkey_len = keys.authkeylen;
1296 ctx->enckey_len = keys.enckeylen;
1297
1298 memzero_explicit(&keys, sizeof(keys));
1299 return aead_setup(tfm, crypto_aead_authsize(tfm));
1300 badkey:
1301 memzero_explicit(&keys, sizeof(keys));
1302 return err;
1303 }
1304
aead_encrypt(struct aead_request * req)1305 static int aead_encrypt(struct aead_request *req)
1306 {
1307 return aead_perform(req, 1, req->assoclen, req->cryptlen, req->iv);
1308 }
1309
aead_decrypt(struct aead_request * req)1310 static int aead_decrypt(struct aead_request *req)
1311 {
1312 return aead_perform(req, 0, req->assoclen, req->cryptlen, req->iv);
1313 }
1314
1315 static struct ixp_alg ixp4xx_algos[] = {
1316 {
1317 .crypto = {
1318 .base.cra_name = "cbc(des)",
1319 .base.cra_blocksize = DES_BLOCK_SIZE,
1320
1321 .min_keysize = DES_KEY_SIZE,
1322 .max_keysize = DES_KEY_SIZE,
1323 .ivsize = DES_BLOCK_SIZE,
1324 },
1325 .cfg_enc = CIPH_ENCR | MOD_DES | MOD_CBC_ENC | KEYLEN_192,
1326 .cfg_dec = CIPH_DECR | MOD_DES | MOD_CBC_DEC | KEYLEN_192,
1327
1328 }, {
1329 .crypto = {
1330 .base.cra_name = "ecb(des)",
1331 .base.cra_blocksize = DES_BLOCK_SIZE,
1332 .min_keysize = DES_KEY_SIZE,
1333 .max_keysize = DES_KEY_SIZE,
1334 },
1335 .cfg_enc = CIPH_ENCR | MOD_DES | MOD_ECB | KEYLEN_192,
1336 .cfg_dec = CIPH_DECR | MOD_DES | MOD_ECB | KEYLEN_192,
1337 }, {
1338 .crypto = {
1339 .base.cra_name = "cbc(des3_ede)",
1340 .base.cra_blocksize = DES3_EDE_BLOCK_SIZE,
1341
1342 .min_keysize = DES3_EDE_KEY_SIZE,
1343 .max_keysize = DES3_EDE_KEY_SIZE,
1344 .ivsize = DES3_EDE_BLOCK_SIZE,
1345 .setkey = ablk_des3_setkey,
1346 },
1347 .cfg_enc = CIPH_ENCR | MOD_3DES | MOD_CBC_ENC | KEYLEN_192,
1348 .cfg_dec = CIPH_DECR | MOD_3DES | MOD_CBC_DEC | KEYLEN_192,
1349 }, {
1350 .crypto = {
1351 .base.cra_name = "ecb(des3_ede)",
1352 .base.cra_blocksize = DES3_EDE_BLOCK_SIZE,
1353
1354 .min_keysize = DES3_EDE_KEY_SIZE,
1355 .max_keysize = DES3_EDE_KEY_SIZE,
1356 .setkey = ablk_des3_setkey,
1357 },
1358 .cfg_enc = CIPH_ENCR | MOD_3DES | MOD_ECB | KEYLEN_192,
1359 .cfg_dec = CIPH_DECR | MOD_3DES | MOD_ECB | KEYLEN_192,
1360 }, {
1361 .crypto = {
1362 .base.cra_name = "cbc(aes)",
1363 .base.cra_blocksize = AES_BLOCK_SIZE,
1364
1365 .min_keysize = AES_MIN_KEY_SIZE,
1366 .max_keysize = AES_MAX_KEY_SIZE,
1367 .ivsize = AES_BLOCK_SIZE,
1368 },
1369 .cfg_enc = CIPH_ENCR | MOD_AES | MOD_CBC_ENC,
1370 .cfg_dec = CIPH_DECR | MOD_AES | MOD_CBC_DEC,
1371 }, {
1372 .crypto = {
1373 .base.cra_name = "ecb(aes)",
1374 .base.cra_blocksize = AES_BLOCK_SIZE,
1375
1376 .min_keysize = AES_MIN_KEY_SIZE,
1377 .max_keysize = AES_MAX_KEY_SIZE,
1378 },
1379 .cfg_enc = CIPH_ENCR | MOD_AES | MOD_ECB,
1380 .cfg_dec = CIPH_DECR | MOD_AES | MOD_ECB,
1381 }, {
1382 .crypto = {
1383 .base.cra_name = "ctr(aes)",
1384 .base.cra_blocksize = 1,
1385
1386 .min_keysize = AES_MIN_KEY_SIZE,
1387 .max_keysize = AES_MAX_KEY_SIZE,
1388 .ivsize = AES_BLOCK_SIZE,
1389 },
1390 .cfg_enc = CIPH_ENCR | MOD_AES | MOD_CTR,
1391 .cfg_dec = CIPH_ENCR | MOD_AES | MOD_CTR,
1392 }, {
1393 .crypto = {
1394 .base.cra_name = "rfc3686(ctr(aes))",
1395 .base.cra_blocksize = 1,
1396
1397 .min_keysize = AES_MIN_KEY_SIZE,
1398 .max_keysize = AES_MAX_KEY_SIZE,
1399 .ivsize = AES_BLOCK_SIZE,
1400 .setkey = ablk_rfc3686_setkey,
1401 .encrypt = ablk_rfc3686_crypt,
1402 .decrypt = ablk_rfc3686_crypt,
1403 },
1404 .cfg_enc = CIPH_ENCR | MOD_AES | MOD_CTR,
1405 .cfg_dec = CIPH_ENCR | MOD_AES | MOD_CTR,
1406 } };
1407
1408 static struct ixp_aead_alg ixp4xx_aeads[] = {
1409 {
1410 .crypto = {
1411 .base = {
1412 .cra_name = "authenc(hmac(md5),cbc(des))",
1413 .cra_blocksize = DES_BLOCK_SIZE,
1414 },
1415 .ivsize = DES_BLOCK_SIZE,
1416 .maxauthsize = MD5_DIGEST_SIZE,
1417 },
1418 .hash = &hash_alg_md5,
1419 .cfg_enc = CIPH_ENCR | MOD_DES | MOD_CBC_ENC | KEYLEN_192,
1420 .cfg_dec = CIPH_DECR | MOD_DES | MOD_CBC_DEC | KEYLEN_192,
1421 }, {
1422 .crypto = {
1423 .base = {
1424 .cra_name = "authenc(hmac(md5),cbc(des3_ede))",
1425 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
1426 },
1427 .ivsize = DES3_EDE_BLOCK_SIZE,
1428 .maxauthsize = MD5_DIGEST_SIZE,
1429 .setkey = des3_aead_setkey,
1430 },
1431 .hash = &hash_alg_md5,
1432 .cfg_enc = CIPH_ENCR | MOD_3DES | MOD_CBC_ENC | KEYLEN_192,
1433 .cfg_dec = CIPH_DECR | MOD_3DES | MOD_CBC_DEC | KEYLEN_192,
1434 }, {
1435 .crypto = {
1436 .base = {
1437 .cra_name = "authenc(hmac(sha1),cbc(des))",
1438 .cra_blocksize = DES_BLOCK_SIZE,
1439 },
1440 .ivsize = DES_BLOCK_SIZE,
1441 .maxauthsize = SHA1_DIGEST_SIZE,
1442 },
1443 .hash = &hash_alg_sha1,
1444 .cfg_enc = CIPH_ENCR | MOD_DES | MOD_CBC_ENC | KEYLEN_192,
1445 .cfg_dec = CIPH_DECR | MOD_DES | MOD_CBC_DEC | KEYLEN_192,
1446 }, {
1447 .crypto = {
1448 .base = {
1449 .cra_name = "authenc(hmac(sha1),cbc(des3_ede))",
1450 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
1451 },
1452 .ivsize = DES3_EDE_BLOCK_SIZE,
1453 .maxauthsize = SHA1_DIGEST_SIZE,
1454 .setkey = des3_aead_setkey,
1455 },
1456 .hash = &hash_alg_sha1,
1457 .cfg_enc = CIPH_ENCR | MOD_3DES | MOD_CBC_ENC | KEYLEN_192,
1458 .cfg_dec = CIPH_DECR | MOD_3DES | MOD_CBC_DEC | KEYLEN_192,
1459 }, {
1460 .crypto = {
1461 .base = {
1462 .cra_name = "authenc(hmac(md5),cbc(aes))",
1463 .cra_blocksize = AES_BLOCK_SIZE,
1464 },
1465 .ivsize = AES_BLOCK_SIZE,
1466 .maxauthsize = MD5_DIGEST_SIZE,
1467 },
1468 .hash = &hash_alg_md5,
1469 .cfg_enc = CIPH_ENCR | MOD_AES | MOD_CBC_ENC,
1470 .cfg_dec = CIPH_DECR | MOD_AES | MOD_CBC_DEC,
1471 }, {
1472 .crypto = {
1473 .base = {
1474 .cra_name = "authenc(hmac(sha1),cbc(aes))",
1475 .cra_blocksize = AES_BLOCK_SIZE,
1476 },
1477 .ivsize = AES_BLOCK_SIZE,
1478 .maxauthsize = SHA1_DIGEST_SIZE,
1479 },
1480 .hash = &hash_alg_sha1,
1481 .cfg_enc = CIPH_ENCR | MOD_AES | MOD_CBC_ENC,
1482 .cfg_dec = CIPH_DECR | MOD_AES | MOD_CBC_DEC,
1483 } };
1484
1485 #define IXP_POSTFIX "-ixp4xx"
1486
ixp_crypto_probe(struct platform_device * _pdev)1487 static int ixp_crypto_probe(struct platform_device *_pdev)
1488 {
1489 struct device *dev = &_pdev->dev;
1490 int num = ARRAY_SIZE(ixp4xx_algos);
1491 int i, err;
1492
1493 pdev = _pdev;
1494
1495 err = init_ixp_crypto(dev);
1496 if (err)
1497 return err;
1498
1499 for (i = 0; i < num; i++) {
1500 struct skcipher_alg *cra = &ixp4xx_algos[i].crypto;
1501
1502 if (snprintf(cra->base.cra_driver_name, CRYPTO_MAX_ALG_NAME,
1503 "%s"IXP_POSTFIX, cra->base.cra_name) >=
1504 CRYPTO_MAX_ALG_NAME)
1505 continue;
1506 if (!support_aes && (ixp4xx_algos[i].cfg_enc & MOD_AES))
1507 continue;
1508
1509 /* block ciphers */
1510 cra->base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY |
1511 CRYPTO_ALG_ASYNC |
1512 CRYPTO_ALG_ALLOCATES_MEMORY |
1513 CRYPTO_ALG_NEED_FALLBACK;
1514 if (!cra->setkey)
1515 cra->setkey = ablk_setkey;
1516 if (!cra->encrypt)
1517 cra->encrypt = ablk_encrypt;
1518 if (!cra->decrypt)
1519 cra->decrypt = ablk_decrypt;
1520 cra->init = init_tfm_ablk;
1521 cra->exit = exit_tfm_ablk;
1522
1523 cra->base.cra_ctxsize = sizeof(struct ixp_ctx);
1524 cra->base.cra_module = THIS_MODULE;
1525 cra->base.cra_alignmask = 3;
1526 cra->base.cra_priority = 300;
1527 if (crypto_register_skcipher(cra))
1528 dev_err(&pdev->dev, "Failed to register '%s'\n",
1529 cra->base.cra_name);
1530 else
1531 ixp4xx_algos[i].registered = 1;
1532 }
1533
1534 for (i = 0; i < ARRAY_SIZE(ixp4xx_aeads); i++) {
1535 struct aead_alg *cra = &ixp4xx_aeads[i].crypto;
1536
1537 if (snprintf(cra->base.cra_driver_name, CRYPTO_MAX_ALG_NAME,
1538 "%s"IXP_POSTFIX, cra->base.cra_name) >=
1539 CRYPTO_MAX_ALG_NAME)
1540 continue;
1541 if (!support_aes && (ixp4xx_algos[i].cfg_enc & MOD_AES))
1542 continue;
1543
1544 /* authenc */
1545 cra->base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY |
1546 CRYPTO_ALG_ASYNC |
1547 CRYPTO_ALG_ALLOCATES_MEMORY;
1548 cra->setkey = cra->setkey ?: aead_setkey;
1549 cra->setauthsize = aead_setauthsize;
1550 cra->encrypt = aead_encrypt;
1551 cra->decrypt = aead_decrypt;
1552 cra->init = init_tfm_aead;
1553 cra->exit = exit_tfm_aead;
1554
1555 cra->base.cra_ctxsize = sizeof(struct ixp_ctx);
1556 cra->base.cra_module = THIS_MODULE;
1557 cra->base.cra_alignmask = 3;
1558 cra->base.cra_priority = 300;
1559
1560 if (crypto_register_aead(cra))
1561 dev_err(&pdev->dev, "Failed to register '%s'\n",
1562 cra->base.cra_driver_name);
1563 else
1564 ixp4xx_aeads[i].registered = 1;
1565 }
1566 return 0;
1567 }
1568
ixp_crypto_remove(struct platform_device * pdev)1569 static int ixp_crypto_remove(struct platform_device *pdev)
1570 {
1571 int num = ARRAY_SIZE(ixp4xx_algos);
1572 int i;
1573
1574 for (i = 0; i < ARRAY_SIZE(ixp4xx_aeads); i++) {
1575 if (ixp4xx_aeads[i].registered)
1576 crypto_unregister_aead(&ixp4xx_aeads[i].crypto);
1577 }
1578
1579 for (i = 0; i < num; i++) {
1580 if (ixp4xx_algos[i].registered)
1581 crypto_unregister_skcipher(&ixp4xx_algos[i].crypto);
1582 }
1583 release_ixp_crypto(&pdev->dev);
1584
1585 return 0;
1586 }
1587 static const struct of_device_id ixp4xx_crypto_of_match[] = {
1588 {
1589 .compatible = "intel,ixp4xx-crypto",
1590 },
1591 {},
1592 };
1593
1594 static struct platform_driver ixp_crypto_driver = {
1595 .probe = ixp_crypto_probe,
1596 .remove = ixp_crypto_remove,
1597 .driver = {
1598 .name = "ixp4xx_crypto",
1599 .of_match_table = ixp4xx_crypto_of_match,
1600 },
1601 };
1602 module_platform_driver(ixp_crypto_driver);
1603
1604 MODULE_LICENSE("GPL");
1605 MODULE_AUTHOR("Christian Hohnstaedt <chohnstaedt@innominate.com>");
1606 MODULE_DESCRIPTION("IXP4xx hardware crypto");
1607
1608