1 /*
2  * Copyright (C) STMicroelectronics SA 2017
3  * Author: Fabien Dessenne <fabien.dessenne@st.com>
4  * License terms:  GNU General Public License (GPL), version 2
5  */
6 
7 #include <linux/clk.h>
8 #include <linux/delay.h>
9 #include <linux/interrupt.h>
10 #include <linux/iopoll.h>
11 #include <linux/module.h>
12 #include <linux/of_device.h>
13 #include <linux/platform_device.h>
14 #include <linux/pm_runtime.h>
15 #include <linux/reset.h>
16 
17 #include <crypto/aes.h>
18 #include <crypto/des.h>
19 #include <crypto/engine.h>
20 #include <crypto/scatterwalk.h>
21 #include <crypto/internal/aead.h>
22 
23 #define DRIVER_NAME             "stm32-cryp"
24 
25 /* Bit [0] encrypt / decrypt */
26 #define FLG_ENCRYPT             BIT(0)
27 /* Bit [8..1] algo & operation mode */
28 #define FLG_AES                 BIT(1)
29 #define FLG_DES                 BIT(2)
30 #define FLG_TDES                BIT(3)
31 #define FLG_ECB                 BIT(4)
32 #define FLG_CBC                 BIT(5)
33 #define FLG_CTR                 BIT(6)
34 #define FLG_GCM                 BIT(7)
35 #define FLG_CCM                 BIT(8)
36 /* Mode mask = bits [15..0] */
37 #define FLG_MODE_MASK           GENMASK(15, 0)
38 /* Bit [31..16] status  */
39 #define FLG_CCM_PADDED_WA       BIT(16)
40 
41 /* Registers */
42 #define CRYP_CR                 0x00000000
43 #define CRYP_SR                 0x00000004
44 #define CRYP_DIN                0x00000008
45 #define CRYP_DOUT               0x0000000C
46 #define CRYP_DMACR              0x00000010
47 #define CRYP_IMSCR              0x00000014
48 #define CRYP_RISR               0x00000018
49 #define CRYP_MISR               0x0000001C
50 #define CRYP_K0LR               0x00000020
51 #define CRYP_K0RR               0x00000024
52 #define CRYP_K1LR               0x00000028
53 #define CRYP_K1RR               0x0000002C
54 #define CRYP_K2LR               0x00000030
55 #define CRYP_K2RR               0x00000034
56 #define CRYP_K3LR               0x00000038
57 #define CRYP_K3RR               0x0000003C
58 #define CRYP_IV0LR              0x00000040
59 #define CRYP_IV0RR              0x00000044
60 #define CRYP_IV1LR              0x00000048
61 #define CRYP_IV1RR              0x0000004C
62 #define CRYP_CSGCMCCM0R         0x00000050
63 #define CRYP_CSGCM0R            0x00000070
64 
65 /* Registers values */
66 #define CR_DEC_NOT_ENC          0x00000004
67 #define CR_TDES_ECB             0x00000000
68 #define CR_TDES_CBC             0x00000008
69 #define CR_DES_ECB              0x00000010
70 #define CR_DES_CBC              0x00000018
71 #define CR_AES_ECB              0x00000020
72 #define CR_AES_CBC              0x00000028
73 #define CR_AES_CTR              0x00000030
74 #define CR_AES_KP               0x00000038
75 #define CR_AES_GCM              0x00080000
76 #define CR_AES_CCM              0x00080008
77 #define CR_AES_UNKNOWN          0xFFFFFFFF
78 #define CR_ALGO_MASK            0x00080038
79 #define CR_DATA32               0x00000000
80 #define CR_DATA16               0x00000040
81 #define CR_DATA8                0x00000080
82 #define CR_DATA1                0x000000C0
83 #define CR_KEY128               0x00000000
84 #define CR_KEY192               0x00000100
85 #define CR_KEY256               0x00000200
86 #define CR_FFLUSH               0x00004000
87 #define CR_CRYPEN               0x00008000
88 #define CR_PH_INIT              0x00000000
89 #define CR_PH_HEADER            0x00010000
90 #define CR_PH_PAYLOAD           0x00020000
91 #define CR_PH_FINAL             0x00030000
92 #define CR_PH_MASK              0x00030000
93 #define CR_NBPBL_SHIFT          20
94 
95 #define SR_BUSY                 0x00000010
96 #define SR_OFNE                 0x00000004
97 
98 #define IMSCR_IN                BIT(0)
99 #define IMSCR_OUT               BIT(1)
100 
101 #define MISR_IN                 BIT(0)
102 #define MISR_OUT                BIT(1)
103 
104 /* Misc */
105 #define AES_BLOCK_32            (AES_BLOCK_SIZE / sizeof(u32))
106 #define GCM_CTR_INIT            2
107 #define _walked_in              (cryp->in_walk.offset - cryp->in_sg->offset)
108 #define _walked_out             (cryp->out_walk.offset - cryp->out_sg->offset)
109 #define CRYP_AUTOSUSPEND_DELAY	50
110 
111 struct stm32_cryp_caps {
112 	bool                    swap_final;
113 	bool                    padding_wa;
114 };
115 
116 struct stm32_cryp_ctx {
117 	struct crypto_engine_ctx enginectx;
118 	struct stm32_cryp       *cryp;
119 	int                     keylen;
120 	u32                     key[AES_KEYSIZE_256 / sizeof(u32)];
121 	unsigned long           flags;
122 };
123 
124 struct stm32_cryp_reqctx {
125 	unsigned long mode;
126 };
127 
128 struct stm32_cryp {
129 	struct list_head        list;
130 	struct device           *dev;
131 	void __iomem            *regs;
132 	struct clk              *clk;
133 	unsigned long           flags;
134 	u32                     irq_status;
135 	const struct stm32_cryp_caps *caps;
136 	struct stm32_cryp_ctx   *ctx;
137 
138 	struct crypto_engine    *engine;
139 
140 	struct ablkcipher_request *req;
141 	struct aead_request     *areq;
142 
143 	size_t                  authsize;
144 	size_t                  hw_blocksize;
145 
146 	size_t                  total_in;
147 	size_t                  total_in_save;
148 	size_t                  total_out;
149 	size_t                  total_out_save;
150 
151 	struct scatterlist      *in_sg;
152 	struct scatterlist      *out_sg;
153 	struct scatterlist      *out_sg_save;
154 
155 	struct scatterlist      in_sgl;
156 	struct scatterlist      out_sgl;
157 	bool                    sgs_copied;
158 
159 	int                     in_sg_len;
160 	int                     out_sg_len;
161 
162 	struct scatter_walk     in_walk;
163 	struct scatter_walk     out_walk;
164 
165 	u32                     last_ctr[4];
166 	u32                     gcm_ctr;
167 };
168 
169 struct stm32_cryp_list {
170 	struct list_head        dev_list;
171 	spinlock_t              lock; /* protect dev_list */
172 };
173 
174 static struct stm32_cryp_list cryp_list = {
175 	.dev_list = LIST_HEAD_INIT(cryp_list.dev_list),
176 	.lock     = __SPIN_LOCK_UNLOCKED(cryp_list.lock),
177 };
178 
179 static inline bool is_aes(struct stm32_cryp *cryp)
180 {
181 	return cryp->flags & FLG_AES;
182 }
183 
184 static inline bool is_des(struct stm32_cryp *cryp)
185 {
186 	return cryp->flags & FLG_DES;
187 }
188 
189 static inline bool is_tdes(struct stm32_cryp *cryp)
190 {
191 	return cryp->flags & FLG_TDES;
192 }
193 
194 static inline bool is_ecb(struct stm32_cryp *cryp)
195 {
196 	return cryp->flags & FLG_ECB;
197 }
198 
199 static inline bool is_cbc(struct stm32_cryp *cryp)
200 {
201 	return cryp->flags & FLG_CBC;
202 }
203 
204 static inline bool is_ctr(struct stm32_cryp *cryp)
205 {
206 	return cryp->flags & FLG_CTR;
207 }
208 
209 static inline bool is_gcm(struct stm32_cryp *cryp)
210 {
211 	return cryp->flags & FLG_GCM;
212 }
213 
214 static inline bool is_ccm(struct stm32_cryp *cryp)
215 {
216 	return cryp->flags & FLG_CCM;
217 }
218 
219 static inline bool is_encrypt(struct stm32_cryp *cryp)
220 {
221 	return cryp->flags & FLG_ENCRYPT;
222 }
223 
224 static inline bool is_decrypt(struct stm32_cryp *cryp)
225 {
226 	return !is_encrypt(cryp);
227 }
228 
229 static inline u32 stm32_cryp_read(struct stm32_cryp *cryp, u32 ofst)
230 {
231 	return readl_relaxed(cryp->regs + ofst);
232 }
233 
234 static inline void stm32_cryp_write(struct stm32_cryp *cryp, u32 ofst, u32 val)
235 {
236 	writel_relaxed(val, cryp->regs + ofst);
237 }
238 
239 static inline int stm32_cryp_wait_busy(struct stm32_cryp *cryp)
240 {
241 	u32 status;
242 
243 	return readl_relaxed_poll_timeout(cryp->regs + CRYP_SR, status,
244 			!(status & SR_BUSY), 10, 100000);
245 }
246 
247 static inline int stm32_cryp_wait_enable(struct stm32_cryp *cryp)
248 {
249 	u32 status;
250 
251 	return readl_relaxed_poll_timeout(cryp->regs + CRYP_CR, status,
252 			!(status & CR_CRYPEN), 10, 100000);
253 }
254 
255 static inline int stm32_cryp_wait_output(struct stm32_cryp *cryp)
256 {
257 	u32 status;
258 
259 	return readl_relaxed_poll_timeout(cryp->regs + CRYP_SR, status,
260 			status & SR_OFNE, 10, 100000);
261 }
262 
263 static int stm32_cryp_read_auth_tag(struct stm32_cryp *cryp);
264 
265 static struct stm32_cryp *stm32_cryp_find_dev(struct stm32_cryp_ctx *ctx)
266 {
267 	struct stm32_cryp *tmp, *cryp = NULL;
268 
269 	spin_lock_bh(&cryp_list.lock);
270 	if (!ctx->cryp) {
271 		list_for_each_entry(tmp, &cryp_list.dev_list, list) {
272 			cryp = tmp;
273 			break;
274 		}
275 		ctx->cryp = cryp;
276 	} else {
277 		cryp = ctx->cryp;
278 	}
279 
280 	spin_unlock_bh(&cryp_list.lock);
281 
282 	return cryp;
283 }
284 
285 static int stm32_cryp_check_aligned(struct scatterlist *sg, size_t total,
286 				    size_t align)
287 {
288 	int len = 0;
289 
290 	if (!total)
291 		return 0;
292 
293 	if (!IS_ALIGNED(total, align))
294 		return -EINVAL;
295 
296 	while (sg) {
297 		if (!IS_ALIGNED(sg->offset, sizeof(u32)))
298 			return -EINVAL;
299 
300 		if (!IS_ALIGNED(sg->length, align))
301 			return -EINVAL;
302 
303 		len += sg->length;
304 		sg = sg_next(sg);
305 	}
306 
307 	if (len != total)
308 		return -EINVAL;
309 
310 	return 0;
311 }
312 
313 static int stm32_cryp_check_io_aligned(struct stm32_cryp *cryp)
314 {
315 	int ret;
316 
317 	ret = stm32_cryp_check_aligned(cryp->in_sg, cryp->total_in,
318 				       cryp->hw_blocksize);
319 	if (ret)
320 		return ret;
321 
322 	ret = stm32_cryp_check_aligned(cryp->out_sg, cryp->total_out,
323 				       cryp->hw_blocksize);
324 
325 	return ret;
326 }
327 
328 static void sg_copy_buf(void *buf, struct scatterlist *sg,
329 			unsigned int start, unsigned int nbytes, int out)
330 {
331 	struct scatter_walk walk;
332 
333 	if (!nbytes)
334 		return;
335 
336 	scatterwalk_start(&walk, sg);
337 	scatterwalk_advance(&walk, start);
338 	scatterwalk_copychunks(buf, &walk, nbytes, out);
339 	scatterwalk_done(&walk, out, 0);
340 }
341 
342 static int stm32_cryp_copy_sgs(struct stm32_cryp *cryp)
343 {
344 	void *buf_in, *buf_out;
345 	int pages, total_in, total_out;
346 
347 	if (!stm32_cryp_check_io_aligned(cryp)) {
348 		cryp->sgs_copied = 0;
349 		return 0;
350 	}
351 
352 	total_in = ALIGN(cryp->total_in, cryp->hw_blocksize);
353 	pages = total_in ? get_order(total_in) : 1;
354 	buf_in = (void *)__get_free_pages(GFP_ATOMIC, pages);
355 
356 	total_out = ALIGN(cryp->total_out, cryp->hw_blocksize);
357 	pages = total_out ? get_order(total_out) : 1;
358 	buf_out = (void *)__get_free_pages(GFP_ATOMIC, pages);
359 
360 	if (!buf_in || !buf_out) {
361 		dev_err(cryp->dev, "Can't allocate pages when unaligned\n");
362 		cryp->sgs_copied = 0;
363 		return -EFAULT;
364 	}
365 
366 	sg_copy_buf(buf_in, cryp->in_sg, 0, cryp->total_in, 0);
367 
368 	sg_init_one(&cryp->in_sgl, buf_in, total_in);
369 	cryp->in_sg = &cryp->in_sgl;
370 	cryp->in_sg_len = 1;
371 
372 	sg_init_one(&cryp->out_sgl, buf_out, total_out);
373 	cryp->out_sg_save = cryp->out_sg;
374 	cryp->out_sg = &cryp->out_sgl;
375 	cryp->out_sg_len = 1;
376 
377 	cryp->sgs_copied = 1;
378 
379 	return 0;
380 }
381 
382 static void stm32_cryp_hw_write_iv(struct stm32_cryp *cryp, u32 *iv)
383 {
384 	if (!iv)
385 		return;
386 
387 	stm32_cryp_write(cryp, CRYP_IV0LR, cpu_to_be32(*iv++));
388 	stm32_cryp_write(cryp, CRYP_IV0RR, cpu_to_be32(*iv++));
389 
390 	if (is_aes(cryp)) {
391 		stm32_cryp_write(cryp, CRYP_IV1LR, cpu_to_be32(*iv++));
392 		stm32_cryp_write(cryp, CRYP_IV1RR, cpu_to_be32(*iv++));
393 	}
394 }
395 
396 static void stm32_cryp_get_iv(struct stm32_cryp *cryp)
397 {
398 	struct ablkcipher_request *req = cryp->req;
399 	u32 *tmp = req->info;
400 
401 	if (!tmp)
402 		return;
403 
404 	*tmp++ = cpu_to_be32(stm32_cryp_read(cryp, CRYP_IV0LR));
405 	*tmp++ = cpu_to_be32(stm32_cryp_read(cryp, CRYP_IV0RR));
406 
407 	if (is_aes(cryp)) {
408 		*tmp++ = cpu_to_be32(stm32_cryp_read(cryp, CRYP_IV1LR));
409 		*tmp++ = cpu_to_be32(stm32_cryp_read(cryp, CRYP_IV1RR));
410 	}
411 }
412 
413 static void stm32_cryp_hw_write_key(struct stm32_cryp *c)
414 {
415 	unsigned int i;
416 	int r_id;
417 
418 	if (is_des(c)) {
419 		stm32_cryp_write(c, CRYP_K1LR, cpu_to_be32(c->ctx->key[0]));
420 		stm32_cryp_write(c, CRYP_K1RR, cpu_to_be32(c->ctx->key[1]));
421 	} else {
422 		r_id = CRYP_K3RR;
423 		for (i = c->ctx->keylen / sizeof(u32); i > 0; i--, r_id -= 4)
424 			stm32_cryp_write(c, r_id,
425 					 cpu_to_be32(c->ctx->key[i - 1]));
426 	}
427 }
428 
429 static u32 stm32_cryp_get_hw_mode(struct stm32_cryp *cryp)
430 {
431 	if (is_aes(cryp) && is_ecb(cryp))
432 		return CR_AES_ECB;
433 
434 	if (is_aes(cryp) && is_cbc(cryp))
435 		return CR_AES_CBC;
436 
437 	if (is_aes(cryp) && is_ctr(cryp))
438 		return CR_AES_CTR;
439 
440 	if (is_aes(cryp) && is_gcm(cryp))
441 		return CR_AES_GCM;
442 
443 	if (is_aes(cryp) && is_ccm(cryp))
444 		return CR_AES_CCM;
445 
446 	if (is_des(cryp) && is_ecb(cryp))
447 		return CR_DES_ECB;
448 
449 	if (is_des(cryp) && is_cbc(cryp))
450 		return CR_DES_CBC;
451 
452 	if (is_tdes(cryp) && is_ecb(cryp))
453 		return CR_TDES_ECB;
454 
455 	if (is_tdes(cryp) && is_cbc(cryp))
456 		return CR_TDES_CBC;
457 
458 	dev_err(cryp->dev, "Unknown mode\n");
459 	return CR_AES_UNKNOWN;
460 }
461 
462 static unsigned int stm32_cryp_get_input_text_len(struct stm32_cryp *cryp)
463 {
464 	return is_encrypt(cryp) ? cryp->areq->cryptlen :
465 				  cryp->areq->cryptlen - cryp->authsize;
466 }
467 
468 static int stm32_cryp_gcm_init(struct stm32_cryp *cryp, u32 cfg)
469 {
470 	int ret;
471 	u32 iv[4];
472 
473 	/* Phase 1 : init */
474 	memcpy(iv, cryp->areq->iv, 12);
475 	iv[3] = cpu_to_be32(GCM_CTR_INIT);
476 	cryp->gcm_ctr = GCM_CTR_INIT;
477 	stm32_cryp_hw_write_iv(cryp, iv);
478 
479 	stm32_cryp_write(cryp, CRYP_CR, cfg | CR_PH_INIT | CR_CRYPEN);
480 
481 	/* Wait for end of processing */
482 	ret = stm32_cryp_wait_enable(cryp);
483 	if (ret)
484 		dev_err(cryp->dev, "Timeout (gcm init)\n");
485 
486 	return ret;
487 }
488 
489 static int stm32_cryp_ccm_init(struct stm32_cryp *cryp, u32 cfg)
490 {
491 	int ret;
492 	u8 iv[AES_BLOCK_SIZE], b0[AES_BLOCK_SIZE];
493 	u32 *d;
494 	unsigned int i, textlen;
495 
496 	/* Phase 1 : init. Firstly set the CTR value to 1 (not 0) */
497 	memcpy(iv, cryp->areq->iv, AES_BLOCK_SIZE);
498 	memset(iv + AES_BLOCK_SIZE - 1 - iv[0], 0, iv[0] + 1);
499 	iv[AES_BLOCK_SIZE - 1] = 1;
500 	stm32_cryp_hw_write_iv(cryp, (u32 *)iv);
501 
502 	/* Build B0 */
503 	memcpy(b0, iv, AES_BLOCK_SIZE);
504 
505 	b0[0] |= (8 * ((cryp->authsize - 2) / 2));
506 
507 	if (cryp->areq->assoclen)
508 		b0[0] |= 0x40;
509 
510 	textlen = stm32_cryp_get_input_text_len(cryp);
511 
512 	b0[AES_BLOCK_SIZE - 2] = textlen >> 8;
513 	b0[AES_BLOCK_SIZE - 1] = textlen & 0xFF;
514 
515 	/* Enable HW */
516 	stm32_cryp_write(cryp, CRYP_CR, cfg | CR_PH_INIT | CR_CRYPEN);
517 
518 	/* Write B0 */
519 	d = (u32 *)b0;
520 
521 	for (i = 0; i < AES_BLOCK_32; i++) {
522 		if (!cryp->caps->padding_wa)
523 			*d = cpu_to_be32(*d);
524 		stm32_cryp_write(cryp, CRYP_DIN, *d++);
525 	}
526 
527 	/* Wait for end of processing */
528 	ret = stm32_cryp_wait_enable(cryp);
529 	if (ret)
530 		dev_err(cryp->dev, "Timeout (ccm init)\n");
531 
532 	return ret;
533 }
534 
535 static int stm32_cryp_hw_init(struct stm32_cryp *cryp)
536 {
537 	int ret;
538 	u32 cfg, hw_mode;
539 
540 	pm_runtime_get_sync(cryp->dev);
541 
542 	/* Disable interrupt */
543 	stm32_cryp_write(cryp, CRYP_IMSCR, 0);
544 
545 	/* Set key */
546 	stm32_cryp_hw_write_key(cryp);
547 
548 	/* Set configuration */
549 	cfg = CR_DATA8 | CR_FFLUSH;
550 
551 	switch (cryp->ctx->keylen) {
552 	case AES_KEYSIZE_128:
553 		cfg |= CR_KEY128;
554 		break;
555 
556 	case AES_KEYSIZE_192:
557 		cfg |= CR_KEY192;
558 		break;
559 
560 	default:
561 	case AES_KEYSIZE_256:
562 		cfg |= CR_KEY256;
563 		break;
564 	}
565 
566 	hw_mode = stm32_cryp_get_hw_mode(cryp);
567 	if (hw_mode == CR_AES_UNKNOWN)
568 		return -EINVAL;
569 
570 	/* AES ECB/CBC decrypt: run key preparation first */
571 	if (is_decrypt(cryp) &&
572 	    ((hw_mode == CR_AES_ECB) || (hw_mode == CR_AES_CBC))) {
573 		stm32_cryp_write(cryp, CRYP_CR, cfg | CR_AES_KP | CR_CRYPEN);
574 
575 		/* Wait for end of processing */
576 		ret = stm32_cryp_wait_busy(cryp);
577 		if (ret) {
578 			dev_err(cryp->dev, "Timeout (key preparation)\n");
579 			return ret;
580 		}
581 	}
582 
583 	cfg |= hw_mode;
584 
585 	if (is_decrypt(cryp))
586 		cfg |= CR_DEC_NOT_ENC;
587 
588 	/* Apply config and flush (valid when CRYPEN = 0) */
589 	stm32_cryp_write(cryp, CRYP_CR, cfg);
590 
591 	switch (hw_mode) {
592 	case CR_AES_GCM:
593 	case CR_AES_CCM:
594 		/* Phase 1 : init */
595 		if (hw_mode == CR_AES_CCM)
596 			ret = stm32_cryp_ccm_init(cryp, cfg);
597 		else
598 			ret = stm32_cryp_gcm_init(cryp, cfg);
599 
600 		if (ret)
601 			return ret;
602 
603 		/* Phase 2 : header (authenticated data) */
604 		if (cryp->areq->assoclen) {
605 			cfg |= CR_PH_HEADER;
606 		} else if (stm32_cryp_get_input_text_len(cryp)) {
607 			cfg |= CR_PH_PAYLOAD;
608 			stm32_cryp_write(cryp, CRYP_CR, cfg);
609 		} else {
610 			cfg |= CR_PH_INIT;
611 		}
612 
613 		break;
614 
615 	case CR_DES_CBC:
616 	case CR_TDES_CBC:
617 	case CR_AES_CBC:
618 	case CR_AES_CTR:
619 		stm32_cryp_hw_write_iv(cryp, (u32 *)cryp->req->info);
620 		break;
621 
622 	default:
623 		break;
624 	}
625 
626 	/* Enable now */
627 	cfg |= CR_CRYPEN;
628 
629 	stm32_cryp_write(cryp, CRYP_CR, cfg);
630 
631 	cryp->flags &= ~FLG_CCM_PADDED_WA;
632 
633 	return 0;
634 }
635 
636 static void stm32_cryp_finish_req(struct stm32_cryp *cryp, int err)
637 {
638 	if (!err && (is_gcm(cryp) || is_ccm(cryp)))
639 		/* Phase 4 : output tag */
640 		err = stm32_cryp_read_auth_tag(cryp);
641 
642 	if (!err && (!(is_gcm(cryp) || is_ccm(cryp))))
643 		stm32_cryp_get_iv(cryp);
644 
645 	if (cryp->sgs_copied) {
646 		void *buf_in, *buf_out;
647 		int pages, len;
648 
649 		buf_in = sg_virt(&cryp->in_sgl);
650 		buf_out = sg_virt(&cryp->out_sgl);
651 
652 		sg_copy_buf(buf_out, cryp->out_sg_save, 0,
653 			    cryp->total_out_save, 1);
654 
655 		len = ALIGN(cryp->total_in_save, cryp->hw_blocksize);
656 		pages = len ? get_order(len) : 1;
657 		free_pages((unsigned long)buf_in, pages);
658 
659 		len = ALIGN(cryp->total_out_save, cryp->hw_blocksize);
660 		pages = len ? get_order(len) : 1;
661 		free_pages((unsigned long)buf_out, pages);
662 	}
663 
664 	pm_runtime_mark_last_busy(cryp->dev);
665 	pm_runtime_put_autosuspend(cryp->dev);
666 
667 	if (is_gcm(cryp) || is_ccm(cryp))
668 		crypto_finalize_aead_request(cryp->engine, cryp->areq, err);
669 	else
670 		crypto_finalize_ablkcipher_request(cryp->engine, cryp->req,
671 						   err);
672 
673 	memset(cryp->ctx->key, 0, cryp->ctx->keylen);
674 }
675 
676 static int stm32_cryp_cpu_start(struct stm32_cryp *cryp)
677 {
678 	/* Enable interrupt and let the IRQ handler do everything */
679 	stm32_cryp_write(cryp, CRYP_IMSCR, IMSCR_IN | IMSCR_OUT);
680 
681 	return 0;
682 }
683 
684 static int stm32_cryp_cipher_one_req(struct crypto_engine *engine, void *areq);
685 static int stm32_cryp_prepare_cipher_req(struct crypto_engine *engine,
686 					 void *areq);
687 
688 static int stm32_cryp_cra_init(struct crypto_tfm *tfm)
689 {
690 	struct stm32_cryp_ctx *ctx = crypto_tfm_ctx(tfm);
691 
692 	tfm->crt_ablkcipher.reqsize = sizeof(struct stm32_cryp_reqctx);
693 
694 	ctx->enginectx.op.do_one_request = stm32_cryp_cipher_one_req;
695 	ctx->enginectx.op.prepare_request = stm32_cryp_prepare_cipher_req;
696 	ctx->enginectx.op.unprepare_request = NULL;
697 	return 0;
698 }
699 
700 static int stm32_cryp_aead_one_req(struct crypto_engine *engine, void *areq);
701 static int stm32_cryp_prepare_aead_req(struct crypto_engine *engine,
702 				       void *areq);
703 
704 static int stm32_cryp_aes_aead_init(struct crypto_aead *tfm)
705 {
706 	struct stm32_cryp_ctx *ctx = crypto_aead_ctx(tfm);
707 
708 	tfm->reqsize = sizeof(struct stm32_cryp_reqctx);
709 
710 	ctx->enginectx.op.do_one_request = stm32_cryp_aead_one_req;
711 	ctx->enginectx.op.prepare_request = stm32_cryp_prepare_aead_req;
712 	ctx->enginectx.op.unprepare_request = NULL;
713 
714 	return 0;
715 }
716 
717 static int stm32_cryp_crypt(struct ablkcipher_request *req, unsigned long mode)
718 {
719 	struct stm32_cryp_ctx *ctx = crypto_ablkcipher_ctx(
720 			crypto_ablkcipher_reqtfm(req));
721 	struct stm32_cryp_reqctx *rctx = ablkcipher_request_ctx(req);
722 	struct stm32_cryp *cryp = stm32_cryp_find_dev(ctx);
723 
724 	if (!cryp)
725 		return -ENODEV;
726 
727 	rctx->mode = mode;
728 
729 	return crypto_transfer_ablkcipher_request_to_engine(cryp->engine, req);
730 }
731 
732 static int stm32_cryp_aead_crypt(struct aead_request *req, unsigned long mode)
733 {
734 	struct stm32_cryp_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
735 	struct stm32_cryp_reqctx *rctx = aead_request_ctx(req);
736 	struct stm32_cryp *cryp = stm32_cryp_find_dev(ctx);
737 
738 	if (!cryp)
739 		return -ENODEV;
740 
741 	rctx->mode = mode;
742 
743 	return crypto_transfer_aead_request_to_engine(cryp->engine, req);
744 }
745 
746 static int stm32_cryp_setkey(struct crypto_ablkcipher *tfm, const u8 *key,
747 			     unsigned int keylen)
748 {
749 	struct stm32_cryp_ctx *ctx = crypto_ablkcipher_ctx(tfm);
750 
751 	memcpy(ctx->key, key, keylen);
752 	ctx->keylen = keylen;
753 
754 	return 0;
755 }
756 
757 static int stm32_cryp_aes_setkey(struct crypto_ablkcipher *tfm, const u8 *key,
758 				 unsigned int keylen)
759 {
760 	if (keylen != AES_KEYSIZE_128 && keylen != AES_KEYSIZE_192 &&
761 	    keylen != AES_KEYSIZE_256)
762 		return -EINVAL;
763 	else
764 		return stm32_cryp_setkey(tfm, key, keylen);
765 }
766 
767 static int stm32_cryp_des_setkey(struct crypto_ablkcipher *tfm, const u8 *key,
768 				 unsigned int keylen)
769 {
770 	u32 tmp[DES_EXPKEY_WORDS];
771 
772 	if (keylen != DES_KEY_SIZE)
773 		return -EINVAL;
774 
775 	if ((crypto_ablkcipher_get_flags(tfm) &
776 	     CRYPTO_TFM_REQ_FORBID_WEAK_KEYS) &&
777 	    unlikely(!des_ekey(tmp, key))) {
778 		crypto_ablkcipher_set_flags(tfm, CRYPTO_TFM_RES_WEAK_KEY);
779 		return -EINVAL;
780 	}
781 
782 	return stm32_cryp_setkey(tfm, key, keylen);
783 }
784 
785 static int stm32_cryp_tdes_setkey(struct crypto_ablkcipher *tfm, const u8 *key,
786 				  unsigned int keylen)
787 {
788 	u32 flags;
789 	int err;
790 
791 	flags = crypto_ablkcipher_get_flags(tfm);
792 	err = __des3_verify_key(&flags, key);
793 	if (unlikely(err)) {
794 		crypto_ablkcipher_set_flags(tfm, flags);
795 		return err;
796 	}
797 
798 	return stm32_cryp_setkey(tfm, key, keylen);
799 }
800 
801 static int stm32_cryp_aes_aead_setkey(struct crypto_aead *tfm, const u8 *key,
802 				      unsigned int keylen)
803 {
804 	struct stm32_cryp_ctx *ctx = crypto_aead_ctx(tfm);
805 
806 	if (keylen != AES_KEYSIZE_128 && keylen != AES_KEYSIZE_192 &&
807 	    keylen != AES_KEYSIZE_256)
808 		return -EINVAL;
809 
810 	memcpy(ctx->key, key, keylen);
811 	ctx->keylen = keylen;
812 
813 	return 0;
814 }
815 
816 static int stm32_cryp_aes_gcm_setauthsize(struct crypto_aead *tfm,
817 					  unsigned int authsize)
818 {
819 	return authsize == AES_BLOCK_SIZE ? 0 : -EINVAL;
820 }
821 
822 static int stm32_cryp_aes_ccm_setauthsize(struct crypto_aead *tfm,
823 					  unsigned int authsize)
824 {
825 	switch (authsize) {
826 	case 4:
827 	case 6:
828 	case 8:
829 	case 10:
830 	case 12:
831 	case 14:
832 	case 16:
833 		break;
834 	default:
835 		return -EINVAL;
836 	}
837 
838 	return 0;
839 }
840 
841 static int stm32_cryp_aes_ecb_encrypt(struct ablkcipher_request *req)
842 {
843 	return stm32_cryp_crypt(req, FLG_AES | FLG_ECB | FLG_ENCRYPT);
844 }
845 
846 static int stm32_cryp_aes_ecb_decrypt(struct ablkcipher_request *req)
847 {
848 	return stm32_cryp_crypt(req, FLG_AES | FLG_ECB);
849 }
850 
851 static int stm32_cryp_aes_cbc_encrypt(struct ablkcipher_request *req)
852 {
853 	return stm32_cryp_crypt(req, FLG_AES | FLG_CBC | FLG_ENCRYPT);
854 }
855 
856 static int stm32_cryp_aes_cbc_decrypt(struct ablkcipher_request *req)
857 {
858 	return stm32_cryp_crypt(req, FLG_AES | FLG_CBC);
859 }
860 
861 static int stm32_cryp_aes_ctr_encrypt(struct ablkcipher_request *req)
862 {
863 	return stm32_cryp_crypt(req, FLG_AES | FLG_CTR | FLG_ENCRYPT);
864 }
865 
866 static int stm32_cryp_aes_ctr_decrypt(struct ablkcipher_request *req)
867 {
868 	return stm32_cryp_crypt(req, FLG_AES | FLG_CTR);
869 }
870 
871 static int stm32_cryp_aes_gcm_encrypt(struct aead_request *req)
872 {
873 	return stm32_cryp_aead_crypt(req, FLG_AES | FLG_GCM | FLG_ENCRYPT);
874 }
875 
876 static int stm32_cryp_aes_gcm_decrypt(struct aead_request *req)
877 {
878 	return stm32_cryp_aead_crypt(req, FLG_AES | FLG_GCM);
879 }
880 
881 static int stm32_cryp_aes_ccm_encrypt(struct aead_request *req)
882 {
883 	return stm32_cryp_aead_crypt(req, FLG_AES | FLG_CCM | FLG_ENCRYPT);
884 }
885 
886 static int stm32_cryp_aes_ccm_decrypt(struct aead_request *req)
887 {
888 	return stm32_cryp_aead_crypt(req, FLG_AES | FLG_CCM);
889 }
890 
891 static int stm32_cryp_des_ecb_encrypt(struct ablkcipher_request *req)
892 {
893 	return stm32_cryp_crypt(req, FLG_DES | FLG_ECB | FLG_ENCRYPT);
894 }
895 
896 static int stm32_cryp_des_ecb_decrypt(struct ablkcipher_request *req)
897 {
898 	return stm32_cryp_crypt(req, FLG_DES | FLG_ECB);
899 }
900 
901 static int stm32_cryp_des_cbc_encrypt(struct ablkcipher_request *req)
902 {
903 	return stm32_cryp_crypt(req, FLG_DES | FLG_CBC | FLG_ENCRYPT);
904 }
905 
906 static int stm32_cryp_des_cbc_decrypt(struct ablkcipher_request *req)
907 {
908 	return stm32_cryp_crypt(req, FLG_DES | FLG_CBC);
909 }
910 
911 static int stm32_cryp_tdes_ecb_encrypt(struct ablkcipher_request *req)
912 {
913 	return stm32_cryp_crypt(req, FLG_TDES | FLG_ECB | FLG_ENCRYPT);
914 }
915 
916 static int stm32_cryp_tdes_ecb_decrypt(struct ablkcipher_request *req)
917 {
918 	return stm32_cryp_crypt(req, FLG_TDES | FLG_ECB);
919 }
920 
921 static int stm32_cryp_tdes_cbc_encrypt(struct ablkcipher_request *req)
922 {
923 	return stm32_cryp_crypt(req, FLG_TDES | FLG_CBC | FLG_ENCRYPT);
924 }
925 
926 static int stm32_cryp_tdes_cbc_decrypt(struct ablkcipher_request *req)
927 {
928 	return stm32_cryp_crypt(req, FLG_TDES | FLG_CBC);
929 }
930 
931 static int stm32_cryp_prepare_req(struct ablkcipher_request *req,
932 				  struct aead_request *areq)
933 {
934 	struct stm32_cryp_ctx *ctx;
935 	struct stm32_cryp *cryp;
936 	struct stm32_cryp_reqctx *rctx;
937 	int ret;
938 
939 	if (!req && !areq)
940 		return -EINVAL;
941 
942 	ctx = req ? crypto_ablkcipher_ctx(crypto_ablkcipher_reqtfm(req)) :
943 		    crypto_aead_ctx(crypto_aead_reqtfm(areq));
944 
945 	cryp = ctx->cryp;
946 
947 	if (!cryp)
948 		return -ENODEV;
949 
950 	rctx = req ? ablkcipher_request_ctx(req) : aead_request_ctx(areq);
951 	rctx->mode &= FLG_MODE_MASK;
952 
953 	ctx->cryp = cryp;
954 
955 	cryp->flags = (cryp->flags & ~FLG_MODE_MASK) | rctx->mode;
956 	cryp->hw_blocksize = is_aes(cryp) ? AES_BLOCK_SIZE : DES_BLOCK_SIZE;
957 	cryp->ctx = ctx;
958 
959 	if (req) {
960 		cryp->req = req;
961 		cryp->areq = NULL;
962 		cryp->total_in = req->nbytes;
963 		cryp->total_out = cryp->total_in;
964 	} else {
965 		/*
966 		 * Length of input and output data:
967 		 * Encryption case:
968 		 *  INPUT  =   AssocData  ||   PlainText
969 		 *          <- assoclen ->  <- cryptlen ->
970 		 *          <------- total_in ----------->
971 		 *
972 		 *  OUTPUT =   AssocData  ||  CipherText  ||   AuthTag
973 		 *          <- assoclen ->  <- cryptlen ->  <- authsize ->
974 		 *          <---------------- total_out ----------------->
975 		 *
976 		 * Decryption case:
977 		 *  INPUT  =   AssocData  ||  CipherText  ||  AuthTag
978 		 *          <- assoclen ->  <--------- cryptlen --------->
979 		 *                                          <- authsize ->
980 		 *          <---------------- total_in ------------------>
981 		 *
982 		 *  OUTPUT =   AssocData  ||   PlainText
983 		 *          <- assoclen ->  <- crypten - authsize ->
984 		 *          <---------- total_out ----------------->
985 		 */
986 		cryp->areq = areq;
987 		cryp->req = NULL;
988 		cryp->authsize = crypto_aead_authsize(crypto_aead_reqtfm(areq));
989 		cryp->total_in = areq->assoclen + areq->cryptlen;
990 		if (is_encrypt(cryp))
991 			/* Append auth tag to output */
992 			cryp->total_out = cryp->total_in + cryp->authsize;
993 		else
994 			/* No auth tag in output */
995 			cryp->total_out = cryp->total_in - cryp->authsize;
996 	}
997 
998 	cryp->total_in_save = cryp->total_in;
999 	cryp->total_out_save = cryp->total_out;
1000 
1001 	cryp->in_sg = req ? req->src : areq->src;
1002 	cryp->out_sg = req ? req->dst : areq->dst;
1003 	cryp->out_sg_save = cryp->out_sg;
1004 
1005 	cryp->in_sg_len = sg_nents_for_len(cryp->in_sg, cryp->total_in);
1006 	if (cryp->in_sg_len < 0) {
1007 		dev_err(cryp->dev, "Cannot get in_sg_len\n");
1008 		ret = cryp->in_sg_len;
1009 		return ret;
1010 	}
1011 
1012 	cryp->out_sg_len = sg_nents_for_len(cryp->out_sg, cryp->total_out);
1013 	if (cryp->out_sg_len < 0) {
1014 		dev_err(cryp->dev, "Cannot get out_sg_len\n");
1015 		ret = cryp->out_sg_len;
1016 		return ret;
1017 	}
1018 
1019 	ret = stm32_cryp_copy_sgs(cryp);
1020 	if (ret)
1021 		return ret;
1022 
1023 	scatterwalk_start(&cryp->in_walk, cryp->in_sg);
1024 	scatterwalk_start(&cryp->out_walk, cryp->out_sg);
1025 
1026 	if (is_gcm(cryp) || is_ccm(cryp)) {
1027 		/* In output, jump after assoc data */
1028 		scatterwalk_advance(&cryp->out_walk, cryp->areq->assoclen);
1029 		cryp->total_out -= cryp->areq->assoclen;
1030 	}
1031 
1032 	ret = stm32_cryp_hw_init(cryp);
1033 	return ret;
1034 }
1035 
1036 static int stm32_cryp_prepare_cipher_req(struct crypto_engine *engine,
1037 					 void *areq)
1038 {
1039 	struct ablkcipher_request *req = container_of(areq,
1040 						      struct ablkcipher_request,
1041 						      base);
1042 
1043 	return stm32_cryp_prepare_req(req, NULL);
1044 }
1045 
1046 static int stm32_cryp_cipher_one_req(struct crypto_engine *engine, void *areq)
1047 {
1048 	struct ablkcipher_request *req = container_of(areq,
1049 						      struct ablkcipher_request,
1050 						      base);
1051 	struct stm32_cryp_ctx *ctx = crypto_ablkcipher_ctx(
1052 			crypto_ablkcipher_reqtfm(req));
1053 	struct stm32_cryp *cryp = ctx->cryp;
1054 
1055 	if (!cryp)
1056 		return -ENODEV;
1057 
1058 	return stm32_cryp_cpu_start(cryp);
1059 }
1060 
1061 static int stm32_cryp_prepare_aead_req(struct crypto_engine *engine, void *areq)
1062 {
1063 	struct aead_request *req = container_of(areq, struct aead_request,
1064 						base);
1065 
1066 	return stm32_cryp_prepare_req(NULL, req);
1067 }
1068 
1069 static int stm32_cryp_aead_one_req(struct crypto_engine *engine, void *areq)
1070 {
1071 	struct aead_request *req = container_of(areq, struct aead_request,
1072 						base);
1073 	struct stm32_cryp_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
1074 	struct stm32_cryp *cryp = ctx->cryp;
1075 
1076 	if (!cryp)
1077 		return -ENODEV;
1078 
1079 	if (unlikely(!cryp->areq->assoclen &&
1080 		     !stm32_cryp_get_input_text_len(cryp))) {
1081 		/* No input data to process: get tag and finish */
1082 		stm32_cryp_finish_req(cryp, 0);
1083 		return 0;
1084 	}
1085 
1086 	return stm32_cryp_cpu_start(cryp);
1087 }
1088 
1089 static u32 *stm32_cryp_next_out(struct stm32_cryp *cryp, u32 *dst,
1090 				unsigned int n)
1091 {
1092 	scatterwalk_advance(&cryp->out_walk, n);
1093 
1094 	if (unlikely(cryp->out_sg->length == _walked_out)) {
1095 		cryp->out_sg = sg_next(cryp->out_sg);
1096 		if (cryp->out_sg) {
1097 			scatterwalk_start(&cryp->out_walk, cryp->out_sg);
1098 			return (sg_virt(cryp->out_sg) + _walked_out);
1099 		}
1100 	}
1101 
1102 	return (u32 *)((u8 *)dst + n);
1103 }
1104 
1105 static u32 *stm32_cryp_next_in(struct stm32_cryp *cryp, u32 *src,
1106 			       unsigned int n)
1107 {
1108 	scatterwalk_advance(&cryp->in_walk, n);
1109 
1110 	if (unlikely(cryp->in_sg->length == _walked_in)) {
1111 		cryp->in_sg = sg_next(cryp->in_sg);
1112 		if (cryp->in_sg) {
1113 			scatterwalk_start(&cryp->in_walk, cryp->in_sg);
1114 			return (sg_virt(cryp->in_sg) + _walked_in);
1115 		}
1116 	}
1117 
1118 	return (u32 *)((u8 *)src + n);
1119 }
1120 
1121 static int stm32_cryp_read_auth_tag(struct stm32_cryp *cryp)
1122 {
1123 	u32 cfg, size_bit, *dst, d32;
1124 	u8 *d8;
1125 	unsigned int i, j;
1126 	int ret = 0;
1127 
1128 	/* Update Config */
1129 	cfg = stm32_cryp_read(cryp, CRYP_CR);
1130 
1131 	cfg &= ~CR_PH_MASK;
1132 	cfg |= CR_PH_FINAL;
1133 	cfg &= ~CR_DEC_NOT_ENC;
1134 	cfg |= CR_CRYPEN;
1135 
1136 	stm32_cryp_write(cryp, CRYP_CR, cfg);
1137 
1138 	if (is_gcm(cryp)) {
1139 		/* GCM: write aad and payload size (in bits) */
1140 		size_bit = cryp->areq->assoclen * 8;
1141 		if (cryp->caps->swap_final)
1142 			size_bit = cpu_to_be32(size_bit);
1143 
1144 		stm32_cryp_write(cryp, CRYP_DIN, 0);
1145 		stm32_cryp_write(cryp, CRYP_DIN, size_bit);
1146 
1147 		size_bit = is_encrypt(cryp) ? cryp->areq->cryptlen :
1148 				cryp->areq->cryptlen - AES_BLOCK_SIZE;
1149 		size_bit *= 8;
1150 		if (cryp->caps->swap_final)
1151 			size_bit = cpu_to_be32(size_bit);
1152 
1153 		stm32_cryp_write(cryp, CRYP_DIN, 0);
1154 		stm32_cryp_write(cryp, CRYP_DIN, size_bit);
1155 	} else {
1156 		/* CCM: write CTR0 */
1157 		u8 iv[AES_BLOCK_SIZE];
1158 		u32 *iv32 = (u32 *)iv;
1159 
1160 		memcpy(iv, cryp->areq->iv, AES_BLOCK_SIZE);
1161 		memset(iv + AES_BLOCK_SIZE - 1 - iv[0], 0, iv[0] + 1);
1162 
1163 		for (i = 0; i < AES_BLOCK_32; i++) {
1164 			if (!cryp->caps->padding_wa)
1165 				*iv32 = cpu_to_be32(*iv32);
1166 			stm32_cryp_write(cryp, CRYP_DIN, *iv32++);
1167 		}
1168 	}
1169 
1170 	/* Wait for output data */
1171 	ret = stm32_cryp_wait_output(cryp);
1172 	if (ret) {
1173 		dev_err(cryp->dev, "Timeout (read tag)\n");
1174 		return ret;
1175 	}
1176 
1177 	if (is_encrypt(cryp)) {
1178 		/* Get and write tag */
1179 		dst = sg_virt(cryp->out_sg) + _walked_out;
1180 
1181 		for (i = 0; i < AES_BLOCK_32; i++) {
1182 			if (cryp->total_out >= sizeof(u32)) {
1183 				/* Read a full u32 */
1184 				*dst = stm32_cryp_read(cryp, CRYP_DOUT);
1185 
1186 				dst = stm32_cryp_next_out(cryp, dst,
1187 							  sizeof(u32));
1188 				cryp->total_out -= sizeof(u32);
1189 			} else if (!cryp->total_out) {
1190 				/* Empty fifo out (data from input padding) */
1191 				stm32_cryp_read(cryp, CRYP_DOUT);
1192 			} else {
1193 				/* Read less than an u32 */
1194 				d32 = stm32_cryp_read(cryp, CRYP_DOUT);
1195 				d8 = (u8 *)&d32;
1196 
1197 				for (j = 0; j < cryp->total_out; j++) {
1198 					*((u8 *)dst) = *(d8++);
1199 					dst = stm32_cryp_next_out(cryp, dst, 1);
1200 				}
1201 				cryp->total_out = 0;
1202 			}
1203 		}
1204 	} else {
1205 		/* Get and check tag */
1206 		u32 in_tag[AES_BLOCK_32], out_tag[AES_BLOCK_32];
1207 
1208 		scatterwalk_map_and_copy(in_tag, cryp->in_sg,
1209 					 cryp->total_in_save - cryp->authsize,
1210 					 cryp->authsize, 0);
1211 
1212 		for (i = 0; i < AES_BLOCK_32; i++)
1213 			out_tag[i] = stm32_cryp_read(cryp, CRYP_DOUT);
1214 
1215 		if (crypto_memneq(in_tag, out_tag, cryp->authsize))
1216 			ret = -EBADMSG;
1217 	}
1218 
1219 	/* Disable cryp */
1220 	cfg &= ~CR_CRYPEN;
1221 	stm32_cryp_write(cryp, CRYP_CR, cfg);
1222 
1223 	return ret;
1224 }
1225 
1226 static void stm32_cryp_check_ctr_counter(struct stm32_cryp *cryp)
1227 {
1228 	u32 cr;
1229 
1230 	if (unlikely(cryp->last_ctr[3] == 0xFFFFFFFF)) {
1231 		cryp->last_ctr[3] = 0;
1232 		cryp->last_ctr[2]++;
1233 		if (!cryp->last_ctr[2]) {
1234 			cryp->last_ctr[1]++;
1235 			if (!cryp->last_ctr[1])
1236 				cryp->last_ctr[0]++;
1237 		}
1238 
1239 		cr = stm32_cryp_read(cryp, CRYP_CR);
1240 		stm32_cryp_write(cryp, CRYP_CR, cr & ~CR_CRYPEN);
1241 
1242 		stm32_cryp_hw_write_iv(cryp, (u32 *)cryp->last_ctr);
1243 
1244 		stm32_cryp_write(cryp, CRYP_CR, cr);
1245 	}
1246 
1247 	cryp->last_ctr[0] = stm32_cryp_read(cryp, CRYP_IV0LR);
1248 	cryp->last_ctr[1] = stm32_cryp_read(cryp, CRYP_IV0RR);
1249 	cryp->last_ctr[2] = stm32_cryp_read(cryp, CRYP_IV1LR);
1250 	cryp->last_ctr[3] = stm32_cryp_read(cryp, CRYP_IV1RR);
1251 }
1252 
1253 static bool stm32_cryp_irq_read_data(struct stm32_cryp *cryp)
1254 {
1255 	unsigned int i, j;
1256 	u32 d32, *dst;
1257 	u8 *d8;
1258 	size_t tag_size;
1259 
1260 	/* Do no read tag now (if any) */
1261 	if (is_encrypt(cryp) && (is_gcm(cryp) || is_ccm(cryp)))
1262 		tag_size = cryp->authsize;
1263 	else
1264 		tag_size = 0;
1265 
1266 	dst = sg_virt(cryp->out_sg) + _walked_out;
1267 
1268 	for (i = 0; i < cryp->hw_blocksize / sizeof(u32); i++) {
1269 		if (likely(cryp->total_out - tag_size >= sizeof(u32))) {
1270 			/* Read a full u32 */
1271 			*dst = stm32_cryp_read(cryp, CRYP_DOUT);
1272 
1273 			dst = stm32_cryp_next_out(cryp, dst, sizeof(u32));
1274 			cryp->total_out -= sizeof(u32);
1275 		} else if (cryp->total_out == tag_size) {
1276 			/* Empty fifo out (data from input padding) */
1277 			d32 = stm32_cryp_read(cryp, CRYP_DOUT);
1278 		} else {
1279 			/* Read less than an u32 */
1280 			d32 = stm32_cryp_read(cryp, CRYP_DOUT);
1281 			d8 = (u8 *)&d32;
1282 
1283 			for (j = 0; j < cryp->total_out - tag_size; j++) {
1284 				*((u8 *)dst) = *(d8++);
1285 				dst = stm32_cryp_next_out(cryp, dst, 1);
1286 			}
1287 			cryp->total_out = tag_size;
1288 		}
1289 	}
1290 
1291 	return !(cryp->total_out - tag_size) || !cryp->total_in;
1292 }
1293 
1294 static void stm32_cryp_irq_write_block(struct stm32_cryp *cryp)
1295 {
1296 	unsigned int i, j;
1297 	u32 *src;
1298 	u8 d8[4];
1299 	size_t tag_size;
1300 
1301 	/* Do no write tag (if any) */
1302 	if (is_decrypt(cryp) && (is_gcm(cryp) || is_ccm(cryp)))
1303 		tag_size = cryp->authsize;
1304 	else
1305 		tag_size = 0;
1306 
1307 	src = sg_virt(cryp->in_sg) + _walked_in;
1308 
1309 	for (i = 0; i < cryp->hw_blocksize / sizeof(u32); i++) {
1310 		if (likely(cryp->total_in - tag_size >= sizeof(u32))) {
1311 			/* Write a full u32 */
1312 			stm32_cryp_write(cryp, CRYP_DIN, *src);
1313 
1314 			src = stm32_cryp_next_in(cryp, src, sizeof(u32));
1315 			cryp->total_in -= sizeof(u32);
1316 		} else if (cryp->total_in == tag_size) {
1317 			/* Write padding data */
1318 			stm32_cryp_write(cryp, CRYP_DIN, 0);
1319 		} else {
1320 			/* Write less than an u32 */
1321 			memset(d8, 0, sizeof(u32));
1322 			for (j = 0; j < cryp->total_in - tag_size; j++) {
1323 				d8[j] = *((u8 *)src);
1324 				src = stm32_cryp_next_in(cryp, src, 1);
1325 			}
1326 
1327 			stm32_cryp_write(cryp, CRYP_DIN, *(u32 *)d8);
1328 			cryp->total_in = tag_size;
1329 		}
1330 	}
1331 }
1332 
1333 static void stm32_cryp_irq_write_gcm_padded_data(struct stm32_cryp *cryp)
1334 {
1335 	int err;
1336 	u32 cfg, tmp[AES_BLOCK_32];
1337 	size_t total_in_ori = cryp->total_in;
1338 	struct scatterlist *out_sg_ori = cryp->out_sg;
1339 	unsigned int i;
1340 
1341 	/* 'Special workaround' procedure described in the datasheet */
1342 
1343 	/* a) disable ip */
1344 	stm32_cryp_write(cryp, CRYP_IMSCR, 0);
1345 	cfg = stm32_cryp_read(cryp, CRYP_CR);
1346 	cfg &= ~CR_CRYPEN;
1347 	stm32_cryp_write(cryp, CRYP_CR, cfg);
1348 
1349 	/* b) Update IV1R */
1350 	stm32_cryp_write(cryp, CRYP_IV1RR, cryp->gcm_ctr - 2);
1351 
1352 	/* c) change mode to CTR */
1353 	cfg &= ~CR_ALGO_MASK;
1354 	cfg |= CR_AES_CTR;
1355 	stm32_cryp_write(cryp, CRYP_CR, cfg);
1356 
1357 	/* a) enable IP */
1358 	cfg |= CR_CRYPEN;
1359 	stm32_cryp_write(cryp, CRYP_CR, cfg);
1360 
1361 	/* b) pad and write the last block */
1362 	stm32_cryp_irq_write_block(cryp);
1363 	cryp->total_in = total_in_ori;
1364 	err = stm32_cryp_wait_output(cryp);
1365 	if (err) {
1366 		dev_err(cryp->dev, "Timeout (write gcm header)\n");
1367 		return stm32_cryp_finish_req(cryp, err);
1368 	}
1369 
1370 	/* c) get and store encrypted data */
1371 	stm32_cryp_irq_read_data(cryp);
1372 	scatterwalk_map_and_copy(tmp, out_sg_ori,
1373 				 cryp->total_in_save - total_in_ori,
1374 				 total_in_ori, 0);
1375 
1376 	/* d) change mode back to AES GCM */
1377 	cfg &= ~CR_ALGO_MASK;
1378 	cfg |= CR_AES_GCM;
1379 	stm32_cryp_write(cryp, CRYP_CR, cfg);
1380 
1381 	/* e) change phase to Final */
1382 	cfg &= ~CR_PH_MASK;
1383 	cfg |= CR_PH_FINAL;
1384 	stm32_cryp_write(cryp, CRYP_CR, cfg);
1385 
1386 	/* f) write padded data */
1387 	for (i = 0; i < AES_BLOCK_32; i++) {
1388 		if (cryp->total_in)
1389 			stm32_cryp_write(cryp, CRYP_DIN, tmp[i]);
1390 		else
1391 			stm32_cryp_write(cryp, CRYP_DIN, 0);
1392 
1393 		cryp->total_in -= min_t(size_t, sizeof(u32), cryp->total_in);
1394 	}
1395 
1396 	/* g) Empty fifo out */
1397 	err = stm32_cryp_wait_output(cryp);
1398 	if (err) {
1399 		dev_err(cryp->dev, "Timeout (write gcm header)\n");
1400 		return stm32_cryp_finish_req(cryp, err);
1401 	}
1402 
1403 	for (i = 0; i < AES_BLOCK_32; i++)
1404 		stm32_cryp_read(cryp, CRYP_DOUT);
1405 
1406 	/* h) run the he normal Final phase */
1407 	stm32_cryp_finish_req(cryp, 0);
1408 }
1409 
1410 static void stm32_cryp_irq_set_npblb(struct stm32_cryp *cryp)
1411 {
1412 	u32 cfg, payload_bytes;
1413 
1414 	/* disable ip, set NPBLB and reneable ip */
1415 	cfg = stm32_cryp_read(cryp, CRYP_CR);
1416 	cfg &= ~CR_CRYPEN;
1417 	stm32_cryp_write(cryp, CRYP_CR, cfg);
1418 
1419 	payload_bytes = is_decrypt(cryp) ? cryp->total_in - cryp->authsize :
1420 					   cryp->total_in;
1421 	cfg |= (cryp->hw_blocksize - payload_bytes) << CR_NBPBL_SHIFT;
1422 	cfg |= CR_CRYPEN;
1423 	stm32_cryp_write(cryp, CRYP_CR, cfg);
1424 }
1425 
1426 static void stm32_cryp_irq_write_ccm_padded_data(struct stm32_cryp *cryp)
1427 {
1428 	int err = 0;
1429 	u32 cfg, iv1tmp;
1430 	u32 cstmp1[AES_BLOCK_32], cstmp2[AES_BLOCK_32], tmp[AES_BLOCK_32];
1431 	size_t last_total_out, total_in_ori = cryp->total_in;
1432 	struct scatterlist *out_sg_ori = cryp->out_sg;
1433 	unsigned int i;
1434 
1435 	/* 'Special workaround' procedure described in the datasheet */
1436 	cryp->flags |= FLG_CCM_PADDED_WA;
1437 
1438 	/* a) disable ip */
1439 	stm32_cryp_write(cryp, CRYP_IMSCR, 0);
1440 
1441 	cfg = stm32_cryp_read(cryp, CRYP_CR);
1442 	cfg &= ~CR_CRYPEN;
1443 	stm32_cryp_write(cryp, CRYP_CR, cfg);
1444 
1445 	/* b) get IV1 from CRYP_CSGCMCCM7 */
1446 	iv1tmp = stm32_cryp_read(cryp, CRYP_CSGCMCCM0R + 7 * 4);
1447 
1448 	/* c) Load CRYP_CSGCMCCMxR */
1449 	for (i = 0; i < ARRAY_SIZE(cstmp1); i++)
1450 		cstmp1[i] = stm32_cryp_read(cryp, CRYP_CSGCMCCM0R + i * 4);
1451 
1452 	/* d) Write IV1R */
1453 	stm32_cryp_write(cryp, CRYP_IV1RR, iv1tmp);
1454 
1455 	/* e) change mode to CTR */
1456 	cfg &= ~CR_ALGO_MASK;
1457 	cfg |= CR_AES_CTR;
1458 	stm32_cryp_write(cryp, CRYP_CR, cfg);
1459 
1460 	/* a) enable IP */
1461 	cfg |= CR_CRYPEN;
1462 	stm32_cryp_write(cryp, CRYP_CR, cfg);
1463 
1464 	/* b) pad and write the last block */
1465 	stm32_cryp_irq_write_block(cryp);
1466 	cryp->total_in = total_in_ori;
1467 	err = stm32_cryp_wait_output(cryp);
1468 	if (err) {
1469 		dev_err(cryp->dev, "Timeout (wite ccm padded data)\n");
1470 		return stm32_cryp_finish_req(cryp, err);
1471 	}
1472 
1473 	/* c) get and store decrypted data */
1474 	last_total_out = cryp->total_out;
1475 	stm32_cryp_irq_read_data(cryp);
1476 
1477 	memset(tmp, 0, sizeof(tmp));
1478 	scatterwalk_map_and_copy(tmp, out_sg_ori,
1479 				 cryp->total_out_save - last_total_out,
1480 				 last_total_out, 0);
1481 
1482 	/* d) Load again CRYP_CSGCMCCMxR */
1483 	for (i = 0; i < ARRAY_SIZE(cstmp2); i++)
1484 		cstmp2[i] = stm32_cryp_read(cryp, CRYP_CSGCMCCM0R + i * 4);
1485 
1486 	/* e) change mode back to AES CCM */
1487 	cfg &= ~CR_ALGO_MASK;
1488 	cfg |= CR_AES_CCM;
1489 	stm32_cryp_write(cryp, CRYP_CR, cfg);
1490 
1491 	/* f) change phase to header */
1492 	cfg &= ~CR_PH_MASK;
1493 	cfg |= CR_PH_HEADER;
1494 	stm32_cryp_write(cryp, CRYP_CR, cfg);
1495 
1496 	/* g) XOR and write padded data */
1497 	for (i = 0; i < ARRAY_SIZE(tmp); i++) {
1498 		tmp[i] ^= cstmp1[i];
1499 		tmp[i] ^= cstmp2[i];
1500 		stm32_cryp_write(cryp, CRYP_DIN, tmp[i]);
1501 	}
1502 
1503 	/* h) wait for completion */
1504 	err = stm32_cryp_wait_busy(cryp);
1505 	if (err)
1506 		dev_err(cryp->dev, "Timeout (wite ccm padded data)\n");
1507 
1508 	/* i) run the he normal Final phase */
1509 	stm32_cryp_finish_req(cryp, err);
1510 }
1511 
1512 static void stm32_cryp_irq_write_data(struct stm32_cryp *cryp)
1513 {
1514 	if (unlikely(!cryp->total_in)) {
1515 		dev_warn(cryp->dev, "No more data to process\n");
1516 		return;
1517 	}
1518 
1519 	if (unlikely(cryp->total_in < AES_BLOCK_SIZE &&
1520 		     (stm32_cryp_get_hw_mode(cryp) == CR_AES_GCM) &&
1521 		     is_encrypt(cryp))) {
1522 		/* Padding for AES GCM encryption */
1523 		if (cryp->caps->padding_wa)
1524 			/* Special case 1 */
1525 			return stm32_cryp_irq_write_gcm_padded_data(cryp);
1526 
1527 		/* Setting padding bytes (NBBLB) */
1528 		stm32_cryp_irq_set_npblb(cryp);
1529 	}
1530 
1531 	if (unlikely((cryp->total_in - cryp->authsize < AES_BLOCK_SIZE) &&
1532 		     (stm32_cryp_get_hw_mode(cryp) == CR_AES_CCM) &&
1533 		     is_decrypt(cryp))) {
1534 		/* Padding for AES CCM decryption */
1535 		if (cryp->caps->padding_wa)
1536 			/* Special case 2 */
1537 			return stm32_cryp_irq_write_ccm_padded_data(cryp);
1538 
1539 		/* Setting padding bytes (NBBLB) */
1540 		stm32_cryp_irq_set_npblb(cryp);
1541 	}
1542 
1543 	if (is_aes(cryp) && is_ctr(cryp))
1544 		stm32_cryp_check_ctr_counter(cryp);
1545 
1546 	stm32_cryp_irq_write_block(cryp);
1547 }
1548 
1549 static void stm32_cryp_irq_write_gcm_header(struct stm32_cryp *cryp)
1550 {
1551 	int err;
1552 	unsigned int i, j;
1553 	u32 cfg, *src;
1554 
1555 	src = sg_virt(cryp->in_sg) + _walked_in;
1556 
1557 	for (i = 0; i < AES_BLOCK_32; i++) {
1558 		stm32_cryp_write(cryp, CRYP_DIN, *src);
1559 
1560 		src = stm32_cryp_next_in(cryp, src, sizeof(u32));
1561 		cryp->total_in -= min_t(size_t, sizeof(u32), cryp->total_in);
1562 
1563 		/* Check if whole header written */
1564 		if ((cryp->total_in_save - cryp->total_in) ==
1565 				cryp->areq->assoclen) {
1566 			/* Write padding if needed */
1567 			for (j = i + 1; j < AES_BLOCK_32; j++)
1568 				stm32_cryp_write(cryp, CRYP_DIN, 0);
1569 
1570 			/* Wait for completion */
1571 			err = stm32_cryp_wait_busy(cryp);
1572 			if (err) {
1573 				dev_err(cryp->dev, "Timeout (gcm header)\n");
1574 				return stm32_cryp_finish_req(cryp, err);
1575 			}
1576 
1577 			if (stm32_cryp_get_input_text_len(cryp)) {
1578 				/* Phase 3 : payload */
1579 				cfg = stm32_cryp_read(cryp, CRYP_CR);
1580 				cfg &= ~CR_CRYPEN;
1581 				stm32_cryp_write(cryp, CRYP_CR, cfg);
1582 
1583 				cfg &= ~CR_PH_MASK;
1584 				cfg |= CR_PH_PAYLOAD;
1585 				cfg |= CR_CRYPEN;
1586 				stm32_cryp_write(cryp, CRYP_CR, cfg);
1587 			} else {
1588 				/* Phase 4 : tag */
1589 				stm32_cryp_write(cryp, CRYP_IMSCR, 0);
1590 				stm32_cryp_finish_req(cryp, 0);
1591 			}
1592 
1593 			break;
1594 		}
1595 
1596 		if (!cryp->total_in)
1597 			break;
1598 	}
1599 }
1600 
1601 static void stm32_cryp_irq_write_ccm_header(struct stm32_cryp *cryp)
1602 {
1603 	int err;
1604 	unsigned int i = 0, j, k;
1605 	u32 alen, cfg, *src;
1606 	u8 d8[4];
1607 
1608 	src = sg_virt(cryp->in_sg) + _walked_in;
1609 	alen = cryp->areq->assoclen;
1610 
1611 	if (!_walked_in) {
1612 		if (cryp->areq->assoclen <= 65280) {
1613 			/* Write first u32 of B1 */
1614 			d8[0] = (alen >> 8) & 0xFF;
1615 			d8[1] = alen & 0xFF;
1616 			d8[2] = *((u8 *)src);
1617 			src = stm32_cryp_next_in(cryp, src, 1);
1618 			d8[3] = *((u8 *)src);
1619 			src = stm32_cryp_next_in(cryp, src, 1);
1620 
1621 			stm32_cryp_write(cryp, CRYP_DIN, *(u32 *)d8);
1622 			i++;
1623 
1624 			cryp->total_in -= min_t(size_t, 2, cryp->total_in);
1625 		} else {
1626 			/* Build the two first u32 of B1 */
1627 			d8[0] = 0xFF;
1628 			d8[1] = 0xFE;
1629 			d8[2] = alen & 0xFF000000;
1630 			d8[3] = alen & 0x00FF0000;
1631 
1632 			stm32_cryp_write(cryp, CRYP_DIN, *(u32 *)d8);
1633 			i++;
1634 
1635 			d8[0] = alen & 0x0000FF00;
1636 			d8[1] = alen & 0x000000FF;
1637 			d8[2] = *((u8 *)src);
1638 			src = stm32_cryp_next_in(cryp, src, 1);
1639 			d8[3] = *((u8 *)src);
1640 			src = stm32_cryp_next_in(cryp, src, 1);
1641 
1642 			stm32_cryp_write(cryp, CRYP_DIN, *(u32 *)d8);
1643 			i++;
1644 
1645 			cryp->total_in -= min_t(size_t, 2, cryp->total_in);
1646 		}
1647 	}
1648 
1649 	/* Write next u32 */
1650 	for (; i < AES_BLOCK_32; i++) {
1651 		/* Build an u32 */
1652 		memset(d8, 0, sizeof(u32));
1653 		for (k = 0; k < sizeof(u32); k++) {
1654 			d8[k] = *((u8 *)src);
1655 			src = stm32_cryp_next_in(cryp, src, 1);
1656 
1657 			cryp->total_in -= min_t(size_t, 1, cryp->total_in);
1658 			if ((cryp->total_in_save - cryp->total_in) == alen)
1659 				break;
1660 		}
1661 
1662 		stm32_cryp_write(cryp, CRYP_DIN, *(u32 *)d8);
1663 
1664 		if ((cryp->total_in_save - cryp->total_in) == alen) {
1665 			/* Write padding if needed */
1666 			for (j = i + 1; j < AES_BLOCK_32; j++)
1667 				stm32_cryp_write(cryp, CRYP_DIN, 0);
1668 
1669 			/* Wait for completion */
1670 			err = stm32_cryp_wait_busy(cryp);
1671 			if (err) {
1672 				dev_err(cryp->dev, "Timeout (ccm header)\n");
1673 				return stm32_cryp_finish_req(cryp, err);
1674 			}
1675 
1676 			if (stm32_cryp_get_input_text_len(cryp)) {
1677 				/* Phase 3 : payload */
1678 				cfg = stm32_cryp_read(cryp, CRYP_CR);
1679 				cfg &= ~CR_CRYPEN;
1680 				stm32_cryp_write(cryp, CRYP_CR, cfg);
1681 
1682 				cfg &= ~CR_PH_MASK;
1683 				cfg |= CR_PH_PAYLOAD;
1684 				cfg |= CR_CRYPEN;
1685 				stm32_cryp_write(cryp, CRYP_CR, cfg);
1686 			} else {
1687 				/* Phase 4 : tag */
1688 				stm32_cryp_write(cryp, CRYP_IMSCR, 0);
1689 				stm32_cryp_finish_req(cryp, 0);
1690 			}
1691 
1692 			break;
1693 		}
1694 	}
1695 }
1696 
1697 static irqreturn_t stm32_cryp_irq_thread(int irq, void *arg)
1698 {
1699 	struct stm32_cryp *cryp = arg;
1700 	u32 ph;
1701 
1702 	if (cryp->irq_status & MISR_OUT)
1703 		/* Output FIFO IRQ: read data */
1704 		if (unlikely(stm32_cryp_irq_read_data(cryp))) {
1705 			/* All bytes processed, finish */
1706 			stm32_cryp_write(cryp, CRYP_IMSCR, 0);
1707 			stm32_cryp_finish_req(cryp, 0);
1708 			return IRQ_HANDLED;
1709 		}
1710 
1711 	if (cryp->irq_status & MISR_IN) {
1712 		if (is_gcm(cryp)) {
1713 			ph = stm32_cryp_read(cryp, CRYP_CR) & CR_PH_MASK;
1714 			if (unlikely(ph == CR_PH_HEADER))
1715 				/* Write Header */
1716 				stm32_cryp_irq_write_gcm_header(cryp);
1717 			else
1718 				/* Input FIFO IRQ: write data */
1719 				stm32_cryp_irq_write_data(cryp);
1720 			cryp->gcm_ctr++;
1721 		} else if (is_ccm(cryp)) {
1722 			ph = stm32_cryp_read(cryp, CRYP_CR) & CR_PH_MASK;
1723 			if (unlikely(ph == CR_PH_HEADER))
1724 				/* Write Header */
1725 				stm32_cryp_irq_write_ccm_header(cryp);
1726 			else
1727 				/* Input FIFO IRQ: write data */
1728 				stm32_cryp_irq_write_data(cryp);
1729 		} else {
1730 			/* Input FIFO IRQ: write data */
1731 			stm32_cryp_irq_write_data(cryp);
1732 		}
1733 	}
1734 
1735 	return IRQ_HANDLED;
1736 }
1737 
1738 static irqreturn_t stm32_cryp_irq(int irq, void *arg)
1739 {
1740 	struct stm32_cryp *cryp = arg;
1741 
1742 	cryp->irq_status = stm32_cryp_read(cryp, CRYP_MISR);
1743 
1744 	return IRQ_WAKE_THREAD;
1745 }
1746 
1747 static struct crypto_alg crypto_algs[] = {
1748 {
1749 	.cra_name		= "ecb(aes)",
1750 	.cra_driver_name	= "stm32-ecb-aes",
1751 	.cra_priority		= 200,
1752 	.cra_flags		= CRYPTO_ALG_TYPE_ABLKCIPHER |
1753 				  CRYPTO_ALG_ASYNC,
1754 	.cra_blocksize		= AES_BLOCK_SIZE,
1755 	.cra_ctxsize		= sizeof(struct stm32_cryp_ctx),
1756 	.cra_alignmask		= 0xf,
1757 	.cra_type		= &crypto_ablkcipher_type,
1758 	.cra_module		= THIS_MODULE,
1759 	.cra_init		= stm32_cryp_cra_init,
1760 	.cra_ablkcipher = {
1761 		.min_keysize	= AES_MIN_KEY_SIZE,
1762 		.max_keysize	= AES_MAX_KEY_SIZE,
1763 		.setkey		= stm32_cryp_aes_setkey,
1764 		.encrypt	= stm32_cryp_aes_ecb_encrypt,
1765 		.decrypt	= stm32_cryp_aes_ecb_decrypt,
1766 	}
1767 },
1768 {
1769 	.cra_name		= "cbc(aes)",
1770 	.cra_driver_name	= "stm32-cbc-aes",
1771 	.cra_priority		= 200,
1772 	.cra_flags		= CRYPTO_ALG_TYPE_ABLKCIPHER |
1773 				  CRYPTO_ALG_ASYNC,
1774 	.cra_blocksize		= AES_BLOCK_SIZE,
1775 	.cra_ctxsize		= sizeof(struct stm32_cryp_ctx),
1776 	.cra_alignmask		= 0xf,
1777 	.cra_type		= &crypto_ablkcipher_type,
1778 	.cra_module		= THIS_MODULE,
1779 	.cra_init		= stm32_cryp_cra_init,
1780 	.cra_ablkcipher = {
1781 		.min_keysize	= AES_MIN_KEY_SIZE,
1782 		.max_keysize	= AES_MAX_KEY_SIZE,
1783 		.ivsize		= AES_BLOCK_SIZE,
1784 		.setkey		= stm32_cryp_aes_setkey,
1785 		.encrypt	= stm32_cryp_aes_cbc_encrypt,
1786 		.decrypt	= stm32_cryp_aes_cbc_decrypt,
1787 	}
1788 },
1789 {
1790 	.cra_name		= "ctr(aes)",
1791 	.cra_driver_name	= "stm32-ctr-aes",
1792 	.cra_priority		= 200,
1793 	.cra_flags		= CRYPTO_ALG_TYPE_ABLKCIPHER |
1794 				  CRYPTO_ALG_ASYNC,
1795 	.cra_blocksize		= 1,
1796 	.cra_ctxsize		= sizeof(struct stm32_cryp_ctx),
1797 	.cra_alignmask		= 0xf,
1798 	.cra_type		= &crypto_ablkcipher_type,
1799 	.cra_module		= THIS_MODULE,
1800 	.cra_init		= stm32_cryp_cra_init,
1801 	.cra_ablkcipher = {
1802 		.min_keysize	= AES_MIN_KEY_SIZE,
1803 		.max_keysize	= AES_MAX_KEY_SIZE,
1804 		.ivsize		= AES_BLOCK_SIZE,
1805 		.setkey		= stm32_cryp_aes_setkey,
1806 		.encrypt	= stm32_cryp_aes_ctr_encrypt,
1807 		.decrypt	= stm32_cryp_aes_ctr_decrypt,
1808 	}
1809 },
1810 {
1811 	.cra_name		= "ecb(des)",
1812 	.cra_driver_name	= "stm32-ecb-des",
1813 	.cra_priority		= 200,
1814 	.cra_flags		= CRYPTO_ALG_TYPE_ABLKCIPHER |
1815 				  CRYPTO_ALG_ASYNC,
1816 	.cra_blocksize		= DES_BLOCK_SIZE,
1817 	.cra_ctxsize		= sizeof(struct stm32_cryp_ctx),
1818 	.cra_alignmask		= 0xf,
1819 	.cra_type		= &crypto_ablkcipher_type,
1820 	.cra_module		= THIS_MODULE,
1821 	.cra_init		= stm32_cryp_cra_init,
1822 	.cra_ablkcipher = {
1823 		.min_keysize	= DES_BLOCK_SIZE,
1824 		.max_keysize	= DES_BLOCK_SIZE,
1825 		.setkey		= stm32_cryp_des_setkey,
1826 		.encrypt	= stm32_cryp_des_ecb_encrypt,
1827 		.decrypt	= stm32_cryp_des_ecb_decrypt,
1828 	}
1829 },
1830 {
1831 	.cra_name		= "cbc(des)",
1832 	.cra_driver_name	= "stm32-cbc-des",
1833 	.cra_priority		= 200,
1834 	.cra_flags		= CRYPTO_ALG_TYPE_ABLKCIPHER |
1835 				  CRYPTO_ALG_ASYNC,
1836 	.cra_blocksize		= DES_BLOCK_SIZE,
1837 	.cra_ctxsize		= sizeof(struct stm32_cryp_ctx),
1838 	.cra_alignmask		= 0xf,
1839 	.cra_type		= &crypto_ablkcipher_type,
1840 	.cra_module		= THIS_MODULE,
1841 	.cra_init		= stm32_cryp_cra_init,
1842 	.cra_ablkcipher = {
1843 		.min_keysize	= DES_BLOCK_SIZE,
1844 		.max_keysize	= DES_BLOCK_SIZE,
1845 		.ivsize		= DES_BLOCK_SIZE,
1846 		.setkey		= stm32_cryp_des_setkey,
1847 		.encrypt	= stm32_cryp_des_cbc_encrypt,
1848 		.decrypt	= stm32_cryp_des_cbc_decrypt,
1849 	}
1850 },
1851 {
1852 	.cra_name		= "ecb(des3_ede)",
1853 	.cra_driver_name	= "stm32-ecb-des3",
1854 	.cra_priority		= 200,
1855 	.cra_flags		= CRYPTO_ALG_TYPE_ABLKCIPHER |
1856 				  CRYPTO_ALG_ASYNC,
1857 	.cra_blocksize		= DES_BLOCK_SIZE,
1858 	.cra_ctxsize		= sizeof(struct stm32_cryp_ctx),
1859 	.cra_alignmask		= 0xf,
1860 	.cra_type		= &crypto_ablkcipher_type,
1861 	.cra_module		= THIS_MODULE,
1862 	.cra_init		= stm32_cryp_cra_init,
1863 	.cra_ablkcipher = {
1864 		.min_keysize	= 3 * DES_BLOCK_SIZE,
1865 		.max_keysize	= 3 * DES_BLOCK_SIZE,
1866 		.setkey		= stm32_cryp_tdes_setkey,
1867 		.encrypt	= stm32_cryp_tdes_ecb_encrypt,
1868 		.decrypt	= stm32_cryp_tdes_ecb_decrypt,
1869 	}
1870 },
1871 {
1872 	.cra_name		= "cbc(des3_ede)",
1873 	.cra_driver_name	= "stm32-cbc-des3",
1874 	.cra_priority		= 200,
1875 	.cra_flags		= CRYPTO_ALG_TYPE_ABLKCIPHER |
1876 				  CRYPTO_ALG_ASYNC,
1877 	.cra_blocksize		= DES_BLOCK_SIZE,
1878 	.cra_ctxsize		= sizeof(struct stm32_cryp_ctx),
1879 	.cra_alignmask		= 0xf,
1880 	.cra_type		= &crypto_ablkcipher_type,
1881 	.cra_module		= THIS_MODULE,
1882 	.cra_init		= stm32_cryp_cra_init,
1883 	.cra_ablkcipher = {
1884 		.min_keysize	= 3 * DES_BLOCK_SIZE,
1885 		.max_keysize	= 3 * DES_BLOCK_SIZE,
1886 		.ivsize		= DES_BLOCK_SIZE,
1887 		.setkey		= stm32_cryp_tdes_setkey,
1888 		.encrypt	= stm32_cryp_tdes_cbc_encrypt,
1889 		.decrypt	= stm32_cryp_tdes_cbc_decrypt,
1890 	}
1891 },
1892 };
1893 
1894 static struct aead_alg aead_algs[] = {
1895 {
1896 	.setkey		= stm32_cryp_aes_aead_setkey,
1897 	.setauthsize	= stm32_cryp_aes_gcm_setauthsize,
1898 	.encrypt	= stm32_cryp_aes_gcm_encrypt,
1899 	.decrypt	= stm32_cryp_aes_gcm_decrypt,
1900 	.init		= stm32_cryp_aes_aead_init,
1901 	.ivsize		= 12,
1902 	.maxauthsize	= AES_BLOCK_SIZE,
1903 
1904 	.base = {
1905 		.cra_name		= "gcm(aes)",
1906 		.cra_driver_name	= "stm32-gcm-aes",
1907 		.cra_priority		= 200,
1908 		.cra_flags		= CRYPTO_ALG_ASYNC,
1909 		.cra_blocksize		= 1,
1910 		.cra_ctxsize		= sizeof(struct stm32_cryp_ctx),
1911 		.cra_alignmask		= 0xf,
1912 		.cra_module		= THIS_MODULE,
1913 	},
1914 },
1915 {
1916 	.setkey		= stm32_cryp_aes_aead_setkey,
1917 	.setauthsize	= stm32_cryp_aes_ccm_setauthsize,
1918 	.encrypt	= stm32_cryp_aes_ccm_encrypt,
1919 	.decrypt	= stm32_cryp_aes_ccm_decrypt,
1920 	.init		= stm32_cryp_aes_aead_init,
1921 	.ivsize		= AES_BLOCK_SIZE,
1922 	.maxauthsize	= AES_BLOCK_SIZE,
1923 
1924 	.base = {
1925 		.cra_name		= "ccm(aes)",
1926 		.cra_driver_name	= "stm32-ccm-aes",
1927 		.cra_priority		= 200,
1928 		.cra_flags		= CRYPTO_ALG_ASYNC,
1929 		.cra_blocksize		= 1,
1930 		.cra_ctxsize		= sizeof(struct stm32_cryp_ctx),
1931 		.cra_alignmask		= 0xf,
1932 		.cra_module		= THIS_MODULE,
1933 	},
1934 },
1935 };
1936 
1937 static const struct stm32_cryp_caps f7_data = {
1938 	.swap_final = true,
1939 	.padding_wa = true,
1940 };
1941 
1942 static const struct stm32_cryp_caps mp1_data = {
1943 	.swap_final = false,
1944 	.padding_wa = false,
1945 };
1946 
1947 static const struct of_device_id stm32_dt_ids[] = {
1948 	{ .compatible = "st,stm32f756-cryp", .data = &f7_data},
1949 	{ .compatible = "st,stm32mp1-cryp", .data = &mp1_data},
1950 	{},
1951 };
1952 MODULE_DEVICE_TABLE(of, stm32_dt_ids);
1953 
1954 static int stm32_cryp_probe(struct platform_device *pdev)
1955 {
1956 	struct device *dev = &pdev->dev;
1957 	struct stm32_cryp *cryp;
1958 	struct resource *res;
1959 	struct reset_control *rst;
1960 	int irq, ret;
1961 
1962 	cryp = devm_kzalloc(dev, sizeof(*cryp), GFP_KERNEL);
1963 	if (!cryp)
1964 		return -ENOMEM;
1965 
1966 	cryp->caps = of_device_get_match_data(dev);
1967 	if (!cryp->caps)
1968 		return -ENODEV;
1969 
1970 	cryp->dev = dev;
1971 
1972 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1973 	cryp->regs = devm_ioremap_resource(dev, res);
1974 	if (IS_ERR(cryp->regs))
1975 		return PTR_ERR(cryp->regs);
1976 
1977 	irq = platform_get_irq(pdev, 0);
1978 	if (irq < 0) {
1979 		dev_err(dev, "Cannot get IRQ resource\n");
1980 		return irq;
1981 	}
1982 
1983 	ret = devm_request_threaded_irq(dev, irq, stm32_cryp_irq,
1984 					stm32_cryp_irq_thread, IRQF_ONESHOT,
1985 					dev_name(dev), cryp);
1986 	if (ret) {
1987 		dev_err(dev, "Cannot grab IRQ\n");
1988 		return ret;
1989 	}
1990 
1991 	cryp->clk = devm_clk_get(dev, NULL);
1992 	if (IS_ERR(cryp->clk)) {
1993 		dev_err(dev, "Could not get clock\n");
1994 		return PTR_ERR(cryp->clk);
1995 	}
1996 
1997 	ret = clk_prepare_enable(cryp->clk);
1998 	if (ret) {
1999 		dev_err(cryp->dev, "Failed to enable clock\n");
2000 		return ret;
2001 	}
2002 
2003 	pm_runtime_set_autosuspend_delay(dev, CRYP_AUTOSUSPEND_DELAY);
2004 	pm_runtime_use_autosuspend(dev);
2005 
2006 	pm_runtime_get_noresume(dev);
2007 	pm_runtime_set_active(dev);
2008 	pm_runtime_enable(dev);
2009 
2010 	rst = devm_reset_control_get(dev, NULL);
2011 	if (!IS_ERR(rst)) {
2012 		reset_control_assert(rst);
2013 		udelay(2);
2014 		reset_control_deassert(rst);
2015 	}
2016 
2017 	platform_set_drvdata(pdev, cryp);
2018 
2019 	spin_lock(&cryp_list.lock);
2020 	list_add(&cryp->list, &cryp_list.dev_list);
2021 	spin_unlock(&cryp_list.lock);
2022 
2023 	/* Initialize crypto engine */
2024 	cryp->engine = crypto_engine_alloc_init(dev, 1);
2025 	if (!cryp->engine) {
2026 		dev_err(dev, "Could not init crypto engine\n");
2027 		ret = -ENOMEM;
2028 		goto err_engine1;
2029 	}
2030 
2031 	ret = crypto_engine_start(cryp->engine);
2032 	if (ret) {
2033 		dev_err(dev, "Could not start crypto engine\n");
2034 		goto err_engine2;
2035 	}
2036 
2037 	ret = crypto_register_algs(crypto_algs, ARRAY_SIZE(crypto_algs));
2038 	if (ret) {
2039 		dev_err(dev, "Could not register algs\n");
2040 		goto err_algs;
2041 	}
2042 
2043 	ret = crypto_register_aeads(aead_algs, ARRAY_SIZE(aead_algs));
2044 	if (ret)
2045 		goto err_aead_algs;
2046 
2047 	dev_info(dev, "Initialized\n");
2048 
2049 	pm_runtime_put_sync(dev);
2050 
2051 	return 0;
2052 
2053 err_aead_algs:
2054 	crypto_unregister_algs(crypto_algs, ARRAY_SIZE(crypto_algs));
2055 err_algs:
2056 err_engine2:
2057 	crypto_engine_exit(cryp->engine);
2058 err_engine1:
2059 	spin_lock(&cryp_list.lock);
2060 	list_del(&cryp->list);
2061 	spin_unlock(&cryp_list.lock);
2062 
2063 	pm_runtime_disable(dev);
2064 	pm_runtime_put_noidle(dev);
2065 	pm_runtime_disable(dev);
2066 	pm_runtime_put_noidle(dev);
2067 
2068 	clk_disable_unprepare(cryp->clk);
2069 
2070 	return ret;
2071 }
2072 
2073 static int stm32_cryp_remove(struct platform_device *pdev)
2074 {
2075 	struct stm32_cryp *cryp = platform_get_drvdata(pdev);
2076 	int ret;
2077 
2078 	if (!cryp)
2079 		return -ENODEV;
2080 
2081 	ret = pm_runtime_get_sync(cryp->dev);
2082 	if (ret < 0)
2083 		return ret;
2084 
2085 	crypto_unregister_aeads(aead_algs, ARRAY_SIZE(aead_algs));
2086 	crypto_unregister_algs(crypto_algs, ARRAY_SIZE(crypto_algs));
2087 
2088 	crypto_engine_exit(cryp->engine);
2089 
2090 	spin_lock(&cryp_list.lock);
2091 	list_del(&cryp->list);
2092 	spin_unlock(&cryp_list.lock);
2093 
2094 	pm_runtime_disable(cryp->dev);
2095 	pm_runtime_put_noidle(cryp->dev);
2096 
2097 	clk_disable_unprepare(cryp->clk);
2098 
2099 	return 0;
2100 }
2101 
2102 #ifdef CONFIG_PM
2103 static int stm32_cryp_runtime_suspend(struct device *dev)
2104 {
2105 	struct stm32_cryp *cryp = dev_get_drvdata(dev);
2106 
2107 	clk_disable_unprepare(cryp->clk);
2108 
2109 	return 0;
2110 }
2111 
2112 static int stm32_cryp_runtime_resume(struct device *dev)
2113 {
2114 	struct stm32_cryp *cryp = dev_get_drvdata(dev);
2115 	int ret;
2116 
2117 	ret = clk_prepare_enable(cryp->clk);
2118 	if (ret) {
2119 		dev_err(cryp->dev, "Failed to prepare_enable clock\n");
2120 		return ret;
2121 	}
2122 
2123 	return 0;
2124 }
2125 #endif
2126 
2127 static const struct dev_pm_ops stm32_cryp_pm_ops = {
2128 	SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend,
2129 				pm_runtime_force_resume)
2130 	SET_RUNTIME_PM_OPS(stm32_cryp_runtime_suspend,
2131 			   stm32_cryp_runtime_resume, NULL)
2132 };
2133 
2134 static struct platform_driver stm32_cryp_driver = {
2135 	.probe  = stm32_cryp_probe,
2136 	.remove = stm32_cryp_remove,
2137 	.driver = {
2138 		.name           = DRIVER_NAME,
2139 		.pm		= &stm32_cryp_pm_ops,
2140 		.of_match_table = stm32_dt_ids,
2141 	},
2142 };
2143 
2144 module_platform_driver(stm32_cryp_driver);
2145 
2146 MODULE_AUTHOR("Fabien Dessenne <fabien.dessenne@st.com>");
2147 MODULE_DESCRIPTION("STMicrolectronics STM32 CRYP hardware driver");
2148 MODULE_LICENSE("GPL");
2149