1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright (C) STMicroelectronics SA 2017
4  * Author: Fabien Dessenne <fabien.dessenne@st.com>
5  */
6 
7 #include <linux/clk.h>
8 #include <linux/delay.h>
9 #include <linux/interrupt.h>
10 #include <linux/iopoll.h>
11 #include <linux/module.h>
12 #include <linux/of_device.h>
13 #include <linux/platform_device.h>
14 #include <linux/pm_runtime.h>
15 #include <linux/reset.h>
16 
17 #include <crypto/aes.h>
18 #include <crypto/internal/des.h>
19 #include <crypto/engine.h>
20 #include <crypto/scatterwalk.h>
21 #include <crypto/internal/aead.h>
22 #include <crypto/internal/skcipher.h>
23 
24 #define DRIVER_NAME             "stm32-cryp"
25 
26 /* Bit [0] encrypt / decrypt */
27 #define FLG_ENCRYPT             BIT(0)
28 /* Bit [8..1] algo & operation mode */
29 #define FLG_AES                 BIT(1)
30 #define FLG_DES                 BIT(2)
31 #define FLG_TDES                BIT(3)
32 #define FLG_ECB                 BIT(4)
33 #define FLG_CBC                 BIT(5)
34 #define FLG_CTR                 BIT(6)
35 #define FLG_GCM                 BIT(7)
36 #define FLG_CCM                 BIT(8)
37 /* Mode mask = bits [15..0] */
38 #define FLG_MODE_MASK           GENMASK(15, 0)
39 /* Bit [31..16] status  */
40 #define FLG_CCM_PADDED_WA       BIT(16)
41 
42 /* Registers */
43 #define CRYP_CR                 0x00000000
44 #define CRYP_SR                 0x00000004
45 #define CRYP_DIN                0x00000008
46 #define CRYP_DOUT               0x0000000C
47 #define CRYP_DMACR              0x00000010
48 #define CRYP_IMSCR              0x00000014
49 #define CRYP_RISR               0x00000018
50 #define CRYP_MISR               0x0000001C
51 #define CRYP_K0LR               0x00000020
52 #define CRYP_K0RR               0x00000024
53 #define CRYP_K1LR               0x00000028
54 #define CRYP_K1RR               0x0000002C
55 #define CRYP_K2LR               0x00000030
56 #define CRYP_K2RR               0x00000034
57 #define CRYP_K3LR               0x00000038
58 #define CRYP_K3RR               0x0000003C
59 #define CRYP_IV0LR              0x00000040
60 #define CRYP_IV0RR              0x00000044
61 #define CRYP_IV1LR              0x00000048
62 #define CRYP_IV1RR              0x0000004C
63 #define CRYP_CSGCMCCM0R         0x00000050
64 #define CRYP_CSGCM0R            0x00000070
65 
66 /* Registers values */
67 #define CR_DEC_NOT_ENC          0x00000004
68 #define CR_TDES_ECB             0x00000000
69 #define CR_TDES_CBC             0x00000008
70 #define CR_DES_ECB              0x00000010
71 #define CR_DES_CBC              0x00000018
72 #define CR_AES_ECB              0x00000020
73 #define CR_AES_CBC              0x00000028
74 #define CR_AES_CTR              0x00000030
75 #define CR_AES_KP               0x00000038
76 #define CR_AES_GCM              0x00080000
77 #define CR_AES_CCM              0x00080008
78 #define CR_AES_UNKNOWN          0xFFFFFFFF
79 #define CR_ALGO_MASK            0x00080038
80 #define CR_DATA32               0x00000000
81 #define CR_DATA16               0x00000040
82 #define CR_DATA8                0x00000080
83 #define CR_DATA1                0x000000C0
84 #define CR_KEY128               0x00000000
85 #define CR_KEY192               0x00000100
86 #define CR_KEY256               0x00000200
87 #define CR_FFLUSH               0x00004000
88 #define CR_CRYPEN               0x00008000
89 #define CR_PH_INIT              0x00000000
90 #define CR_PH_HEADER            0x00010000
91 #define CR_PH_PAYLOAD           0x00020000
92 #define CR_PH_FINAL             0x00030000
93 #define CR_PH_MASK              0x00030000
94 #define CR_NBPBL_SHIFT          20
95 
96 #define SR_BUSY                 0x00000010
97 #define SR_OFNE                 0x00000004
98 
99 #define IMSCR_IN                BIT(0)
100 #define IMSCR_OUT               BIT(1)
101 
102 #define MISR_IN                 BIT(0)
103 #define MISR_OUT                BIT(1)
104 
105 /* Misc */
106 #define AES_BLOCK_32            (AES_BLOCK_SIZE / sizeof(u32))
107 #define GCM_CTR_INIT            2
108 #define _walked_in              (cryp->in_walk.offset - cryp->in_sg->offset)
109 #define _walked_out             (cryp->out_walk.offset - cryp->out_sg->offset)
110 #define CRYP_AUTOSUSPEND_DELAY	50
111 
112 struct stm32_cryp_caps {
113 	bool                    swap_final;
114 	bool                    padding_wa;
115 };
116 
117 struct stm32_cryp_ctx {
118 	struct crypto_engine_ctx enginectx;
119 	struct stm32_cryp       *cryp;
120 	int                     keylen;
121 	__be32                  key[AES_KEYSIZE_256 / sizeof(u32)];
122 	unsigned long           flags;
123 };
124 
125 struct stm32_cryp_reqctx {
126 	unsigned long mode;
127 };
128 
129 struct stm32_cryp {
130 	struct list_head        list;
131 	struct device           *dev;
132 	void __iomem            *regs;
133 	struct clk              *clk;
134 	unsigned long           flags;
135 	u32                     irq_status;
136 	const struct stm32_cryp_caps *caps;
137 	struct stm32_cryp_ctx   *ctx;
138 
139 	struct crypto_engine    *engine;
140 
141 	struct skcipher_request *req;
142 	struct aead_request     *areq;
143 
144 	size_t                  authsize;
145 	size_t                  hw_blocksize;
146 
147 	size_t                  total_in;
148 	size_t                  total_in_save;
149 	size_t                  total_out;
150 	size_t                  total_out_save;
151 
152 	struct scatterlist      *in_sg;
153 	struct scatterlist      *out_sg;
154 	struct scatterlist      *out_sg_save;
155 
156 	struct scatterlist      in_sgl;
157 	struct scatterlist      out_sgl;
158 	bool                    sgs_copied;
159 
160 	int                     in_sg_len;
161 	int                     out_sg_len;
162 
163 	struct scatter_walk     in_walk;
164 	struct scatter_walk     out_walk;
165 
166 	u32                     last_ctr[4];
167 	u32                     gcm_ctr;
168 };
169 
170 struct stm32_cryp_list {
171 	struct list_head        dev_list;
172 	spinlock_t              lock; /* protect dev_list */
173 };
174 
175 static struct stm32_cryp_list cryp_list = {
176 	.dev_list = LIST_HEAD_INIT(cryp_list.dev_list),
177 	.lock     = __SPIN_LOCK_UNLOCKED(cryp_list.lock),
178 };
179 
180 static inline bool is_aes(struct stm32_cryp *cryp)
181 {
182 	return cryp->flags & FLG_AES;
183 }
184 
185 static inline bool is_des(struct stm32_cryp *cryp)
186 {
187 	return cryp->flags & FLG_DES;
188 }
189 
190 static inline bool is_tdes(struct stm32_cryp *cryp)
191 {
192 	return cryp->flags & FLG_TDES;
193 }
194 
195 static inline bool is_ecb(struct stm32_cryp *cryp)
196 {
197 	return cryp->flags & FLG_ECB;
198 }
199 
200 static inline bool is_cbc(struct stm32_cryp *cryp)
201 {
202 	return cryp->flags & FLG_CBC;
203 }
204 
205 static inline bool is_ctr(struct stm32_cryp *cryp)
206 {
207 	return cryp->flags & FLG_CTR;
208 }
209 
210 static inline bool is_gcm(struct stm32_cryp *cryp)
211 {
212 	return cryp->flags & FLG_GCM;
213 }
214 
215 static inline bool is_ccm(struct stm32_cryp *cryp)
216 {
217 	return cryp->flags & FLG_CCM;
218 }
219 
220 static inline bool is_encrypt(struct stm32_cryp *cryp)
221 {
222 	return cryp->flags & FLG_ENCRYPT;
223 }
224 
225 static inline bool is_decrypt(struct stm32_cryp *cryp)
226 {
227 	return !is_encrypt(cryp);
228 }
229 
230 static inline u32 stm32_cryp_read(struct stm32_cryp *cryp, u32 ofst)
231 {
232 	return readl_relaxed(cryp->regs + ofst);
233 }
234 
235 static inline void stm32_cryp_write(struct stm32_cryp *cryp, u32 ofst, u32 val)
236 {
237 	writel_relaxed(val, cryp->regs + ofst);
238 }
239 
240 static inline int stm32_cryp_wait_busy(struct stm32_cryp *cryp)
241 {
242 	u32 status;
243 
244 	return readl_relaxed_poll_timeout(cryp->regs + CRYP_SR, status,
245 			!(status & SR_BUSY), 10, 100000);
246 }
247 
248 static inline int stm32_cryp_wait_enable(struct stm32_cryp *cryp)
249 {
250 	u32 status;
251 
252 	return readl_relaxed_poll_timeout(cryp->regs + CRYP_CR, status,
253 			!(status & CR_CRYPEN), 10, 100000);
254 }
255 
256 static inline int stm32_cryp_wait_output(struct stm32_cryp *cryp)
257 {
258 	u32 status;
259 
260 	return readl_relaxed_poll_timeout(cryp->regs + CRYP_SR, status,
261 			status & SR_OFNE, 10, 100000);
262 }
263 
264 static int stm32_cryp_read_auth_tag(struct stm32_cryp *cryp);
265 
266 static struct stm32_cryp *stm32_cryp_find_dev(struct stm32_cryp_ctx *ctx)
267 {
268 	struct stm32_cryp *tmp, *cryp = NULL;
269 
270 	spin_lock_bh(&cryp_list.lock);
271 	if (!ctx->cryp) {
272 		list_for_each_entry(tmp, &cryp_list.dev_list, list) {
273 			cryp = tmp;
274 			break;
275 		}
276 		ctx->cryp = cryp;
277 	} else {
278 		cryp = ctx->cryp;
279 	}
280 
281 	spin_unlock_bh(&cryp_list.lock);
282 
283 	return cryp;
284 }
285 
286 static int stm32_cryp_check_aligned(struct scatterlist *sg, size_t total,
287 				    size_t align)
288 {
289 	int len = 0;
290 
291 	if (!total)
292 		return 0;
293 
294 	if (!IS_ALIGNED(total, align))
295 		return -EINVAL;
296 
297 	while (sg) {
298 		if (!IS_ALIGNED(sg->offset, sizeof(u32)))
299 			return -EINVAL;
300 
301 		if (!IS_ALIGNED(sg->length, align))
302 			return -EINVAL;
303 
304 		len += sg->length;
305 		sg = sg_next(sg);
306 	}
307 
308 	if (len != total)
309 		return -EINVAL;
310 
311 	return 0;
312 }
313 
314 static int stm32_cryp_check_io_aligned(struct stm32_cryp *cryp)
315 {
316 	int ret;
317 
318 	ret = stm32_cryp_check_aligned(cryp->in_sg, cryp->total_in,
319 				       cryp->hw_blocksize);
320 	if (ret)
321 		return ret;
322 
323 	ret = stm32_cryp_check_aligned(cryp->out_sg, cryp->total_out,
324 				       cryp->hw_blocksize);
325 
326 	return ret;
327 }
328 
329 static void sg_copy_buf(void *buf, struct scatterlist *sg,
330 			unsigned int start, unsigned int nbytes, int out)
331 {
332 	struct scatter_walk walk;
333 
334 	if (!nbytes)
335 		return;
336 
337 	scatterwalk_start(&walk, sg);
338 	scatterwalk_advance(&walk, start);
339 	scatterwalk_copychunks(buf, &walk, nbytes, out);
340 	scatterwalk_done(&walk, out, 0);
341 }
342 
343 static int stm32_cryp_copy_sgs(struct stm32_cryp *cryp)
344 {
345 	void *buf_in, *buf_out;
346 	int pages, total_in, total_out;
347 
348 	if (!stm32_cryp_check_io_aligned(cryp)) {
349 		cryp->sgs_copied = 0;
350 		return 0;
351 	}
352 
353 	total_in = ALIGN(cryp->total_in, cryp->hw_blocksize);
354 	pages = total_in ? get_order(total_in) : 1;
355 	buf_in = (void *)__get_free_pages(GFP_ATOMIC, pages);
356 
357 	total_out = ALIGN(cryp->total_out, cryp->hw_blocksize);
358 	pages = total_out ? get_order(total_out) : 1;
359 	buf_out = (void *)__get_free_pages(GFP_ATOMIC, pages);
360 
361 	if (!buf_in || !buf_out) {
362 		dev_err(cryp->dev, "Can't allocate pages when unaligned\n");
363 		cryp->sgs_copied = 0;
364 		return -EFAULT;
365 	}
366 
367 	sg_copy_buf(buf_in, cryp->in_sg, 0, cryp->total_in, 0);
368 
369 	sg_init_one(&cryp->in_sgl, buf_in, total_in);
370 	cryp->in_sg = &cryp->in_sgl;
371 	cryp->in_sg_len = 1;
372 
373 	sg_init_one(&cryp->out_sgl, buf_out, total_out);
374 	cryp->out_sg_save = cryp->out_sg;
375 	cryp->out_sg = &cryp->out_sgl;
376 	cryp->out_sg_len = 1;
377 
378 	cryp->sgs_copied = 1;
379 
380 	return 0;
381 }
382 
383 static void stm32_cryp_hw_write_iv(struct stm32_cryp *cryp, __be32 *iv)
384 {
385 	if (!iv)
386 		return;
387 
388 	stm32_cryp_write(cryp, CRYP_IV0LR, be32_to_cpu(*iv++));
389 	stm32_cryp_write(cryp, CRYP_IV0RR, be32_to_cpu(*iv++));
390 
391 	if (is_aes(cryp)) {
392 		stm32_cryp_write(cryp, CRYP_IV1LR, be32_to_cpu(*iv++));
393 		stm32_cryp_write(cryp, CRYP_IV1RR, be32_to_cpu(*iv++));
394 	}
395 }
396 
397 static void stm32_cryp_get_iv(struct stm32_cryp *cryp)
398 {
399 	struct skcipher_request *req = cryp->req;
400 	__be32 *tmp = (void *)req->iv;
401 
402 	if (!tmp)
403 		return;
404 
405 	*tmp++ = cpu_to_be32(stm32_cryp_read(cryp, CRYP_IV0LR));
406 	*tmp++ = cpu_to_be32(stm32_cryp_read(cryp, CRYP_IV0RR));
407 
408 	if (is_aes(cryp)) {
409 		*tmp++ = cpu_to_be32(stm32_cryp_read(cryp, CRYP_IV1LR));
410 		*tmp++ = cpu_to_be32(stm32_cryp_read(cryp, CRYP_IV1RR));
411 	}
412 }
413 
414 static void stm32_cryp_hw_write_key(struct stm32_cryp *c)
415 {
416 	unsigned int i;
417 	int r_id;
418 
419 	if (is_des(c)) {
420 		stm32_cryp_write(c, CRYP_K1LR, be32_to_cpu(c->ctx->key[0]));
421 		stm32_cryp_write(c, CRYP_K1RR, be32_to_cpu(c->ctx->key[1]));
422 	} else {
423 		r_id = CRYP_K3RR;
424 		for (i = c->ctx->keylen / sizeof(u32); i > 0; i--, r_id -= 4)
425 			stm32_cryp_write(c, r_id,
426 					 be32_to_cpu(c->ctx->key[i - 1]));
427 	}
428 }
429 
430 static u32 stm32_cryp_get_hw_mode(struct stm32_cryp *cryp)
431 {
432 	if (is_aes(cryp) && is_ecb(cryp))
433 		return CR_AES_ECB;
434 
435 	if (is_aes(cryp) && is_cbc(cryp))
436 		return CR_AES_CBC;
437 
438 	if (is_aes(cryp) && is_ctr(cryp))
439 		return CR_AES_CTR;
440 
441 	if (is_aes(cryp) && is_gcm(cryp))
442 		return CR_AES_GCM;
443 
444 	if (is_aes(cryp) && is_ccm(cryp))
445 		return CR_AES_CCM;
446 
447 	if (is_des(cryp) && is_ecb(cryp))
448 		return CR_DES_ECB;
449 
450 	if (is_des(cryp) && is_cbc(cryp))
451 		return CR_DES_CBC;
452 
453 	if (is_tdes(cryp) && is_ecb(cryp))
454 		return CR_TDES_ECB;
455 
456 	if (is_tdes(cryp) && is_cbc(cryp))
457 		return CR_TDES_CBC;
458 
459 	dev_err(cryp->dev, "Unknown mode\n");
460 	return CR_AES_UNKNOWN;
461 }
462 
463 static unsigned int stm32_cryp_get_input_text_len(struct stm32_cryp *cryp)
464 {
465 	return is_encrypt(cryp) ? cryp->areq->cryptlen :
466 				  cryp->areq->cryptlen - cryp->authsize;
467 }
468 
469 static int stm32_cryp_gcm_init(struct stm32_cryp *cryp, u32 cfg)
470 {
471 	int ret;
472 	__be32 iv[4];
473 
474 	/* Phase 1 : init */
475 	memcpy(iv, cryp->areq->iv, 12);
476 	iv[3] = cpu_to_be32(GCM_CTR_INIT);
477 	cryp->gcm_ctr = GCM_CTR_INIT;
478 	stm32_cryp_hw_write_iv(cryp, iv);
479 
480 	stm32_cryp_write(cryp, CRYP_CR, cfg | CR_PH_INIT | CR_CRYPEN);
481 
482 	/* Wait for end of processing */
483 	ret = stm32_cryp_wait_enable(cryp);
484 	if (ret)
485 		dev_err(cryp->dev, "Timeout (gcm init)\n");
486 
487 	return ret;
488 }
489 
490 static int stm32_cryp_ccm_init(struct stm32_cryp *cryp, u32 cfg)
491 {
492 	int ret;
493 	u8 iv[AES_BLOCK_SIZE], b0[AES_BLOCK_SIZE];
494 	__be32 *bd;
495 	u32 *d;
496 	unsigned int i, textlen;
497 
498 	/* Phase 1 : init. Firstly set the CTR value to 1 (not 0) */
499 	memcpy(iv, cryp->areq->iv, AES_BLOCK_SIZE);
500 	memset(iv + AES_BLOCK_SIZE - 1 - iv[0], 0, iv[0] + 1);
501 	iv[AES_BLOCK_SIZE - 1] = 1;
502 	stm32_cryp_hw_write_iv(cryp, (__be32 *)iv);
503 
504 	/* Build B0 */
505 	memcpy(b0, iv, AES_BLOCK_SIZE);
506 
507 	b0[0] |= (8 * ((cryp->authsize - 2) / 2));
508 
509 	if (cryp->areq->assoclen)
510 		b0[0] |= 0x40;
511 
512 	textlen = stm32_cryp_get_input_text_len(cryp);
513 
514 	b0[AES_BLOCK_SIZE - 2] = textlen >> 8;
515 	b0[AES_BLOCK_SIZE - 1] = textlen & 0xFF;
516 
517 	/* Enable HW */
518 	stm32_cryp_write(cryp, CRYP_CR, cfg | CR_PH_INIT | CR_CRYPEN);
519 
520 	/* Write B0 */
521 	d = (u32 *)b0;
522 	bd = (__be32 *)b0;
523 
524 	for (i = 0; i < AES_BLOCK_32; i++) {
525 		u32 xd = d[i];
526 
527 		if (!cryp->caps->padding_wa)
528 			xd = be32_to_cpu(bd[i]);
529 		stm32_cryp_write(cryp, CRYP_DIN, xd);
530 	}
531 
532 	/* Wait for end of processing */
533 	ret = stm32_cryp_wait_enable(cryp);
534 	if (ret)
535 		dev_err(cryp->dev, "Timeout (ccm init)\n");
536 
537 	return ret;
538 }
539 
540 static int stm32_cryp_hw_init(struct stm32_cryp *cryp)
541 {
542 	int ret;
543 	u32 cfg, hw_mode;
544 
545 	pm_runtime_get_sync(cryp->dev);
546 
547 	/* Disable interrupt */
548 	stm32_cryp_write(cryp, CRYP_IMSCR, 0);
549 
550 	/* Set key */
551 	stm32_cryp_hw_write_key(cryp);
552 
553 	/* Set configuration */
554 	cfg = CR_DATA8 | CR_FFLUSH;
555 
556 	switch (cryp->ctx->keylen) {
557 	case AES_KEYSIZE_128:
558 		cfg |= CR_KEY128;
559 		break;
560 
561 	case AES_KEYSIZE_192:
562 		cfg |= CR_KEY192;
563 		break;
564 
565 	default:
566 	case AES_KEYSIZE_256:
567 		cfg |= CR_KEY256;
568 		break;
569 	}
570 
571 	hw_mode = stm32_cryp_get_hw_mode(cryp);
572 	if (hw_mode == CR_AES_UNKNOWN)
573 		return -EINVAL;
574 
575 	/* AES ECB/CBC decrypt: run key preparation first */
576 	if (is_decrypt(cryp) &&
577 	    ((hw_mode == CR_AES_ECB) || (hw_mode == CR_AES_CBC))) {
578 		stm32_cryp_write(cryp, CRYP_CR, cfg | CR_AES_KP | CR_CRYPEN);
579 
580 		/* Wait for end of processing */
581 		ret = stm32_cryp_wait_busy(cryp);
582 		if (ret) {
583 			dev_err(cryp->dev, "Timeout (key preparation)\n");
584 			return ret;
585 		}
586 	}
587 
588 	cfg |= hw_mode;
589 
590 	if (is_decrypt(cryp))
591 		cfg |= CR_DEC_NOT_ENC;
592 
593 	/* Apply config and flush (valid when CRYPEN = 0) */
594 	stm32_cryp_write(cryp, CRYP_CR, cfg);
595 
596 	switch (hw_mode) {
597 	case CR_AES_GCM:
598 	case CR_AES_CCM:
599 		/* Phase 1 : init */
600 		if (hw_mode == CR_AES_CCM)
601 			ret = stm32_cryp_ccm_init(cryp, cfg);
602 		else
603 			ret = stm32_cryp_gcm_init(cryp, cfg);
604 
605 		if (ret)
606 			return ret;
607 
608 		/* Phase 2 : header (authenticated data) */
609 		if (cryp->areq->assoclen) {
610 			cfg |= CR_PH_HEADER;
611 		} else if (stm32_cryp_get_input_text_len(cryp)) {
612 			cfg |= CR_PH_PAYLOAD;
613 			stm32_cryp_write(cryp, CRYP_CR, cfg);
614 		} else {
615 			cfg |= CR_PH_INIT;
616 		}
617 
618 		break;
619 
620 	case CR_DES_CBC:
621 	case CR_TDES_CBC:
622 	case CR_AES_CBC:
623 	case CR_AES_CTR:
624 		stm32_cryp_hw_write_iv(cryp, (__be32 *)cryp->req->iv);
625 		break;
626 
627 	default:
628 		break;
629 	}
630 
631 	/* Enable now */
632 	cfg |= CR_CRYPEN;
633 
634 	stm32_cryp_write(cryp, CRYP_CR, cfg);
635 
636 	cryp->flags &= ~FLG_CCM_PADDED_WA;
637 
638 	return 0;
639 }
640 
641 static void stm32_cryp_finish_req(struct stm32_cryp *cryp, int err)
642 {
643 	if (!err && (is_gcm(cryp) || is_ccm(cryp)))
644 		/* Phase 4 : output tag */
645 		err = stm32_cryp_read_auth_tag(cryp);
646 
647 	if (!err && (!(is_gcm(cryp) || is_ccm(cryp))))
648 		stm32_cryp_get_iv(cryp);
649 
650 	if (cryp->sgs_copied) {
651 		void *buf_in, *buf_out;
652 		int pages, len;
653 
654 		buf_in = sg_virt(&cryp->in_sgl);
655 		buf_out = sg_virt(&cryp->out_sgl);
656 
657 		sg_copy_buf(buf_out, cryp->out_sg_save, 0,
658 			    cryp->total_out_save, 1);
659 
660 		len = ALIGN(cryp->total_in_save, cryp->hw_blocksize);
661 		pages = len ? get_order(len) : 1;
662 		free_pages((unsigned long)buf_in, pages);
663 
664 		len = ALIGN(cryp->total_out_save, cryp->hw_blocksize);
665 		pages = len ? get_order(len) : 1;
666 		free_pages((unsigned long)buf_out, pages);
667 	}
668 
669 	pm_runtime_mark_last_busy(cryp->dev);
670 	pm_runtime_put_autosuspend(cryp->dev);
671 
672 	if (is_gcm(cryp) || is_ccm(cryp))
673 		crypto_finalize_aead_request(cryp->engine, cryp->areq, err);
674 	else
675 		crypto_finalize_skcipher_request(cryp->engine, cryp->req,
676 						   err);
677 
678 	memset(cryp->ctx->key, 0, cryp->ctx->keylen);
679 }
680 
681 static int stm32_cryp_cpu_start(struct stm32_cryp *cryp)
682 {
683 	/* Enable interrupt and let the IRQ handler do everything */
684 	stm32_cryp_write(cryp, CRYP_IMSCR, IMSCR_IN | IMSCR_OUT);
685 
686 	return 0;
687 }
688 
689 static int stm32_cryp_cipher_one_req(struct crypto_engine *engine, void *areq);
690 static int stm32_cryp_prepare_cipher_req(struct crypto_engine *engine,
691 					 void *areq);
692 
693 static int stm32_cryp_init_tfm(struct crypto_skcipher *tfm)
694 {
695 	struct stm32_cryp_ctx *ctx = crypto_skcipher_ctx(tfm);
696 
697 	crypto_skcipher_set_reqsize(tfm, sizeof(struct stm32_cryp_reqctx));
698 
699 	ctx->enginectx.op.do_one_request = stm32_cryp_cipher_one_req;
700 	ctx->enginectx.op.prepare_request = stm32_cryp_prepare_cipher_req;
701 	ctx->enginectx.op.unprepare_request = NULL;
702 	return 0;
703 }
704 
705 static int stm32_cryp_aead_one_req(struct crypto_engine *engine, void *areq);
706 static int stm32_cryp_prepare_aead_req(struct crypto_engine *engine,
707 				       void *areq);
708 
709 static int stm32_cryp_aes_aead_init(struct crypto_aead *tfm)
710 {
711 	struct stm32_cryp_ctx *ctx = crypto_aead_ctx(tfm);
712 
713 	tfm->reqsize = sizeof(struct stm32_cryp_reqctx);
714 
715 	ctx->enginectx.op.do_one_request = stm32_cryp_aead_one_req;
716 	ctx->enginectx.op.prepare_request = stm32_cryp_prepare_aead_req;
717 	ctx->enginectx.op.unprepare_request = NULL;
718 
719 	return 0;
720 }
721 
722 static int stm32_cryp_crypt(struct skcipher_request *req, unsigned long mode)
723 {
724 	struct stm32_cryp_ctx *ctx = crypto_skcipher_ctx(
725 			crypto_skcipher_reqtfm(req));
726 	struct stm32_cryp_reqctx *rctx = skcipher_request_ctx(req);
727 	struct stm32_cryp *cryp = stm32_cryp_find_dev(ctx);
728 
729 	if (!cryp)
730 		return -ENODEV;
731 
732 	rctx->mode = mode;
733 
734 	return crypto_transfer_skcipher_request_to_engine(cryp->engine, req);
735 }
736 
737 static int stm32_cryp_aead_crypt(struct aead_request *req, unsigned long mode)
738 {
739 	struct stm32_cryp_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
740 	struct stm32_cryp_reqctx *rctx = aead_request_ctx(req);
741 	struct stm32_cryp *cryp = stm32_cryp_find_dev(ctx);
742 
743 	if (!cryp)
744 		return -ENODEV;
745 
746 	rctx->mode = mode;
747 
748 	return crypto_transfer_aead_request_to_engine(cryp->engine, req);
749 }
750 
751 static int stm32_cryp_setkey(struct crypto_skcipher *tfm, const u8 *key,
752 			     unsigned int keylen)
753 {
754 	struct stm32_cryp_ctx *ctx = crypto_skcipher_ctx(tfm);
755 
756 	memcpy(ctx->key, key, keylen);
757 	ctx->keylen = keylen;
758 
759 	return 0;
760 }
761 
762 static int stm32_cryp_aes_setkey(struct crypto_skcipher *tfm, const u8 *key,
763 				 unsigned int keylen)
764 {
765 	if (keylen != AES_KEYSIZE_128 && keylen != AES_KEYSIZE_192 &&
766 	    keylen != AES_KEYSIZE_256)
767 		return -EINVAL;
768 	else
769 		return stm32_cryp_setkey(tfm, key, keylen);
770 }
771 
772 static int stm32_cryp_des_setkey(struct crypto_skcipher *tfm, const u8 *key,
773 				 unsigned int keylen)
774 {
775 	return verify_skcipher_des_key(tfm, key) ?:
776 	       stm32_cryp_setkey(tfm, key, keylen);
777 }
778 
779 static int stm32_cryp_tdes_setkey(struct crypto_skcipher *tfm, const u8 *key,
780 				  unsigned int keylen)
781 {
782 	return verify_skcipher_des3_key(tfm, key) ?:
783 	       stm32_cryp_setkey(tfm, key, keylen);
784 }
785 
786 static int stm32_cryp_aes_aead_setkey(struct crypto_aead *tfm, const u8 *key,
787 				      unsigned int keylen)
788 {
789 	struct stm32_cryp_ctx *ctx = crypto_aead_ctx(tfm);
790 
791 	if (keylen != AES_KEYSIZE_128 && keylen != AES_KEYSIZE_192 &&
792 	    keylen != AES_KEYSIZE_256)
793 		return -EINVAL;
794 
795 	memcpy(ctx->key, key, keylen);
796 	ctx->keylen = keylen;
797 
798 	return 0;
799 }
800 
801 static int stm32_cryp_aes_gcm_setauthsize(struct crypto_aead *tfm,
802 					  unsigned int authsize)
803 {
804 	return authsize == AES_BLOCK_SIZE ? 0 : -EINVAL;
805 }
806 
807 static int stm32_cryp_aes_ccm_setauthsize(struct crypto_aead *tfm,
808 					  unsigned int authsize)
809 {
810 	switch (authsize) {
811 	case 4:
812 	case 6:
813 	case 8:
814 	case 10:
815 	case 12:
816 	case 14:
817 	case 16:
818 		break;
819 	default:
820 		return -EINVAL;
821 	}
822 
823 	return 0;
824 }
825 
826 static int stm32_cryp_aes_ecb_encrypt(struct skcipher_request *req)
827 {
828 	return stm32_cryp_crypt(req, FLG_AES | FLG_ECB | FLG_ENCRYPT);
829 }
830 
831 static int stm32_cryp_aes_ecb_decrypt(struct skcipher_request *req)
832 {
833 	return stm32_cryp_crypt(req, FLG_AES | FLG_ECB);
834 }
835 
836 static int stm32_cryp_aes_cbc_encrypt(struct skcipher_request *req)
837 {
838 	return stm32_cryp_crypt(req, FLG_AES | FLG_CBC | FLG_ENCRYPT);
839 }
840 
841 static int stm32_cryp_aes_cbc_decrypt(struct skcipher_request *req)
842 {
843 	return stm32_cryp_crypt(req, FLG_AES | FLG_CBC);
844 }
845 
846 static int stm32_cryp_aes_ctr_encrypt(struct skcipher_request *req)
847 {
848 	return stm32_cryp_crypt(req, FLG_AES | FLG_CTR | FLG_ENCRYPT);
849 }
850 
851 static int stm32_cryp_aes_ctr_decrypt(struct skcipher_request *req)
852 {
853 	return stm32_cryp_crypt(req, FLG_AES | FLG_CTR);
854 }
855 
856 static int stm32_cryp_aes_gcm_encrypt(struct aead_request *req)
857 {
858 	return stm32_cryp_aead_crypt(req, FLG_AES | FLG_GCM | FLG_ENCRYPT);
859 }
860 
861 static int stm32_cryp_aes_gcm_decrypt(struct aead_request *req)
862 {
863 	return stm32_cryp_aead_crypt(req, FLG_AES | FLG_GCM);
864 }
865 
866 static int stm32_cryp_aes_ccm_encrypt(struct aead_request *req)
867 {
868 	return stm32_cryp_aead_crypt(req, FLG_AES | FLG_CCM | FLG_ENCRYPT);
869 }
870 
871 static int stm32_cryp_aes_ccm_decrypt(struct aead_request *req)
872 {
873 	return stm32_cryp_aead_crypt(req, FLG_AES | FLG_CCM);
874 }
875 
876 static int stm32_cryp_des_ecb_encrypt(struct skcipher_request *req)
877 {
878 	return stm32_cryp_crypt(req, FLG_DES | FLG_ECB | FLG_ENCRYPT);
879 }
880 
881 static int stm32_cryp_des_ecb_decrypt(struct skcipher_request *req)
882 {
883 	return stm32_cryp_crypt(req, FLG_DES | FLG_ECB);
884 }
885 
886 static int stm32_cryp_des_cbc_encrypt(struct skcipher_request *req)
887 {
888 	return stm32_cryp_crypt(req, FLG_DES | FLG_CBC | FLG_ENCRYPT);
889 }
890 
891 static int stm32_cryp_des_cbc_decrypt(struct skcipher_request *req)
892 {
893 	return stm32_cryp_crypt(req, FLG_DES | FLG_CBC);
894 }
895 
896 static int stm32_cryp_tdes_ecb_encrypt(struct skcipher_request *req)
897 {
898 	return stm32_cryp_crypt(req, FLG_TDES | FLG_ECB | FLG_ENCRYPT);
899 }
900 
901 static int stm32_cryp_tdes_ecb_decrypt(struct skcipher_request *req)
902 {
903 	return stm32_cryp_crypt(req, FLG_TDES | FLG_ECB);
904 }
905 
906 static int stm32_cryp_tdes_cbc_encrypt(struct skcipher_request *req)
907 {
908 	return stm32_cryp_crypt(req, FLG_TDES | FLG_CBC | FLG_ENCRYPT);
909 }
910 
911 static int stm32_cryp_tdes_cbc_decrypt(struct skcipher_request *req)
912 {
913 	return stm32_cryp_crypt(req, FLG_TDES | FLG_CBC);
914 }
915 
916 static int stm32_cryp_prepare_req(struct skcipher_request *req,
917 				  struct aead_request *areq)
918 {
919 	struct stm32_cryp_ctx *ctx;
920 	struct stm32_cryp *cryp;
921 	struct stm32_cryp_reqctx *rctx;
922 	int ret;
923 
924 	if (!req && !areq)
925 		return -EINVAL;
926 
927 	ctx = req ? crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)) :
928 		    crypto_aead_ctx(crypto_aead_reqtfm(areq));
929 
930 	cryp = ctx->cryp;
931 
932 	if (!cryp)
933 		return -ENODEV;
934 
935 	rctx = req ? skcipher_request_ctx(req) : aead_request_ctx(areq);
936 	rctx->mode &= FLG_MODE_MASK;
937 
938 	ctx->cryp = cryp;
939 
940 	cryp->flags = (cryp->flags & ~FLG_MODE_MASK) | rctx->mode;
941 	cryp->hw_blocksize = is_aes(cryp) ? AES_BLOCK_SIZE : DES_BLOCK_SIZE;
942 	cryp->ctx = ctx;
943 
944 	if (req) {
945 		cryp->req = req;
946 		cryp->areq = NULL;
947 		cryp->total_in = req->cryptlen;
948 		cryp->total_out = cryp->total_in;
949 	} else {
950 		/*
951 		 * Length of input and output data:
952 		 * Encryption case:
953 		 *  INPUT  =   AssocData  ||   PlainText
954 		 *          <- assoclen ->  <- cryptlen ->
955 		 *          <------- total_in ----------->
956 		 *
957 		 *  OUTPUT =   AssocData  ||  CipherText  ||   AuthTag
958 		 *          <- assoclen ->  <- cryptlen ->  <- authsize ->
959 		 *          <---------------- total_out ----------------->
960 		 *
961 		 * Decryption case:
962 		 *  INPUT  =   AssocData  ||  CipherText  ||  AuthTag
963 		 *          <- assoclen ->  <--------- cryptlen --------->
964 		 *                                          <- authsize ->
965 		 *          <---------------- total_in ------------------>
966 		 *
967 		 *  OUTPUT =   AssocData  ||   PlainText
968 		 *          <- assoclen ->  <- crypten - authsize ->
969 		 *          <---------- total_out ----------------->
970 		 */
971 		cryp->areq = areq;
972 		cryp->req = NULL;
973 		cryp->authsize = crypto_aead_authsize(crypto_aead_reqtfm(areq));
974 		cryp->total_in = areq->assoclen + areq->cryptlen;
975 		if (is_encrypt(cryp))
976 			/* Append auth tag to output */
977 			cryp->total_out = cryp->total_in + cryp->authsize;
978 		else
979 			/* No auth tag in output */
980 			cryp->total_out = cryp->total_in - cryp->authsize;
981 	}
982 
983 	cryp->total_in_save = cryp->total_in;
984 	cryp->total_out_save = cryp->total_out;
985 
986 	cryp->in_sg = req ? req->src : areq->src;
987 	cryp->out_sg = req ? req->dst : areq->dst;
988 	cryp->out_sg_save = cryp->out_sg;
989 
990 	cryp->in_sg_len = sg_nents_for_len(cryp->in_sg, cryp->total_in);
991 	if (cryp->in_sg_len < 0) {
992 		dev_err(cryp->dev, "Cannot get in_sg_len\n");
993 		ret = cryp->in_sg_len;
994 		return ret;
995 	}
996 
997 	cryp->out_sg_len = sg_nents_for_len(cryp->out_sg, cryp->total_out);
998 	if (cryp->out_sg_len < 0) {
999 		dev_err(cryp->dev, "Cannot get out_sg_len\n");
1000 		ret = cryp->out_sg_len;
1001 		return ret;
1002 	}
1003 
1004 	ret = stm32_cryp_copy_sgs(cryp);
1005 	if (ret)
1006 		return ret;
1007 
1008 	scatterwalk_start(&cryp->in_walk, cryp->in_sg);
1009 	scatterwalk_start(&cryp->out_walk, cryp->out_sg);
1010 
1011 	if (is_gcm(cryp) || is_ccm(cryp)) {
1012 		/* In output, jump after assoc data */
1013 		scatterwalk_advance(&cryp->out_walk, cryp->areq->assoclen);
1014 		cryp->total_out -= cryp->areq->assoclen;
1015 	}
1016 
1017 	ret = stm32_cryp_hw_init(cryp);
1018 	return ret;
1019 }
1020 
1021 static int stm32_cryp_prepare_cipher_req(struct crypto_engine *engine,
1022 					 void *areq)
1023 {
1024 	struct skcipher_request *req = container_of(areq,
1025 						      struct skcipher_request,
1026 						      base);
1027 
1028 	return stm32_cryp_prepare_req(req, NULL);
1029 }
1030 
1031 static int stm32_cryp_cipher_one_req(struct crypto_engine *engine, void *areq)
1032 {
1033 	struct skcipher_request *req = container_of(areq,
1034 						      struct skcipher_request,
1035 						      base);
1036 	struct stm32_cryp_ctx *ctx = crypto_skcipher_ctx(
1037 			crypto_skcipher_reqtfm(req));
1038 	struct stm32_cryp *cryp = ctx->cryp;
1039 
1040 	if (!cryp)
1041 		return -ENODEV;
1042 
1043 	return stm32_cryp_cpu_start(cryp);
1044 }
1045 
1046 static int stm32_cryp_prepare_aead_req(struct crypto_engine *engine, void *areq)
1047 {
1048 	struct aead_request *req = container_of(areq, struct aead_request,
1049 						base);
1050 
1051 	return stm32_cryp_prepare_req(NULL, req);
1052 }
1053 
1054 static int stm32_cryp_aead_one_req(struct crypto_engine *engine, void *areq)
1055 {
1056 	struct aead_request *req = container_of(areq, struct aead_request,
1057 						base);
1058 	struct stm32_cryp_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
1059 	struct stm32_cryp *cryp = ctx->cryp;
1060 
1061 	if (!cryp)
1062 		return -ENODEV;
1063 
1064 	if (unlikely(!cryp->areq->assoclen &&
1065 		     !stm32_cryp_get_input_text_len(cryp))) {
1066 		/* No input data to process: get tag and finish */
1067 		stm32_cryp_finish_req(cryp, 0);
1068 		return 0;
1069 	}
1070 
1071 	return stm32_cryp_cpu_start(cryp);
1072 }
1073 
1074 static u32 *stm32_cryp_next_out(struct stm32_cryp *cryp, u32 *dst,
1075 				unsigned int n)
1076 {
1077 	scatterwalk_advance(&cryp->out_walk, n);
1078 
1079 	if (unlikely(cryp->out_sg->length == _walked_out)) {
1080 		cryp->out_sg = sg_next(cryp->out_sg);
1081 		if (cryp->out_sg) {
1082 			scatterwalk_start(&cryp->out_walk, cryp->out_sg);
1083 			return (sg_virt(cryp->out_sg) + _walked_out);
1084 		}
1085 	}
1086 
1087 	return (u32 *)((u8 *)dst + n);
1088 }
1089 
1090 static u32 *stm32_cryp_next_in(struct stm32_cryp *cryp, u32 *src,
1091 			       unsigned int n)
1092 {
1093 	scatterwalk_advance(&cryp->in_walk, n);
1094 
1095 	if (unlikely(cryp->in_sg->length == _walked_in)) {
1096 		cryp->in_sg = sg_next(cryp->in_sg);
1097 		if (cryp->in_sg) {
1098 			scatterwalk_start(&cryp->in_walk, cryp->in_sg);
1099 			return (sg_virt(cryp->in_sg) + _walked_in);
1100 		}
1101 	}
1102 
1103 	return (u32 *)((u8 *)src + n);
1104 }
1105 
1106 static int stm32_cryp_read_auth_tag(struct stm32_cryp *cryp)
1107 {
1108 	u32 cfg, size_bit, *dst, d32;
1109 	u8 *d8;
1110 	unsigned int i, j;
1111 	int ret = 0;
1112 
1113 	/* Update Config */
1114 	cfg = stm32_cryp_read(cryp, CRYP_CR);
1115 
1116 	cfg &= ~CR_PH_MASK;
1117 	cfg |= CR_PH_FINAL;
1118 	cfg &= ~CR_DEC_NOT_ENC;
1119 	cfg |= CR_CRYPEN;
1120 
1121 	stm32_cryp_write(cryp, CRYP_CR, cfg);
1122 
1123 	if (is_gcm(cryp)) {
1124 		/* GCM: write aad and payload size (in bits) */
1125 		size_bit = cryp->areq->assoclen * 8;
1126 		if (cryp->caps->swap_final)
1127 			size_bit = (__force u32)cpu_to_be32(size_bit);
1128 
1129 		stm32_cryp_write(cryp, CRYP_DIN, 0);
1130 		stm32_cryp_write(cryp, CRYP_DIN, size_bit);
1131 
1132 		size_bit = is_encrypt(cryp) ? cryp->areq->cryptlen :
1133 				cryp->areq->cryptlen - AES_BLOCK_SIZE;
1134 		size_bit *= 8;
1135 		if (cryp->caps->swap_final)
1136 			size_bit = (__force u32)cpu_to_be32(size_bit);
1137 
1138 		stm32_cryp_write(cryp, CRYP_DIN, 0);
1139 		stm32_cryp_write(cryp, CRYP_DIN, size_bit);
1140 	} else {
1141 		/* CCM: write CTR0 */
1142 		u8 iv[AES_BLOCK_SIZE];
1143 		u32 *iv32 = (u32 *)iv;
1144 		__be32 *biv;
1145 
1146 		biv = (void *)iv;
1147 
1148 		memcpy(iv, cryp->areq->iv, AES_BLOCK_SIZE);
1149 		memset(iv + AES_BLOCK_SIZE - 1 - iv[0], 0, iv[0] + 1);
1150 
1151 		for (i = 0; i < AES_BLOCK_32; i++) {
1152 			u32 xiv = iv32[i];
1153 
1154 			if (!cryp->caps->padding_wa)
1155 				xiv = be32_to_cpu(biv[i]);
1156 			stm32_cryp_write(cryp, CRYP_DIN, xiv);
1157 		}
1158 	}
1159 
1160 	/* Wait for output data */
1161 	ret = stm32_cryp_wait_output(cryp);
1162 	if (ret) {
1163 		dev_err(cryp->dev, "Timeout (read tag)\n");
1164 		return ret;
1165 	}
1166 
1167 	if (is_encrypt(cryp)) {
1168 		/* Get and write tag */
1169 		dst = sg_virt(cryp->out_sg) + _walked_out;
1170 
1171 		for (i = 0; i < AES_BLOCK_32; i++) {
1172 			if (cryp->total_out >= sizeof(u32)) {
1173 				/* Read a full u32 */
1174 				*dst = stm32_cryp_read(cryp, CRYP_DOUT);
1175 
1176 				dst = stm32_cryp_next_out(cryp, dst,
1177 							  sizeof(u32));
1178 				cryp->total_out -= sizeof(u32);
1179 			} else if (!cryp->total_out) {
1180 				/* Empty fifo out (data from input padding) */
1181 				stm32_cryp_read(cryp, CRYP_DOUT);
1182 			} else {
1183 				/* Read less than an u32 */
1184 				d32 = stm32_cryp_read(cryp, CRYP_DOUT);
1185 				d8 = (u8 *)&d32;
1186 
1187 				for (j = 0; j < cryp->total_out; j++) {
1188 					*((u8 *)dst) = *(d8++);
1189 					dst = stm32_cryp_next_out(cryp, dst, 1);
1190 				}
1191 				cryp->total_out = 0;
1192 			}
1193 		}
1194 	} else {
1195 		/* Get and check tag */
1196 		u32 in_tag[AES_BLOCK_32], out_tag[AES_BLOCK_32];
1197 
1198 		scatterwalk_map_and_copy(in_tag, cryp->in_sg,
1199 					 cryp->total_in_save - cryp->authsize,
1200 					 cryp->authsize, 0);
1201 
1202 		for (i = 0; i < AES_BLOCK_32; i++)
1203 			out_tag[i] = stm32_cryp_read(cryp, CRYP_DOUT);
1204 
1205 		if (crypto_memneq(in_tag, out_tag, cryp->authsize))
1206 			ret = -EBADMSG;
1207 	}
1208 
1209 	/* Disable cryp */
1210 	cfg &= ~CR_CRYPEN;
1211 	stm32_cryp_write(cryp, CRYP_CR, cfg);
1212 
1213 	return ret;
1214 }
1215 
1216 static void stm32_cryp_check_ctr_counter(struct stm32_cryp *cryp)
1217 {
1218 	u32 cr;
1219 
1220 	if (unlikely(cryp->last_ctr[3] == 0xFFFFFFFF)) {
1221 		cryp->last_ctr[3] = 0;
1222 		cryp->last_ctr[2]++;
1223 		if (!cryp->last_ctr[2]) {
1224 			cryp->last_ctr[1]++;
1225 			if (!cryp->last_ctr[1])
1226 				cryp->last_ctr[0]++;
1227 		}
1228 
1229 		cr = stm32_cryp_read(cryp, CRYP_CR);
1230 		stm32_cryp_write(cryp, CRYP_CR, cr & ~CR_CRYPEN);
1231 
1232 		stm32_cryp_hw_write_iv(cryp, (u32 *)cryp->last_ctr);
1233 
1234 		stm32_cryp_write(cryp, CRYP_CR, cr);
1235 	}
1236 
1237 	cryp->last_ctr[0] = stm32_cryp_read(cryp, CRYP_IV0LR);
1238 	cryp->last_ctr[1] = stm32_cryp_read(cryp, CRYP_IV0RR);
1239 	cryp->last_ctr[2] = stm32_cryp_read(cryp, CRYP_IV1LR);
1240 	cryp->last_ctr[3] = stm32_cryp_read(cryp, CRYP_IV1RR);
1241 }
1242 
1243 static bool stm32_cryp_irq_read_data(struct stm32_cryp *cryp)
1244 {
1245 	unsigned int i, j;
1246 	u32 d32, *dst;
1247 	u8 *d8;
1248 	size_t tag_size;
1249 
1250 	/* Do no read tag now (if any) */
1251 	if (is_encrypt(cryp) && (is_gcm(cryp) || is_ccm(cryp)))
1252 		tag_size = cryp->authsize;
1253 	else
1254 		tag_size = 0;
1255 
1256 	dst = sg_virt(cryp->out_sg) + _walked_out;
1257 
1258 	for (i = 0; i < cryp->hw_blocksize / sizeof(u32); i++) {
1259 		if (likely(cryp->total_out - tag_size >= sizeof(u32))) {
1260 			/* Read a full u32 */
1261 			*dst = stm32_cryp_read(cryp, CRYP_DOUT);
1262 
1263 			dst = stm32_cryp_next_out(cryp, dst, sizeof(u32));
1264 			cryp->total_out -= sizeof(u32);
1265 		} else if (cryp->total_out == tag_size) {
1266 			/* Empty fifo out (data from input padding) */
1267 			d32 = stm32_cryp_read(cryp, CRYP_DOUT);
1268 		} else {
1269 			/* Read less than an u32 */
1270 			d32 = stm32_cryp_read(cryp, CRYP_DOUT);
1271 			d8 = (u8 *)&d32;
1272 
1273 			for (j = 0; j < cryp->total_out - tag_size; j++) {
1274 				*((u8 *)dst) = *(d8++);
1275 				dst = stm32_cryp_next_out(cryp, dst, 1);
1276 			}
1277 			cryp->total_out = tag_size;
1278 		}
1279 	}
1280 
1281 	return !(cryp->total_out - tag_size) || !cryp->total_in;
1282 }
1283 
1284 static void stm32_cryp_irq_write_block(struct stm32_cryp *cryp)
1285 {
1286 	unsigned int i, j;
1287 	u32 *src;
1288 	u8 d8[4];
1289 	size_t tag_size;
1290 
1291 	/* Do no write tag (if any) */
1292 	if (is_decrypt(cryp) && (is_gcm(cryp) || is_ccm(cryp)))
1293 		tag_size = cryp->authsize;
1294 	else
1295 		tag_size = 0;
1296 
1297 	src = sg_virt(cryp->in_sg) + _walked_in;
1298 
1299 	for (i = 0; i < cryp->hw_blocksize / sizeof(u32); i++) {
1300 		if (likely(cryp->total_in - tag_size >= sizeof(u32))) {
1301 			/* Write a full u32 */
1302 			stm32_cryp_write(cryp, CRYP_DIN, *src);
1303 
1304 			src = stm32_cryp_next_in(cryp, src, sizeof(u32));
1305 			cryp->total_in -= sizeof(u32);
1306 		} else if (cryp->total_in == tag_size) {
1307 			/* Write padding data */
1308 			stm32_cryp_write(cryp, CRYP_DIN, 0);
1309 		} else {
1310 			/* Write less than an u32 */
1311 			memset(d8, 0, sizeof(u32));
1312 			for (j = 0; j < cryp->total_in - tag_size; j++) {
1313 				d8[j] = *((u8 *)src);
1314 				src = stm32_cryp_next_in(cryp, src, 1);
1315 			}
1316 
1317 			stm32_cryp_write(cryp, CRYP_DIN, *(u32 *)d8);
1318 			cryp->total_in = tag_size;
1319 		}
1320 	}
1321 }
1322 
1323 static void stm32_cryp_irq_write_gcm_padded_data(struct stm32_cryp *cryp)
1324 {
1325 	int err;
1326 	u32 cfg, tmp[AES_BLOCK_32];
1327 	size_t total_in_ori = cryp->total_in;
1328 	struct scatterlist *out_sg_ori = cryp->out_sg;
1329 	unsigned int i;
1330 
1331 	/* 'Special workaround' procedure described in the datasheet */
1332 
1333 	/* a) disable ip */
1334 	stm32_cryp_write(cryp, CRYP_IMSCR, 0);
1335 	cfg = stm32_cryp_read(cryp, CRYP_CR);
1336 	cfg &= ~CR_CRYPEN;
1337 	stm32_cryp_write(cryp, CRYP_CR, cfg);
1338 
1339 	/* b) Update IV1R */
1340 	stm32_cryp_write(cryp, CRYP_IV1RR, cryp->gcm_ctr - 2);
1341 
1342 	/* c) change mode to CTR */
1343 	cfg &= ~CR_ALGO_MASK;
1344 	cfg |= CR_AES_CTR;
1345 	stm32_cryp_write(cryp, CRYP_CR, cfg);
1346 
1347 	/* a) enable IP */
1348 	cfg |= CR_CRYPEN;
1349 	stm32_cryp_write(cryp, CRYP_CR, cfg);
1350 
1351 	/* b) pad and write the last block */
1352 	stm32_cryp_irq_write_block(cryp);
1353 	cryp->total_in = total_in_ori;
1354 	err = stm32_cryp_wait_output(cryp);
1355 	if (err) {
1356 		dev_err(cryp->dev, "Timeout (write gcm header)\n");
1357 		return stm32_cryp_finish_req(cryp, err);
1358 	}
1359 
1360 	/* c) get and store encrypted data */
1361 	stm32_cryp_irq_read_data(cryp);
1362 	scatterwalk_map_and_copy(tmp, out_sg_ori,
1363 				 cryp->total_in_save - total_in_ori,
1364 				 total_in_ori, 0);
1365 
1366 	/* d) change mode back to AES GCM */
1367 	cfg &= ~CR_ALGO_MASK;
1368 	cfg |= CR_AES_GCM;
1369 	stm32_cryp_write(cryp, CRYP_CR, cfg);
1370 
1371 	/* e) change phase to Final */
1372 	cfg &= ~CR_PH_MASK;
1373 	cfg |= CR_PH_FINAL;
1374 	stm32_cryp_write(cryp, CRYP_CR, cfg);
1375 
1376 	/* f) write padded data */
1377 	for (i = 0; i < AES_BLOCK_32; i++) {
1378 		if (cryp->total_in)
1379 			stm32_cryp_write(cryp, CRYP_DIN, tmp[i]);
1380 		else
1381 			stm32_cryp_write(cryp, CRYP_DIN, 0);
1382 
1383 		cryp->total_in -= min_t(size_t, sizeof(u32), cryp->total_in);
1384 	}
1385 
1386 	/* g) Empty fifo out */
1387 	err = stm32_cryp_wait_output(cryp);
1388 	if (err) {
1389 		dev_err(cryp->dev, "Timeout (write gcm header)\n");
1390 		return stm32_cryp_finish_req(cryp, err);
1391 	}
1392 
1393 	for (i = 0; i < AES_BLOCK_32; i++)
1394 		stm32_cryp_read(cryp, CRYP_DOUT);
1395 
1396 	/* h) run the he normal Final phase */
1397 	stm32_cryp_finish_req(cryp, 0);
1398 }
1399 
1400 static void stm32_cryp_irq_set_npblb(struct stm32_cryp *cryp)
1401 {
1402 	u32 cfg, payload_bytes;
1403 
1404 	/* disable ip, set NPBLB and reneable ip */
1405 	cfg = stm32_cryp_read(cryp, CRYP_CR);
1406 	cfg &= ~CR_CRYPEN;
1407 	stm32_cryp_write(cryp, CRYP_CR, cfg);
1408 
1409 	payload_bytes = is_decrypt(cryp) ? cryp->total_in - cryp->authsize :
1410 					   cryp->total_in;
1411 	cfg |= (cryp->hw_blocksize - payload_bytes) << CR_NBPBL_SHIFT;
1412 	cfg |= CR_CRYPEN;
1413 	stm32_cryp_write(cryp, CRYP_CR, cfg);
1414 }
1415 
1416 static void stm32_cryp_irq_write_ccm_padded_data(struct stm32_cryp *cryp)
1417 {
1418 	int err = 0;
1419 	u32 cfg, iv1tmp;
1420 	u32 cstmp1[AES_BLOCK_32], cstmp2[AES_BLOCK_32], tmp[AES_BLOCK_32];
1421 	size_t last_total_out, total_in_ori = cryp->total_in;
1422 	struct scatterlist *out_sg_ori = cryp->out_sg;
1423 	unsigned int i;
1424 
1425 	/* 'Special workaround' procedure described in the datasheet */
1426 	cryp->flags |= FLG_CCM_PADDED_WA;
1427 
1428 	/* a) disable ip */
1429 	stm32_cryp_write(cryp, CRYP_IMSCR, 0);
1430 
1431 	cfg = stm32_cryp_read(cryp, CRYP_CR);
1432 	cfg &= ~CR_CRYPEN;
1433 	stm32_cryp_write(cryp, CRYP_CR, cfg);
1434 
1435 	/* b) get IV1 from CRYP_CSGCMCCM7 */
1436 	iv1tmp = stm32_cryp_read(cryp, CRYP_CSGCMCCM0R + 7 * 4);
1437 
1438 	/* c) Load CRYP_CSGCMCCMxR */
1439 	for (i = 0; i < ARRAY_SIZE(cstmp1); i++)
1440 		cstmp1[i] = stm32_cryp_read(cryp, CRYP_CSGCMCCM0R + i * 4);
1441 
1442 	/* d) Write IV1R */
1443 	stm32_cryp_write(cryp, CRYP_IV1RR, iv1tmp);
1444 
1445 	/* e) change mode to CTR */
1446 	cfg &= ~CR_ALGO_MASK;
1447 	cfg |= CR_AES_CTR;
1448 	stm32_cryp_write(cryp, CRYP_CR, cfg);
1449 
1450 	/* a) enable IP */
1451 	cfg |= CR_CRYPEN;
1452 	stm32_cryp_write(cryp, CRYP_CR, cfg);
1453 
1454 	/* b) pad and write the last block */
1455 	stm32_cryp_irq_write_block(cryp);
1456 	cryp->total_in = total_in_ori;
1457 	err = stm32_cryp_wait_output(cryp);
1458 	if (err) {
1459 		dev_err(cryp->dev, "Timeout (wite ccm padded data)\n");
1460 		return stm32_cryp_finish_req(cryp, err);
1461 	}
1462 
1463 	/* c) get and store decrypted data */
1464 	last_total_out = cryp->total_out;
1465 	stm32_cryp_irq_read_data(cryp);
1466 
1467 	memset(tmp, 0, sizeof(tmp));
1468 	scatterwalk_map_and_copy(tmp, out_sg_ori,
1469 				 cryp->total_out_save - last_total_out,
1470 				 last_total_out, 0);
1471 
1472 	/* d) Load again CRYP_CSGCMCCMxR */
1473 	for (i = 0; i < ARRAY_SIZE(cstmp2); i++)
1474 		cstmp2[i] = stm32_cryp_read(cryp, CRYP_CSGCMCCM0R + i * 4);
1475 
1476 	/* e) change mode back to AES CCM */
1477 	cfg &= ~CR_ALGO_MASK;
1478 	cfg |= CR_AES_CCM;
1479 	stm32_cryp_write(cryp, CRYP_CR, cfg);
1480 
1481 	/* f) change phase to header */
1482 	cfg &= ~CR_PH_MASK;
1483 	cfg |= CR_PH_HEADER;
1484 	stm32_cryp_write(cryp, CRYP_CR, cfg);
1485 
1486 	/* g) XOR and write padded data */
1487 	for (i = 0; i < ARRAY_SIZE(tmp); i++) {
1488 		tmp[i] ^= cstmp1[i];
1489 		tmp[i] ^= cstmp2[i];
1490 		stm32_cryp_write(cryp, CRYP_DIN, tmp[i]);
1491 	}
1492 
1493 	/* h) wait for completion */
1494 	err = stm32_cryp_wait_busy(cryp);
1495 	if (err)
1496 		dev_err(cryp->dev, "Timeout (wite ccm padded data)\n");
1497 
1498 	/* i) run the he normal Final phase */
1499 	stm32_cryp_finish_req(cryp, err);
1500 }
1501 
1502 static void stm32_cryp_irq_write_data(struct stm32_cryp *cryp)
1503 {
1504 	if (unlikely(!cryp->total_in)) {
1505 		dev_warn(cryp->dev, "No more data to process\n");
1506 		return;
1507 	}
1508 
1509 	if (unlikely(cryp->total_in < AES_BLOCK_SIZE &&
1510 		     (stm32_cryp_get_hw_mode(cryp) == CR_AES_GCM) &&
1511 		     is_encrypt(cryp))) {
1512 		/* Padding for AES GCM encryption */
1513 		if (cryp->caps->padding_wa)
1514 			/* Special case 1 */
1515 			return stm32_cryp_irq_write_gcm_padded_data(cryp);
1516 
1517 		/* Setting padding bytes (NBBLB) */
1518 		stm32_cryp_irq_set_npblb(cryp);
1519 	}
1520 
1521 	if (unlikely((cryp->total_in - cryp->authsize < AES_BLOCK_SIZE) &&
1522 		     (stm32_cryp_get_hw_mode(cryp) == CR_AES_CCM) &&
1523 		     is_decrypt(cryp))) {
1524 		/* Padding for AES CCM decryption */
1525 		if (cryp->caps->padding_wa)
1526 			/* Special case 2 */
1527 			return stm32_cryp_irq_write_ccm_padded_data(cryp);
1528 
1529 		/* Setting padding bytes (NBBLB) */
1530 		stm32_cryp_irq_set_npblb(cryp);
1531 	}
1532 
1533 	if (is_aes(cryp) && is_ctr(cryp))
1534 		stm32_cryp_check_ctr_counter(cryp);
1535 
1536 	stm32_cryp_irq_write_block(cryp);
1537 }
1538 
1539 static void stm32_cryp_irq_write_gcm_header(struct stm32_cryp *cryp)
1540 {
1541 	int err;
1542 	unsigned int i, j;
1543 	u32 cfg, *src;
1544 
1545 	src = sg_virt(cryp->in_sg) + _walked_in;
1546 
1547 	for (i = 0; i < AES_BLOCK_32; i++) {
1548 		stm32_cryp_write(cryp, CRYP_DIN, *src);
1549 
1550 		src = stm32_cryp_next_in(cryp, src, sizeof(u32));
1551 		cryp->total_in -= min_t(size_t, sizeof(u32), cryp->total_in);
1552 
1553 		/* Check if whole header written */
1554 		if ((cryp->total_in_save - cryp->total_in) ==
1555 				cryp->areq->assoclen) {
1556 			/* Write padding if needed */
1557 			for (j = i + 1; j < AES_BLOCK_32; j++)
1558 				stm32_cryp_write(cryp, CRYP_DIN, 0);
1559 
1560 			/* Wait for completion */
1561 			err = stm32_cryp_wait_busy(cryp);
1562 			if (err) {
1563 				dev_err(cryp->dev, "Timeout (gcm header)\n");
1564 				return stm32_cryp_finish_req(cryp, err);
1565 			}
1566 
1567 			if (stm32_cryp_get_input_text_len(cryp)) {
1568 				/* Phase 3 : payload */
1569 				cfg = stm32_cryp_read(cryp, CRYP_CR);
1570 				cfg &= ~CR_CRYPEN;
1571 				stm32_cryp_write(cryp, CRYP_CR, cfg);
1572 
1573 				cfg &= ~CR_PH_MASK;
1574 				cfg |= CR_PH_PAYLOAD;
1575 				cfg |= CR_CRYPEN;
1576 				stm32_cryp_write(cryp, CRYP_CR, cfg);
1577 			} else {
1578 				/* Phase 4 : tag */
1579 				stm32_cryp_write(cryp, CRYP_IMSCR, 0);
1580 				stm32_cryp_finish_req(cryp, 0);
1581 			}
1582 
1583 			break;
1584 		}
1585 
1586 		if (!cryp->total_in)
1587 			break;
1588 	}
1589 }
1590 
1591 static void stm32_cryp_irq_write_ccm_header(struct stm32_cryp *cryp)
1592 {
1593 	int err;
1594 	unsigned int i = 0, j, k;
1595 	u32 alen, cfg, *src;
1596 	u8 d8[4];
1597 
1598 	src = sg_virt(cryp->in_sg) + _walked_in;
1599 	alen = cryp->areq->assoclen;
1600 
1601 	if (!_walked_in) {
1602 		if (cryp->areq->assoclen <= 65280) {
1603 			/* Write first u32 of B1 */
1604 			d8[0] = (alen >> 8) & 0xFF;
1605 			d8[1] = alen & 0xFF;
1606 			d8[2] = *((u8 *)src);
1607 			src = stm32_cryp_next_in(cryp, src, 1);
1608 			d8[3] = *((u8 *)src);
1609 			src = stm32_cryp_next_in(cryp, src, 1);
1610 
1611 			stm32_cryp_write(cryp, CRYP_DIN, *(u32 *)d8);
1612 			i++;
1613 
1614 			cryp->total_in -= min_t(size_t, 2, cryp->total_in);
1615 		} else {
1616 			/* Build the two first u32 of B1 */
1617 			d8[0] = 0xFF;
1618 			d8[1] = 0xFE;
1619 			d8[2] = alen & 0xFF000000;
1620 			d8[3] = alen & 0x00FF0000;
1621 
1622 			stm32_cryp_write(cryp, CRYP_DIN, *(u32 *)d8);
1623 			i++;
1624 
1625 			d8[0] = alen & 0x0000FF00;
1626 			d8[1] = alen & 0x000000FF;
1627 			d8[2] = *((u8 *)src);
1628 			src = stm32_cryp_next_in(cryp, src, 1);
1629 			d8[3] = *((u8 *)src);
1630 			src = stm32_cryp_next_in(cryp, src, 1);
1631 
1632 			stm32_cryp_write(cryp, CRYP_DIN, *(u32 *)d8);
1633 			i++;
1634 
1635 			cryp->total_in -= min_t(size_t, 2, cryp->total_in);
1636 		}
1637 	}
1638 
1639 	/* Write next u32 */
1640 	for (; i < AES_BLOCK_32; i++) {
1641 		/* Build an u32 */
1642 		memset(d8, 0, sizeof(u32));
1643 		for (k = 0; k < sizeof(u32); k++) {
1644 			d8[k] = *((u8 *)src);
1645 			src = stm32_cryp_next_in(cryp, src, 1);
1646 
1647 			cryp->total_in -= min_t(size_t, 1, cryp->total_in);
1648 			if ((cryp->total_in_save - cryp->total_in) == alen)
1649 				break;
1650 		}
1651 
1652 		stm32_cryp_write(cryp, CRYP_DIN, *(u32 *)d8);
1653 
1654 		if ((cryp->total_in_save - cryp->total_in) == alen) {
1655 			/* Write padding if needed */
1656 			for (j = i + 1; j < AES_BLOCK_32; j++)
1657 				stm32_cryp_write(cryp, CRYP_DIN, 0);
1658 
1659 			/* Wait for completion */
1660 			err = stm32_cryp_wait_busy(cryp);
1661 			if (err) {
1662 				dev_err(cryp->dev, "Timeout (ccm header)\n");
1663 				return stm32_cryp_finish_req(cryp, err);
1664 			}
1665 
1666 			if (stm32_cryp_get_input_text_len(cryp)) {
1667 				/* Phase 3 : payload */
1668 				cfg = stm32_cryp_read(cryp, CRYP_CR);
1669 				cfg &= ~CR_CRYPEN;
1670 				stm32_cryp_write(cryp, CRYP_CR, cfg);
1671 
1672 				cfg &= ~CR_PH_MASK;
1673 				cfg |= CR_PH_PAYLOAD;
1674 				cfg |= CR_CRYPEN;
1675 				stm32_cryp_write(cryp, CRYP_CR, cfg);
1676 			} else {
1677 				/* Phase 4 : tag */
1678 				stm32_cryp_write(cryp, CRYP_IMSCR, 0);
1679 				stm32_cryp_finish_req(cryp, 0);
1680 			}
1681 
1682 			break;
1683 		}
1684 	}
1685 }
1686 
1687 static irqreturn_t stm32_cryp_irq_thread(int irq, void *arg)
1688 {
1689 	struct stm32_cryp *cryp = arg;
1690 	u32 ph;
1691 
1692 	if (cryp->irq_status & MISR_OUT)
1693 		/* Output FIFO IRQ: read data */
1694 		if (unlikely(stm32_cryp_irq_read_data(cryp))) {
1695 			/* All bytes processed, finish */
1696 			stm32_cryp_write(cryp, CRYP_IMSCR, 0);
1697 			stm32_cryp_finish_req(cryp, 0);
1698 			return IRQ_HANDLED;
1699 		}
1700 
1701 	if (cryp->irq_status & MISR_IN) {
1702 		if (is_gcm(cryp)) {
1703 			ph = stm32_cryp_read(cryp, CRYP_CR) & CR_PH_MASK;
1704 			if (unlikely(ph == CR_PH_HEADER))
1705 				/* Write Header */
1706 				stm32_cryp_irq_write_gcm_header(cryp);
1707 			else
1708 				/* Input FIFO IRQ: write data */
1709 				stm32_cryp_irq_write_data(cryp);
1710 			cryp->gcm_ctr++;
1711 		} else if (is_ccm(cryp)) {
1712 			ph = stm32_cryp_read(cryp, CRYP_CR) & CR_PH_MASK;
1713 			if (unlikely(ph == CR_PH_HEADER))
1714 				/* Write Header */
1715 				stm32_cryp_irq_write_ccm_header(cryp);
1716 			else
1717 				/* Input FIFO IRQ: write data */
1718 				stm32_cryp_irq_write_data(cryp);
1719 		} else {
1720 			/* Input FIFO IRQ: write data */
1721 			stm32_cryp_irq_write_data(cryp);
1722 		}
1723 	}
1724 
1725 	return IRQ_HANDLED;
1726 }
1727 
1728 static irqreturn_t stm32_cryp_irq(int irq, void *arg)
1729 {
1730 	struct stm32_cryp *cryp = arg;
1731 
1732 	cryp->irq_status = stm32_cryp_read(cryp, CRYP_MISR);
1733 
1734 	return IRQ_WAKE_THREAD;
1735 }
1736 
1737 static struct skcipher_alg crypto_algs[] = {
1738 {
1739 	.base.cra_name		= "ecb(aes)",
1740 	.base.cra_driver_name	= "stm32-ecb-aes",
1741 	.base.cra_priority	= 200,
1742 	.base.cra_flags		= CRYPTO_ALG_ASYNC,
1743 	.base.cra_blocksize	= AES_BLOCK_SIZE,
1744 	.base.cra_ctxsize	= sizeof(struct stm32_cryp_ctx),
1745 	.base.cra_alignmask	= 0xf,
1746 	.base.cra_module	= THIS_MODULE,
1747 
1748 	.init			= stm32_cryp_init_tfm,
1749 	.min_keysize		= AES_MIN_KEY_SIZE,
1750 	.max_keysize		= AES_MAX_KEY_SIZE,
1751 	.setkey			= stm32_cryp_aes_setkey,
1752 	.encrypt		= stm32_cryp_aes_ecb_encrypt,
1753 	.decrypt		= stm32_cryp_aes_ecb_decrypt,
1754 },
1755 {
1756 	.base.cra_name		= "cbc(aes)",
1757 	.base.cra_driver_name	= "stm32-cbc-aes",
1758 	.base.cra_priority	= 200,
1759 	.base.cra_flags		= CRYPTO_ALG_ASYNC,
1760 	.base.cra_blocksize	= AES_BLOCK_SIZE,
1761 	.base.cra_ctxsize	= sizeof(struct stm32_cryp_ctx),
1762 	.base.cra_alignmask	= 0xf,
1763 	.base.cra_module	= THIS_MODULE,
1764 
1765 	.init			= stm32_cryp_init_tfm,
1766 	.min_keysize		= AES_MIN_KEY_SIZE,
1767 	.max_keysize		= AES_MAX_KEY_SIZE,
1768 	.ivsize			= AES_BLOCK_SIZE,
1769 	.setkey			= stm32_cryp_aes_setkey,
1770 	.encrypt		= stm32_cryp_aes_cbc_encrypt,
1771 	.decrypt		= stm32_cryp_aes_cbc_decrypt,
1772 },
1773 {
1774 	.base.cra_name		= "ctr(aes)",
1775 	.base.cra_driver_name	= "stm32-ctr-aes",
1776 	.base.cra_priority	= 200,
1777 	.base.cra_flags		= CRYPTO_ALG_ASYNC,
1778 	.base.cra_blocksize	= 1,
1779 	.base.cra_ctxsize	= sizeof(struct stm32_cryp_ctx),
1780 	.base.cra_alignmask	= 0xf,
1781 	.base.cra_module	= THIS_MODULE,
1782 
1783 	.init			= stm32_cryp_init_tfm,
1784 	.min_keysize		= AES_MIN_KEY_SIZE,
1785 	.max_keysize		= AES_MAX_KEY_SIZE,
1786 	.ivsize			= AES_BLOCK_SIZE,
1787 	.setkey			= stm32_cryp_aes_setkey,
1788 	.encrypt		= stm32_cryp_aes_ctr_encrypt,
1789 	.decrypt		= stm32_cryp_aes_ctr_decrypt,
1790 },
1791 {
1792 	.base.cra_name		= "ecb(des)",
1793 	.base.cra_driver_name	= "stm32-ecb-des",
1794 	.base.cra_priority	= 200,
1795 	.base.cra_flags		= CRYPTO_ALG_ASYNC,
1796 	.base.cra_blocksize	= DES_BLOCK_SIZE,
1797 	.base.cra_ctxsize	= sizeof(struct stm32_cryp_ctx),
1798 	.base.cra_alignmask	= 0xf,
1799 	.base.cra_module	= THIS_MODULE,
1800 
1801 	.init			= stm32_cryp_init_tfm,
1802 	.min_keysize		= DES_BLOCK_SIZE,
1803 	.max_keysize		= DES_BLOCK_SIZE,
1804 	.setkey			= stm32_cryp_des_setkey,
1805 	.encrypt		= stm32_cryp_des_ecb_encrypt,
1806 	.decrypt		= stm32_cryp_des_ecb_decrypt,
1807 },
1808 {
1809 	.base.cra_name		= "cbc(des)",
1810 	.base.cra_driver_name	= "stm32-cbc-des",
1811 	.base.cra_priority	= 200,
1812 	.base.cra_flags		= CRYPTO_ALG_ASYNC,
1813 	.base.cra_blocksize	= DES_BLOCK_SIZE,
1814 	.base.cra_ctxsize	= sizeof(struct stm32_cryp_ctx),
1815 	.base.cra_alignmask	= 0xf,
1816 	.base.cra_module	= THIS_MODULE,
1817 
1818 	.init			= stm32_cryp_init_tfm,
1819 	.min_keysize		= DES_BLOCK_SIZE,
1820 	.max_keysize		= DES_BLOCK_SIZE,
1821 	.ivsize			= DES_BLOCK_SIZE,
1822 	.setkey			= stm32_cryp_des_setkey,
1823 	.encrypt		= stm32_cryp_des_cbc_encrypt,
1824 	.decrypt		= stm32_cryp_des_cbc_decrypt,
1825 },
1826 {
1827 	.base.cra_name		= "ecb(des3_ede)",
1828 	.base.cra_driver_name	= "stm32-ecb-des3",
1829 	.base.cra_priority	= 200,
1830 	.base.cra_flags		= CRYPTO_ALG_ASYNC,
1831 	.base.cra_blocksize	= DES_BLOCK_SIZE,
1832 	.base.cra_ctxsize	= sizeof(struct stm32_cryp_ctx),
1833 	.base.cra_alignmask	= 0xf,
1834 	.base.cra_module	= THIS_MODULE,
1835 
1836 	.init			= stm32_cryp_init_tfm,
1837 	.min_keysize		= 3 * DES_BLOCK_SIZE,
1838 	.max_keysize		= 3 * DES_BLOCK_SIZE,
1839 	.setkey			= stm32_cryp_tdes_setkey,
1840 	.encrypt		= stm32_cryp_tdes_ecb_encrypt,
1841 	.decrypt		= stm32_cryp_tdes_ecb_decrypt,
1842 },
1843 {
1844 	.base.cra_name		= "cbc(des3_ede)",
1845 	.base.cra_driver_name	= "stm32-cbc-des3",
1846 	.base.cra_priority	= 200,
1847 	.base.cra_flags		= CRYPTO_ALG_ASYNC,
1848 	.base.cra_blocksize	= DES_BLOCK_SIZE,
1849 	.base.cra_ctxsize	= sizeof(struct stm32_cryp_ctx),
1850 	.base.cra_alignmask	= 0xf,
1851 	.base.cra_module	= THIS_MODULE,
1852 
1853 	.init			= stm32_cryp_init_tfm,
1854 	.min_keysize		= 3 * DES_BLOCK_SIZE,
1855 	.max_keysize		= 3 * DES_BLOCK_SIZE,
1856 	.ivsize			= DES_BLOCK_SIZE,
1857 	.setkey			= stm32_cryp_tdes_setkey,
1858 	.encrypt		= stm32_cryp_tdes_cbc_encrypt,
1859 	.decrypt		= stm32_cryp_tdes_cbc_decrypt,
1860 },
1861 };
1862 
1863 static struct aead_alg aead_algs[] = {
1864 {
1865 	.setkey		= stm32_cryp_aes_aead_setkey,
1866 	.setauthsize	= stm32_cryp_aes_gcm_setauthsize,
1867 	.encrypt	= stm32_cryp_aes_gcm_encrypt,
1868 	.decrypt	= stm32_cryp_aes_gcm_decrypt,
1869 	.init		= stm32_cryp_aes_aead_init,
1870 	.ivsize		= 12,
1871 	.maxauthsize	= AES_BLOCK_SIZE,
1872 
1873 	.base = {
1874 		.cra_name		= "gcm(aes)",
1875 		.cra_driver_name	= "stm32-gcm-aes",
1876 		.cra_priority		= 200,
1877 		.cra_flags		= CRYPTO_ALG_ASYNC,
1878 		.cra_blocksize		= 1,
1879 		.cra_ctxsize		= sizeof(struct stm32_cryp_ctx),
1880 		.cra_alignmask		= 0xf,
1881 		.cra_module		= THIS_MODULE,
1882 	},
1883 },
1884 {
1885 	.setkey		= stm32_cryp_aes_aead_setkey,
1886 	.setauthsize	= stm32_cryp_aes_ccm_setauthsize,
1887 	.encrypt	= stm32_cryp_aes_ccm_encrypt,
1888 	.decrypt	= stm32_cryp_aes_ccm_decrypt,
1889 	.init		= stm32_cryp_aes_aead_init,
1890 	.ivsize		= AES_BLOCK_SIZE,
1891 	.maxauthsize	= AES_BLOCK_SIZE,
1892 
1893 	.base = {
1894 		.cra_name		= "ccm(aes)",
1895 		.cra_driver_name	= "stm32-ccm-aes",
1896 		.cra_priority		= 200,
1897 		.cra_flags		= CRYPTO_ALG_ASYNC,
1898 		.cra_blocksize		= 1,
1899 		.cra_ctxsize		= sizeof(struct stm32_cryp_ctx),
1900 		.cra_alignmask		= 0xf,
1901 		.cra_module		= THIS_MODULE,
1902 	},
1903 },
1904 };
1905 
1906 static const struct stm32_cryp_caps f7_data = {
1907 	.swap_final = true,
1908 	.padding_wa = true,
1909 };
1910 
1911 static const struct stm32_cryp_caps mp1_data = {
1912 	.swap_final = false,
1913 	.padding_wa = false,
1914 };
1915 
1916 static const struct of_device_id stm32_dt_ids[] = {
1917 	{ .compatible = "st,stm32f756-cryp", .data = &f7_data},
1918 	{ .compatible = "st,stm32mp1-cryp", .data = &mp1_data},
1919 	{},
1920 };
1921 MODULE_DEVICE_TABLE(of, stm32_dt_ids);
1922 
1923 static int stm32_cryp_probe(struct platform_device *pdev)
1924 {
1925 	struct device *dev = &pdev->dev;
1926 	struct stm32_cryp *cryp;
1927 	struct reset_control *rst;
1928 	int irq, ret;
1929 
1930 	cryp = devm_kzalloc(dev, sizeof(*cryp), GFP_KERNEL);
1931 	if (!cryp)
1932 		return -ENOMEM;
1933 
1934 	cryp->caps = of_device_get_match_data(dev);
1935 	if (!cryp->caps)
1936 		return -ENODEV;
1937 
1938 	cryp->dev = dev;
1939 
1940 	cryp->regs = devm_platform_ioremap_resource(pdev, 0);
1941 	if (IS_ERR(cryp->regs))
1942 		return PTR_ERR(cryp->regs);
1943 
1944 	irq = platform_get_irq(pdev, 0);
1945 	if (irq < 0)
1946 		return irq;
1947 
1948 	ret = devm_request_threaded_irq(dev, irq, stm32_cryp_irq,
1949 					stm32_cryp_irq_thread, IRQF_ONESHOT,
1950 					dev_name(dev), cryp);
1951 	if (ret) {
1952 		dev_err(dev, "Cannot grab IRQ\n");
1953 		return ret;
1954 	}
1955 
1956 	cryp->clk = devm_clk_get(dev, NULL);
1957 	if (IS_ERR(cryp->clk)) {
1958 		dev_err(dev, "Could not get clock\n");
1959 		return PTR_ERR(cryp->clk);
1960 	}
1961 
1962 	ret = clk_prepare_enable(cryp->clk);
1963 	if (ret) {
1964 		dev_err(cryp->dev, "Failed to enable clock\n");
1965 		return ret;
1966 	}
1967 
1968 	pm_runtime_set_autosuspend_delay(dev, CRYP_AUTOSUSPEND_DELAY);
1969 	pm_runtime_use_autosuspend(dev);
1970 
1971 	pm_runtime_get_noresume(dev);
1972 	pm_runtime_set_active(dev);
1973 	pm_runtime_enable(dev);
1974 
1975 	rst = devm_reset_control_get(dev, NULL);
1976 	if (!IS_ERR(rst)) {
1977 		reset_control_assert(rst);
1978 		udelay(2);
1979 		reset_control_deassert(rst);
1980 	}
1981 
1982 	platform_set_drvdata(pdev, cryp);
1983 
1984 	spin_lock(&cryp_list.lock);
1985 	list_add(&cryp->list, &cryp_list.dev_list);
1986 	spin_unlock(&cryp_list.lock);
1987 
1988 	/* Initialize crypto engine */
1989 	cryp->engine = crypto_engine_alloc_init(dev, 1);
1990 	if (!cryp->engine) {
1991 		dev_err(dev, "Could not init crypto engine\n");
1992 		ret = -ENOMEM;
1993 		goto err_engine1;
1994 	}
1995 
1996 	ret = crypto_engine_start(cryp->engine);
1997 	if (ret) {
1998 		dev_err(dev, "Could not start crypto engine\n");
1999 		goto err_engine2;
2000 	}
2001 
2002 	ret = crypto_register_skciphers(crypto_algs, ARRAY_SIZE(crypto_algs));
2003 	if (ret) {
2004 		dev_err(dev, "Could not register algs\n");
2005 		goto err_algs;
2006 	}
2007 
2008 	ret = crypto_register_aeads(aead_algs, ARRAY_SIZE(aead_algs));
2009 	if (ret)
2010 		goto err_aead_algs;
2011 
2012 	dev_info(dev, "Initialized\n");
2013 
2014 	pm_runtime_put_sync(dev);
2015 
2016 	return 0;
2017 
2018 err_aead_algs:
2019 	crypto_unregister_skciphers(crypto_algs, ARRAY_SIZE(crypto_algs));
2020 err_algs:
2021 err_engine2:
2022 	crypto_engine_exit(cryp->engine);
2023 err_engine1:
2024 	spin_lock(&cryp_list.lock);
2025 	list_del(&cryp->list);
2026 	spin_unlock(&cryp_list.lock);
2027 
2028 	pm_runtime_disable(dev);
2029 	pm_runtime_put_noidle(dev);
2030 	pm_runtime_disable(dev);
2031 	pm_runtime_put_noidle(dev);
2032 
2033 	clk_disable_unprepare(cryp->clk);
2034 
2035 	return ret;
2036 }
2037 
2038 static int stm32_cryp_remove(struct platform_device *pdev)
2039 {
2040 	struct stm32_cryp *cryp = platform_get_drvdata(pdev);
2041 	int ret;
2042 
2043 	if (!cryp)
2044 		return -ENODEV;
2045 
2046 	ret = pm_runtime_get_sync(cryp->dev);
2047 	if (ret < 0)
2048 		return ret;
2049 
2050 	crypto_unregister_aeads(aead_algs, ARRAY_SIZE(aead_algs));
2051 	crypto_unregister_skciphers(crypto_algs, ARRAY_SIZE(crypto_algs));
2052 
2053 	crypto_engine_exit(cryp->engine);
2054 
2055 	spin_lock(&cryp_list.lock);
2056 	list_del(&cryp->list);
2057 	spin_unlock(&cryp_list.lock);
2058 
2059 	pm_runtime_disable(cryp->dev);
2060 	pm_runtime_put_noidle(cryp->dev);
2061 
2062 	clk_disable_unprepare(cryp->clk);
2063 
2064 	return 0;
2065 }
2066 
2067 #ifdef CONFIG_PM
2068 static int stm32_cryp_runtime_suspend(struct device *dev)
2069 {
2070 	struct stm32_cryp *cryp = dev_get_drvdata(dev);
2071 
2072 	clk_disable_unprepare(cryp->clk);
2073 
2074 	return 0;
2075 }
2076 
2077 static int stm32_cryp_runtime_resume(struct device *dev)
2078 {
2079 	struct stm32_cryp *cryp = dev_get_drvdata(dev);
2080 	int ret;
2081 
2082 	ret = clk_prepare_enable(cryp->clk);
2083 	if (ret) {
2084 		dev_err(cryp->dev, "Failed to prepare_enable clock\n");
2085 		return ret;
2086 	}
2087 
2088 	return 0;
2089 }
2090 #endif
2091 
2092 static const struct dev_pm_ops stm32_cryp_pm_ops = {
2093 	SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend,
2094 				pm_runtime_force_resume)
2095 	SET_RUNTIME_PM_OPS(stm32_cryp_runtime_suspend,
2096 			   stm32_cryp_runtime_resume, NULL)
2097 };
2098 
2099 static struct platform_driver stm32_cryp_driver = {
2100 	.probe  = stm32_cryp_probe,
2101 	.remove = stm32_cryp_remove,
2102 	.driver = {
2103 		.name           = DRIVER_NAME,
2104 		.pm		= &stm32_cryp_pm_ops,
2105 		.of_match_table = stm32_dt_ids,
2106 	},
2107 };
2108 
2109 module_platform_driver(stm32_cryp_driver);
2110 
2111 MODULE_AUTHOR("Fabien Dessenne <fabien.dessenne@st.com>");
2112 MODULE_DESCRIPTION("STMicrolectronics STM32 CRYP hardware driver");
2113 MODULE_LICENSE("GPL");
2114