1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * sun8i-ce-cipher.c - hardware cryptographic offloader for
4  * Allwinner H3/A64/H5/H2+/H6/R40 SoC
5  *
6  * Copyright (C) 2016-2019 Corentin LABBE <clabbe.montjoie@gmail.com>
7  *
8  * This file add support for AES cipher with 128,192,256 bits keysize in
9  * CBC and ECB mode.
10  *
11  * You could find a link for the datasheet in Documentation/arch/arm/sunxi.rst
12  */
13 
14 #include <linux/bottom_half.h>
15 #include <linux/crypto.h>
16 #include <linux/dma-mapping.h>
17 #include <linux/io.h>
18 #include <linux/pm_runtime.h>
19 #include <crypto/scatterwalk.h>
20 #include <crypto/internal/des.h>
21 #include <crypto/internal/skcipher.h>
22 #include "sun8i-ce.h"
23 
24 static int sun8i_ce_cipher_need_fallback(struct skcipher_request *areq)
25 {
26 	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq);
27 	struct scatterlist *sg;
28 	struct skcipher_alg *alg = crypto_skcipher_alg(tfm);
29 	struct sun8i_ce_alg_template *algt;
30 	unsigned int todo, len;
31 
32 	algt = container_of(alg, struct sun8i_ce_alg_template, alg.skcipher.base);
33 
34 	if (sg_nents_for_len(areq->src, areq->cryptlen) > MAX_SG ||
35 	    sg_nents_for_len(areq->dst, areq->cryptlen) > MAX_SG) {
36 		algt->stat_fb_maxsg++;
37 		return true;
38 	}
39 
40 	if (areq->cryptlen < crypto_skcipher_ivsize(tfm)) {
41 		algt->stat_fb_leniv++;
42 		return true;
43 	}
44 
45 	if (areq->cryptlen == 0) {
46 		algt->stat_fb_len0++;
47 		return true;
48 	}
49 
50 	if (areq->cryptlen % 16) {
51 		algt->stat_fb_mod16++;
52 		return true;
53 	}
54 
55 	len = areq->cryptlen;
56 	sg = areq->src;
57 	while (sg) {
58 		if (!IS_ALIGNED(sg->offset, sizeof(u32))) {
59 			algt->stat_fb_srcali++;
60 			return true;
61 		}
62 		todo = min(len, sg->length);
63 		if (todo % 4) {
64 			algt->stat_fb_srclen++;
65 			return true;
66 		}
67 		len -= todo;
68 		sg = sg_next(sg);
69 	}
70 
71 	len = areq->cryptlen;
72 	sg = areq->dst;
73 	while (sg) {
74 		if (!IS_ALIGNED(sg->offset, sizeof(u32))) {
75 			algt->stat_fb_dstali++;
76 			return true;
77 		}
78 		todo = min(len, sg->length);
79 		if (todo % 4) {
80 			algt->stat_fb_dstlen++;
81 			return true;
82 		}
83 		len -= todo;
84 		sg = sg_next(sg);
85 	}
86 	return false;
87 }
88 
89 static int sun8i_ce_cipher_fallback(struct skcipher_request *areq)
90 {
91 	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq);
92 	struct sun8i_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm);
93 	struct sun8i_cipher_req_ctx *rctx = skcipher_request_ctx(areq);
94 	int err;
95 
96 	if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG)) {
97 		struct skcipher_alg *alg = crypto_skcipher_alg(tfm);
98 		struct sun8i_ce_alg_template *algt __maybe_unused;
99 
100 		algt = container_of(alg, struct sun8i_ce_alg_template,
101 				    alg.skcipher.base);
102 
103 #ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG
104 		algt->stat_fb++;
105 #endif
106 	}
107 
108 	skcipher_request_set_tfm(&rctx->fallback_req, op->fallback_tfm);
109 	skcipher_request_set_callback(&rctx->fallback_req, areq->base.flags,
110 				      areq->base.complete, areq->base.data);
111 	skcipher_request_set_crypt(&rctx->fallback_req, areq->src, areq->dst,
112 				   areq->cryptlen, areq->iv);
113 	if (rctx->op_dir & CE_DECRYPTION)
114 		err = crypto_skcipher_decrypt(&rctx->fallback_req);
115 	else
116 		err = crypto_skcipher_encrypt(&rctx->fallback_req);
117 	return err;
118 }
119 
120 static int sun8i_ce_cipher_prepare(struct crypto_engine *engine, void *async_req)
121 {
122 	struct skcipher_request *areq = container_of(async_req, struct skcipher_request, base);
123 	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq);
124 	struct sun8i_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm);
125 	struct sun8i_ce_dev *ce = op->ce;
126 	struct sun8i_cipher_req_ctx *rctx = skcipher_request_ctx(areq);
127 	struct skcipher_alg *alg = crypto_skcipher_alg(tfm);
128 	struct sun8i_ce_alg_template *algt;
129 	struct sun8i_ce_flow *chan;
130 	struct ce_task *cet;
131 	struct scatterlist *sg;
132 	unsigned int todo, len, offset, ivsize;
133 	u32 common, sym;
134 	int flow, i;
135 	int nr_sgs = 0;
136 	int nr_sgd = 0;
137 	int err = 0;
138 	int ns = sg_nents_for_len(areq->src, areq->cryptlen);
139 	int nd = sg_nents_for_len(areq->dst, areq->cryptlen);
140 
141 	algt = container_of(alg, struct sun8i_ce_alg_template, alg.skcipher.base);
142 
143 	dev_dbg(ce->dev, "%s %s %u %x IV(%p %u) key=%u\n", __func__,
144 		crypto_tfm_alg_name(areq->base.tfm),
145 		areq->cryptlen,
146 		rctx->op_dir, areq->iv, crypto_skcipher_ivsize(tfm),
147 		op->keylen);
148 
149 #ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG
150 	algt->stat_req++;
151 #endif
152 
153 	flow = rctx->flow;
154 
155 	chan = &ce->chanlist[flow];
156 
157 	cet = chan->tl;
158 	memset(cet, 0, sizeof(struct ce_task));
159 
160 	cet->t_id = cpu_to_le32(flow);
161 	common = ce->variant->alg_cipher[algt->ce_algo_id];
162 	common |= rctx->op_dir | CE_COMM_INT;
163 	cet->t_common_ctl = cpu_to_le32(common);
164 	/* CTS and recent CE (H6) need length in bytes, in word otherwise */
165 	if (ce->variant->cipher_t_dlen_in_bytes)
166 		cet->t_dlen = cpu_to_le32(areq->cryptlen);
167 	else
168 		cet->t_dlen = cpu_to_le32(areq->cryptlen / 4);
169 
170 	sym = ce->variant->op_mode[algt->ce_blockmode];
171 	len = op->keylen;
172 	switch (len) {
173 	case 128 / 8:
174 		sym |= CE_AES_128BITS;
175 		break;
176 	case 192 / 8:
177 		sym |= CE_AES_192BITS;
178 		break;
179 	case 256 / 8:
180 		sym |= CE_AES_256BITS;
181 		break;
182 	}
183 
184 	cet->t_sym_ctl = cpu_to_le32(sym);
185 	cet->t_asym_ctl = 0;
186 
187 	rctx->addr_key = dma_map_single(ce->dev, op->key, op->keylen, DMA_TO_DEVICE);
188 	if (dma_mapping_error(ce->dev, rctx->addr_key)) {
189 		dev_err(ce->dev, "Cannot DMA MAP KEY\n");
190 		err = -EFAULT;
191 		goto theend;
192 	}
193 	cet->t_key = cpu_to_le32(rctx->addr_key);
194 
195 	ivsize = crypto_skcipher_ivsize(tfm);
196 	if (areq->iv && crypto_skcipher_ivsize(tfm) > 0) {
197 		rctx->ivlen = ivsize;
198 		if (rctx->op_dir & CE_DECRYPTION) {
199 			offset = areq->cryptlen - ivsize;
200 			scatterwalk_map_and_copy(chan->backup_iv, areq->src,
201 						 offset, ivsize, 0);
202 		}
203 		memcpy(chan->bounce_iv, areq->iv, ivsize);
204 		rctx->addr_iv = dma_map_single(ce->dev, chan->bounce_iv, rctx->ivlen,
205 					       DMA_TO_DEVICE);
206 		if (dma_mapping_error(ce->dev, rctx->addr_iv)) {
207 			dev_err(ce->dev, "Cannot DMA MAP IV\n");
208 			err = -ENOMEM;
209 			goto theend_iv;
210 		}
211 		cet->t_iv = cpu_to_le32(rctx->addr_iv);
212 	}
213 
214 	if (areq->src == areq->dst) {
215 		nr_sgs = dma_map_sg(ce->dev, areq->src, ns, DMA_BIDIRECTIONAL);
216 		if (nr_sgs <= 0 || nr_sgs > MAX_SG) {
217 			dev_err(ce->dev, "Invalid sg number %d\n", nr_sgs);
218 			err = -EINVAL;
219 			goto theend_iv;
220 		}
221 		nr_sgd = nr_sgs;
222 	} else {
223 		nr_sgs = dma_map_sg(ce->dev, areq->src, ns, DMA_TO_DEVICE);
224 		if (nr_sgs <= 0 || nr_sgs > MAX_SG) {
225 			dev_err(ce->dev, "Invalid sg number %d\n", nr_sgs);
226 			err = -EINVAL;
227 			goto theend_iv;
228 		}
229 		nr_sgd = dma_map_sg(ce->dev, areq->dst, nd, DMA_FROM_DEVICE);
230 		if (nr_sgd <= 0 || nr_sgd > MAX_SG) {
231 			dev_err(ce->dev, "Invalid sg number %d\n", nr_sgd);
232 			err = -EINVAL;
233 			goto theend_sgs;
234 		}
235 	}
236 
237 	len = areq->cryptlen;
238 	for_each_sg(areq->src, sg, nr_sgs, i) {
239 		cet->t_src[i].addr = cpu_to_le32(sg_dma_address(sg));
240 		todo = min(len, sg_dma_len(sg));
241 		cet->t_src[i].len = cpu_to_le32(todo / 4);
242 		dev_dbg(ce->dev, "%s total=%u SG(%d %u off=%d) todo=%u\n", __func__,
243 			areq->cryptlen, i, cet->t_src[i].len, sg->offset, todo);
244 		len -= todo;
245 	}
246 	if (len > 0) {
247 		dev_err(ce->dev, "remaining len %d\n", len);
248 		err = -EINVAL;
249 		goto theend_sgs;
250 	}
251 
252 	len = areq->cryptlen;
253 	for_each_sg(areq->dst, sg, nr_sgd, i) {
254 		cet->t_dst[i].addr = cpu_to_le32(sg_dma_address(sg));
255 		todo = min(len, sg_dma_len(sg));
256 		cet->t_dst[i].len = cpu_to_le32(todo / 4);
257 		dev_dbg(ce->dev, "%s total=%u SG(%d %u off=%d) todo=%u\n", __func__,
258 			areq->cryptlen, i, cet->t_dst[i].len, sg->offset, todo);
259 		len -= todo;
260 	}
261 	if (len > 0) {
262 		dev_err(ce->dev, "remaining len %d\n", len);
263 		err = -EINVAL;
264 		goto theend_sgs;
265 	}
266 
267 	chan->timeout = areq->cryptlen;
268 	rctx->nr_sgs = nr_sgs;
269 	rctx->nr_sgd = nr_sgd;
270 	return 0;
271 
272 theend_sgs:
273 	if (areq->src == areq->dst) {
274 		dma_unmap_sg(ce->dev, areq->src, ns, DMA_BIDIRECTIONAL);
275 	} else {
276 		if (nr_sgs > 0)
277 			dma_unmap_sg(ce->dev, areq->src, ns, DMA_TO_DEVICE);
278 		dma_unmap_sg(ce->dev, areq->dst, nd, DMA_FROM_DEVICE);
279 	}
280 
281 theend_iv:
282 	if (areq->iv && ivsize > 0) {
283 		if (rctx->addr_iv)
284 			dma_unmap_single(ce->dev, rctx->addr_iv, rctx->ivlen, DMA_TO_DEVICE);
285 		offset = areq->cryptlen - ivsize;
286 		if (rctx->op_dir & CE_DECRYPTION) {
287 			memcpy(areq->iv, chan->backup_iv, ivsize);
288 			memzero_explicit(chan->backup_iv, ivsize);
289 		} else {
290 			scatterwalk_map_and_copy(areq->iv, areq->dst, offset,
291 						 ivsize, 0);
292 		}
293 		memzero_explicit(chan->bounce_iv, ivsize);
294 	}
295 
296 	dma_unmap_single(ce->dev, rctx->addr_key, op->keylen, DMA_TO_DEVICE);
297 
298 theend:
299 	return err;
300 }
301 
302 static void sun8i_ce_cipher_run(struct crypto_engine *engine, void *areq)
303 {
304 	struct skcipher_request *breq = container_of(areq, struct skcipher_request, base);
305 	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(breq);
306 	struct sun8i_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm);
307 	struct sun8i_ce_dev *ce = op->ce;
308 	struct sun8i_cipher_req_ctx *rctx = skcipher_request_ctx(breq);
309 	int flow, err;
310 
311 	flow = rctx->flow;
312 	err = sun8i_ce_run_task(ce, flow, crypto_tfm_alg_name(breq->base.tfm));
313 	local_bh_disable();
314 	crypto_finalize_skcipher_request(engine, breq, err);
315 	local_bh_enable();
316 }
317 
318 static void sun8i_ce_cipher_unprepare(struct crypto_engine *engine,
319 				      void *async_req)
320 {
321 	struct skcipher_request *areq = container_of(async_req, struct skcipher_request, base);
322 	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq);
323 	struct sun8i_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm);
324 	struct sun8i_ce_dev *ce = op->ce;
325 	struct sun8i_cipher_req_ctx *rctx = skcipher_request_ctx(areq);
326 	struct sun8i_ce_flow *chan;
327 	struct ce_task *cet;
328 	unsigned int ivsize, offset;
329 	int nr_sgs = rctx->nr_sgs;
330 	int nr_sgd = rctx->nr_sgd;
331 	int flow;
332 
333 	flow = rctx->flow;
334 	chan = &ce->chanlist[flow];
335 	cet = chan->tl;
336 	ivsize = crypto_skcipher_ivsize(tfm);
337 
338 	if (areq->src == areq->dst) {
339 		dma_unmap_sg(ce->dev, areq->src, nr_sgs, DMA_BIDIRECTIONAL);
340 	} else {
341 		if (nr_sgs > 0)
342 			dma_unmap_sg(ce->dev, areq->src, nr_sgs, DMA_TO_DEVICE);
343 		dma_unmap_sg(ce->dev, areq->dst, nr_sgd, DMA_FROM_DEVICE);
344 	}
345 
346 	if (areq->iv && ivsize > 0) {
347 		if (cet->t_iv)
348 			dma_unmap_single(ce->dev, rctx->addr_iv, rctx->ivlen, DMA_TO_DEVICE);
349 		offset = areq->cryptlen - ivsize;
350 		if (rctx->op_dir & CE_DECRYPTION) {
351 			memcpy(areq->iv, chan->backup_iv, ivsize);
352 			memzero_explicit(chan->backup_iv, ivsize);
353 		} else {
354 			scatterwalk_map_and_copy(areq->iv, areq->dst, offset,
355 						 ivsize, 0);
356 		}
357 		memzero_explicit(chan->bounce_iv, ivsize);
358 	}
359 
360 	dma_unmap_single(ce->dev, rctx->addr_key, op->keylen, DMA_TO_DEVICE);
361 }
362 
363 int sun8i_ce_cipher_do_one(struct crypto_engine *engine, void *areq)
364 {
365 	int err = sun8i_ce_cipher_prepare(engine, areq);
366 
367 	if (err)
368 		return err;
369 
370 	sun8i_ce_cipher_run(engine, areq);
371 	sun8i_ce_cipher_unprepare(engine, areq);
372 	return 0;
373 }
374 
375 int sun8i_ce_skdecrypt(struct skcipher_request *areq)
376 {
377 	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq);
378 	struct sun8i_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm);
379 	struct sun8i_cipher_req_ctx *rctx = skcipher_request_ctx(areq);
380 	struct crypto_engine *engine;
381 	int e;
382 
383 	rctx->op_dir = CE_DECRYPTION;
384 	if (sun8i_ce_cipher_need_fallback(areq))
385 		return sun8i_ce_cipher_fallback(areq);
386 
387 	e = sun8i_ce_get_engine_number(op->ce);
388 	rctx->flow = e;
389 	engine = op->ce->chanlist[e].engine;
390 
391 	return crypto_transfer_skcipher_request_to_engine(engine, areq);
392 }
393 
394 int sun8i_ce_skencrypt(struct skcipher_request *areq)
395 {
396 	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq);
397 	struct sun8i_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm);
398 	struct sun8i_cipher_req_ctx *rctx = skcipher_request_ctx(areq);
399 	struct crypto_engine *engine;
400 	int e;
401 
402 	rctx->op_dir = CE_ENCRYPTION;
403 	if (sun8i_ce_cipher_need_fallback(areq))
404 		return sun8i_ce_cipher_fallback(areq);
405 
406 	e = sun8i_ce_get_engine_number(op->ce);
407 	rctx->flow = e;
408 	engine = op->ce->chanlist[e].engine;
409 
410 	return crypto_transfer_skcipher_request_to_engine(engine, areq);
411 }
412 
413 int sun8i_ce_cipher_init(struct crypto_tfm *tfm)
414 {
415 	struct sun8i_cipher_tfm_ctx *op = crypto_tfm_ctx(tfm);
416 	struct sun8i_ce_alg_template *algt;
417 	const char *name = crypto_tfm_alg_name(tfm);
418 	struct crypto_skcipher *sktfm = __crypto_skcipher_cast(tfm);
419 	struct skcipher_alg *alg = crypto_skcipher_alg(sktfm);
420 	int err;
421 
422 	memset(op, 0, sizeof(struct sun8i_cipher_tfm_ctx));
423 
424 	algt = container_of(alg, struct sun8i_ce_alg_template, alg.skcipher.base);
425 	op->ce = algt->ce;
426 
427 	op->fallback_tfm = crypto_alloc_skcipher(name, 0, CRYPTO_ALG_NEED_FALLBACK);
428 	if (IS_ERR(op->fallback_tfm)) {
429 		dev_err(op->ce->dev, "ERROR: Cannot allocate fallback for %s %ld\n",
430 			name, PTR_ERR(op->fallback_tfm));
431 		return PTR_ERR(op->fallback_tfm);
432 	}
433 
434 	sktfm->reqsize = sizeof(struct sun8i_cipher_req_ctx) +
435 			 crypto_skcipher_reqsize(op->fallback_tfm);
436 
437 	memcpy(algt->fbname,
438 	       crypto_tfm_alg_driver_name(crypto_skcipher_tfm(op->fallback_tfm)),
439 	       CRYPTO_MAX_ALG_NAME);
440 
441 	err = pm_runtime_get_sync(op->ce->dev);
442 	if (err < 0)
443 		goto error_pm;
444 
445 	return 0;
446 error_pm:
447 	pm_runtime_put_noidle(op->ce->dev);
448 	crypto_free_skcipher(op->fallback_tfm);
449 	return err;
450 }
451 
452 void sun8i_ce_cipher_exit(struct crypto_tfm *tfm)
453 {
454 	struct sun8i_cipher_tfm_ctx *op = crypto_tfm_ctx(tfm);
455 
456 	kfree_sensitive(op->key);
457 	crypto_free_skcipher(op->fallback_tfm);
458 	pm_runtime_put_sync_suspend(op->ce->dev);
459 }
460 
461 int sun8i_ce_aes_setkey(struct crypto_skcipher *tfm, const u8 *key,
462 			unsigned int keylen)
463 {
464 	struct sun8i_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm);
465 	struct sun8i_ce_dev *ce = op->ce;
466 
467 	switch (keylen) {
468 	case 128 / 8:
469 		break;
470 	case 192 / 8:
471 		break;
472 	case 256 / 8:
473 		break;
474 	default:
475 		dev_dbg(ce->dev, "ERROR: Invalid keylen %u\n", keylen);
476 		return -EINVAL;
477 	}
478 	kfree_sensitive(op->key);
479 	op->keylen = keylen;
480 	op->key = kmemdup(key, keylen, GFP_KERNEL | GFP_DMA);
481 	if (!op->key)
482 		return -ENOMEM;
483 
484 	crypto_skcipher_clear_flags(op->fallback_tfm, CRYPTO_TFM_REQ_MASK);
485 	crypto_skcipher_set_flags(op->fallback_tfm, tfm->base.crt_flags & CRYPTO_TFM_REQ_MASK);
486 
487 	return crypto_skcipher_setkey(op->fallback_tfm, key, keylen);
488 }
489 
490 int sun8i_ce_des3_setkey(struct crypto_skcipher *tfm, const u8 *key,
491 			 unsigned int keylen)
492 {
493 	struct sun8i_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm);
494 	int err;
495 
496 	err = verify_skcipher_des3_key(tfm, key);
497 	if (err)
498 		return err;
499 
500 	kfree_sensitive(op->key);
501 	op->keylen = keylen;
502 	op->key = kmemdup(key, keylen, GFP_KERNEL | GFP_DMA);
503 	if (!op->key)
504 		return -ENOMEM;
505 
506 	crypto_skcipher_clear_flags(op->fallback_tfm, CRYPTO_TFM_REQ_MASK);
507 	crypto_skcipher_set_flags(op->fallback_tfm, tfm->base.crt_flags & CRYPTO_TFM_REQ_MASK);
508 
509 	return crypto_skcipher_setkey(op->fallback_tfm, key, keylen);
510 }
511