1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * sl3516-ce-cipher.c - hardware cryptographic offloader for Storlink SL3516 SoC
4  *
5  * Copyright (C) 2021 Corentin LABBE <clabbe@baylibre.com>
6  *
7  * This file adds support for AES cipher with 128,192,256 bits keysize in
8  * ECB mode.
9  */
10 
11 #include <crypto/engine.h>
12 #include <crypto/internal/skcipher.h>
13 #include <crypto/scatterwalk.h>
14 #include <linux/dma-mapping.h>
15 #include <linux/delay.h>
16 #include <linux/err.h>
17 #include <linux/io.h>
18 #include <linux/kernel.h>
19 #include <linux/pm_runtime.h>
20 #include <linux/slab.h>
21 #include <linux/string.h>
22 #include "sl3516-ce.h"
23 
24 /* sl3516_ce_need_fallback - check if a request can be handled by the CE */
25 static bool sl3516_ce_need_fallback(struct skcipher_request *areq)
26 {
27 	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq);
28 	struct sl3516_ce_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm);
29 	struct sl3516_ce_dev *ce = op->ce;
30 	struct scatterlist *in_sg;
31 	struct scatterlist *out_sg;
32 	struct scatterlist *sg;
33 
34 	if (areq->cryptlen == 0 || areq->cryptlen % 16) {
35 		ce->fallback_mod16++;
36 		return true;
37 	}
38 
39 	/*
40 	 * check if we have enough descriptors for TX
41 	 * Note: TX need one control desc for each SG
42 	 */
43 	if (sg_nents(areq->src) > MAXDESC / 2) {
44 		ce->fallback_sg_count_tx++;
45 		return true;
46 	}
47 	/* check if we have enough descriptors for RX */
48 	if (sg_nents(areq->dst) > MAXDESC) {
49 		ce->fallback_sg_count_rx++;
50 		return true;
51 	}
52 
53 	sg = areq->src;
54 	while (sg) {
55 		if ((sg->length % 16) != 0) {
56 			ce->fallback_mod16++;
57 			return true;
58 		}
59 		if ((sg_dma_len(sg) % 16) != 0) {
60 			ce->fallback_mod16++;
61 			return true;
62 		}
63 		if (!IS_ALIGNED(sg->offset, 16)) {
64 			ce->fallback_align16++;
65 			return true;
66 		}
67 		sg = sg_next(sg);
68 	}
69 	sg = areq->dst;
70 	while (sg) {
71 		if ((sg->length % 16) != 0) {
72 			ce->fallback_mod16++;
73 			return true;
74 		}
75 		if ((sg_dma_len(sg) % 16) != 0) {
76 			ce->fallback_mod16++;
77 			return true;
78 		}
79 		if (!IS_ALIGNED(sg->offset, 16)) {
80 			ce->fallback_align16++;
81 			return true;
82 		}
83 		sg = sg_next(sg);
84 	}
85 
86 	/* need same numbers of SG (with same length) for source and destination */
87 	in_sg = areq->src;
88 	out_sg = areq->dst;
89 	while (in_sg && out_sg) {
90 		if (in_sg->length != out_sg->length) {
91 			ce->fallback_not_same_len++;
92 			return true;
93 		}
94 		in_sg = sg_next(in_sg);
95 		out_sg = sg_next(out_sg);
96 	}
97 	if (in_sg || out_sg)
98 		return true;
99 
100 	return false;
101 }
102 
103 static int sl3516_ce_cipher_fallback(struct skcipher_request *areq)
104 {
105 	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq);
106 	struct sl3516_ce_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm);
107 	struct sl3516_ce_cipher_req_ctx *rctx = skcipher_request_ctx(areq);
108 	struct skcipher_alg *alg = crypto_skcipher_alg(tfm);
109 	struct sl3516_ce_alg_template *algt;
110 	int err;
111 
112 	algt = container_of(alg, struct sl3516_ce_alg_template, alg.skcipher.base);
113 	algt->stat_fb++;
114 
115 	skcipher_request_set_tfm(&rctx->fallback_req, op->fallback_tfm);
116 	skcipher_request_set_callback(&rctx->fallback_req, areq->base.flags,
117 				      areq->base.complete, areq->base.data);
118 	skcipher_request_set_crypt(&rctx->fallback_req, areq->src, areq->dst,
119 				   areq->cryptlen, areq->iv);
120 	if (rctx->op_dir == CE_DECRYPTION)
121 		err = crypto_skcipher_decrypt(&rctx->fallback_req);
122 	else
123 		err = crypto_skcipher_encrypt(&rctx->fallback_req);
124 	return err;
125 }
126 
127 static int sl3516_ce_cipher(struct skcipher_request *areq)
128 {
129 	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq);
130 	struct sl3516_ce_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm);
131 	struct sl3516_ce_dev *ce = op->ce;
132 	struct sl3516_ce_cipher_req_ctx *rctx = skcipher_request_ctx(areq);
133 	struct skcipher_alg *alg = crypto_skcipher_alg(tfm);
134 	struct sl3516_ce_alg_template *algt;
135 	struct scatterlist *sg;
136 	unsigned int todo, len;
137 	struct pkt_control_ecb *ecb;
138 	int nr_sgs = 0;
139 	int nr_sgd = 0;
140 	int err = 0;
141 	int i;
142 
143 	algt = container_of(alg, struct sl3516_ce_alg_template, alg.skcipher.base);
144 
145 	dev_dbg(ce->dev, "%s %s %u %x IV(%p %u) key=%u\n", __func__,
146 		crypto_tfm_alg_name(areq->base.tfm),
147 		areq->cryptlen,
148 		rctx->op_dir, areq->iv, crypto_skcipher_ivsize(tfm),
149 		op->keylen);
150 
151 	algt->stat_req++;
152 
153 	if (areq->src == areq->dst) {
154 		nr_sgs = dma_map_sg(ce->dev, areq->src, sg_nents(areq->src),
155 				    DMA_BIDIRECTIONAL);
156 		if (nr_sgs <= 0 || nr_sgs > MAXDESC / 2) {
157 			dev_err(ce->dev, "Invalid sg number %d\n", nr_sgs);
158 			err = -EINVAL;
159 			goto theend;
160 		}
161 		nr_sgd = nr_sgs;
162 	} else {
163 		nr_sgs = dma_map_sg(ce->dev, areq->src, sg_nents(areq->src),
164 				    DMA_TO_DEVICE);
165 		if (nr_sgs <= 0 || nr_sgs > MAXDESC / 2) {
166 			dev_err(ce->dev, "Invalid sg number %d\n", nr_sgs);
167 			err = -EINVAL;
168 			goto theend;
169 		}
170 		nr_sgd = dma_map_sg(ce->dev, areq->dst, sg_nents(areq->dst),
171 				    DMA_FROM_DEVICE);
172 		if (nr_sgd <= 0 || nr_sgd > MAXDESC) {
173 			dev_err(ce->dev, "Invalid sg number %d\n", nr_sgd);
174 			err = -EINVAL;
175 			goto theend_sgs;
176 		}
177 	}
178 
179 	len = areq->cryptlen;
180 	i = 0;
181 	sg = areq->src;
182 	while (i < nr_sgs && sg && len) {
183 		if (sg_dma_len(sg) == 0)
184 			goto sgs_next;
185 		rctx->t_src[i].addr = sg_dma_address(sg);
186 		todo = min(len, sg_dma_len(sg));
187 		rctx->t_src[i].len = todo;
188 		dev_dbg(ce->dev, "%s total=%u SGS(%d %u off=%d) todo=%u\n", __func__,
189 			areq->cryptlen, i, rctx->t_src[i].len, sg->offset, todo);
190 		len -= todo;
191 		i++;
192 sgs_next:
193 		sg = sg_next(sg);
194 	}
195 	if (len > 0) {
196 		dev_err(ce->dev, "remaining len %d/%u nr_sgs=%d\n", len, areq->cryptlen, nr_sgs);
197 		err = -EINVAL;
198 		goto theend_sgs;
199 	}
200 
201 	len = areq->cryptlen;
202 	i = 0;
203 	sg = areq->dst;
204 	while (i < nr_sgd && sg && len) {
205 		if (sg_dma_len(sg) == 0)
206 			goto sgd_next;
207 		rctx->t_dst[i].addr = sg_dma_address(sg);
208 		todo = min(len, sg_dma_len(sg));
209 		rctx->t_dst[i].len = todo;
210 		dev_dbg(ce->dev, "%s total=%u SGD(%d %u off=%d) todo=%u\n", __func__,
211 			areq->cryptlen, i, rctx->t_dst[i].len, sg->offset, todo);
212 		len -= todo;
213 		i++;
214 
215 sgd_next:
216 		sg = sg_next(sg);
217 	}
218 	if (len > 0) {
219 		dev_err(ce->dev, "remaining len %d\n", len);
220 		err = -EINVAL;
221 		goto theend_sgs;
222 	}
223 
224 	switch (algt->mode) {
225 	case ECB_AES:
226 		rctx->pctrllen = sizeof(struct pkt_control_ecb);
227 		ecb = (struct pkt_control_ecb *)ce->pctrl;
228 
229 		rctx->tqflag = TQ0_TYPE_CTRL;
230 		rctx->tqflag |= TQ1_CIPHER;
231 		ecb->control.op_mode = rctx->op_dir;
232 		ecb->control.cipher_algorithm = ECB_AES;
233 		ecb->cipher.header_len = 0;
234 		ecb->cipher.algorithm_len = areq->cryptlen;
235 		cpu_to_be32_array((__be32 *)ecb->key, (u32 *)op->key, op->keylen / 4);
236 		rctx->h = &ecb->cipher;
237 
238 		rctx->tqflag |= TQ4_KEY0;
239 		rctx->tqflag |= TQ5_KEY4;
240 		rctx->tqflag |= TQ6_KEY6;
241 		ecb->control.aesnk = op->keylen / 4;
242 		break;
243 	}
244 
245 	rctx->nr_sgs = nr_sgs;
246 	rctx->nr_sgd = nr_sgd;
247 	err = sl3516_ce_run_task(ce, rctx, crypto_tfm_alg_name(areq->base.tfm));
248 
249 theend_sgs:
250 	if (areq->src == areq->dst) {
251 		dma_unmap_sg(ce->dev, areq->src, sg_nents(areq->src),
252 			     DMA_BIDIRECTIONAL);
253 	} else {
254 		dma_unmap_sg(ce->dev, areq->src, sg_nents(areq->src),
255 			     DMA_TO_DEVICE);
256 		dma_unmap_sg(ce->dev, areq->dst, sg_nents(areq->dst),
257 			     DMA_FROM_DEVICE);
258 	}
259 
260 theend:
261 
262 	return err;
263 }
264 
265 int sl3516_ce_handle_cipher_request(struct crypto_engine *engine, void *areq)
266 {
267 	int err;
268 	struct skcipher_request *breq = container_of(areq, struct skcipher_request, base);
269 
270 	err = sl3516_ce_cipher(breq);
271 	local_bh_disable();
272 	crypto_finalize_skcipher_request(engine, breq, err);
273 	local_bh_enable();
274 
275 	return 0;
276 }
277 
278 int sl3516_ce_skdecrypt(struct skcipher_request *areq)
279 {
280 	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq);
281 	struct sl3516_ce_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm);
282 	struct sl3516_ce_cipher_req_ctx *rctx = skcipher_request_ctx(areq);
283 	struct crypto_engine *engine;
284 
285 	memset(rctx, 0, sizeof(struct sl3516_ce_cipher_req_ctx));
286 	rctx->op_dir = CE_DECRYPTION;
287 
288 	if (sl3516_ce_need_fallback(areq))
289 		return sl3516_ce_cipher_fallback(areq);
290 
291 	engine = op->ce->engine;
292 
293 	return crypto_transfer_skcipher_request_to_engine(engine, areq);
294 }
295 
296 int sl3516_ce_skencrypt(struct skcipher_request *areq)
297 {
298 	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq);
299 	struct sl3516_ce_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm);
300 	struct sl3516_ce_cipher_req_ctx *rctx = skcipher_request_ctx(areq);
301 	struct crypto_engine *engine;
302 
303 	memset(rctx, 0, sizeof(struct sl3516_ce_cipher_req_ctx));
304 	rctx->op_dir = CE_ENCRYPTION;
305 
306 	if (sl3516_ce_need_fallback(areq))
307 		return sl3516_ce_cipher_fallback(areq);
308 
309 	engine = op->ce->engine;
310 
311 	return crypto_transfer_skcipher_request_to_engine(engine, areq);
312 }
313 
314 int sl3516_ce_cipher_init(struct crypto_tfm *tfm)
315 {
316 	struct sl3516_ce_cipher_tfm_ctx *op = crypto_tfm_ctx(tfm);
317 	struct sl3516_ce_alg_template *algt;
318 	const char *name = crypto_tfm_alg_name(tfm);
319 	struct crypto_skcipher *sktfm = __crypto_skcipher_cast(tfm);
320 	struct skcipher_alg *alg = crypto_skcipher_alg(sktfm);
321 	int err;
322 
323 	memset(op, 0, sizeof(struct sl3516_ce_cipher_tfm_ctx));
324 
325 	algt = container_of(alg, struct sl3516_ce_alg_template, alg.skcipher.base);
326 	op->ce = algt->ce;
327 
328 	op->fallback_tfm = crypto_alloc_skcipher(name, 0, CRYPTO_ALG_NEED_FALLBACK);
329 	if (IS_ERR(op->fallback_tfm)) {
330 		dev_err(op->ce->dev, "ERROR: Cannot allocate fallback for %s %ld\n",
331 			name, PTR_ERR(op->fallback_tfm));
332 		return PTR_ERR(op->fallback_tfm);
333 	}
334 
335 	sktfm->reqsize = sizeof(struct sl3516_ce_cipher_req_ctx) +
336 			 crypto_skcipher_reqsize(op->fallback_tfm);
337 
338 	dev_info(op->ce->dev, "Fallback for %s is %s\n",
339 		 crypto_tfm_alg_driver_name(&sktfm->base),
340 		 crypto_tfm_alg_driver_name(crypto_skcipher_tfm(op->fallback_tfm)));
341 
342 	err = pm_runtime_get_sync(op->ce->dev);
343 	if (err < 0)
344 		goto error_pm;
345 
346 	return 0;
347 error_pm:
348 	pm_runtime_put_noidle(op->ce->dev);
349 	crypto_free_skcipher(op->fallback_tfm);
350 	return err;
351 }
352 
353 void sl3516_ce_cipher_exit(struct crypto_tfm *tfm)
354 {
355 	struct sl3516_ce_cipher_tfm_ctx *op = crypto_tfm_ctx(tfm);
356 
357 	kfree_sensitive(op->key);
358 	crypto_free_skcipher(op->fallback_tfm);
359 	pm_runtime_put_sync_suspend(op->ce->dev);
360 }
361 
362 int sl3516_ce_aes_setkey(struct crypto_skcipher *tfm, const u8 *key,
363 			 unsigned int keylen)
364 {
365 	struct sl3516_ce_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm);
366 	struct sl3516_ce_dev *ce = op->ce;
367 
368 	switch (keylen) {
369 	case 128 / 8:
370 		break;
371 	case 192 / 8:
372 		break;
373 	case 256 / 8:
374 		break;
375 	default:
376 		dev_dbg(ce->dev, "ERROR: Invalid keylen %u\n", keylen);
377 		return -EINVAL;
378 	}
379 	kfree_sensitive(op->key);
380 	op->keylen = keylen;
381 	op->key = kmemdup(key, keylen, GFP_KERNEL | GFP_DMA);
382 	if (!op->key)
383 		return -ENOMEM;
384 
385 	crypto_skcipher_clear_flags(op->fallback_tfm, CRYPTO_TFM_REQ_MASK);
386 	crypto_skcipher_set_flags(op->fallback_tfm, tfm->base.crt_flags & CRYPTO_TFM_REQ_MASK);
387 
388 	return crypto_skcipher_setkey(op->fallback_tfm, key, keylen);
389 }
390