1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * amlogic-cipher.c - hardware cryptographic offloader for Amlogic GXL SoC
4  *
5  * Copyright (C) 2018-2019 Corentin LABBE <clabbe@baylibre.com>
6  *
7  * This file add support for AES cipher with 128,192,256 bits keysize in
8  * CBC and ECB mode.
9  */
10 
11 #include <linux/crypto.h>
12 #include <linux/delay.h>
13 #include <linux/io.h>
14 #include <crypto/scatterwalk.h>
15 #include <linux/scatterlist.h>
16 #include <linux/dma-mapping.h>
17 #include <crypto/internal/skcipher.h>
18 #include "amlogic-gxl.h"
19 
get_engine_number(struct meson_dev * mc)20 static int get_engine_number(struct meson_dev *mc)
21 {
22 	return atomic_inc_return(&mc->flow) % MAXFLOW;
23 }
24 
meson_cipher_need_fallback(struct skcipher_request * areq)25 static bool meson_cipher_need_fallback(struct skcipher_request *areq)
26 {
27 	struct scatterlist *src_sg = areq->src;
28 	struct scatterlist *dst_sg = areq->dst;
29 
30 	if (areq->cryptlen == 0)
31 		return true;
32 
33 	if (sg_nents(src_sg) != sg_nents(dst_sg))
34 		return true;
35 
36 	/* KEY/IV descriptors use 3 desc */
37 	if (sg_nents(src_sg) > MAXDESC - 3 || sg_nents(dst_sg) > MAXDESC - 3)
38 		return true;
39 
40 	while (src_sg && dst_sg) {
41 		if ((src_sg->length % 16) != 0)
42 			return true;
43 		if ((dst_sg->length % 16) != 0)
44 			return true;
45 		if (src_sg->length != dst_sg->length)
46 			return true;
47 		if (!IS_ALIGNED(src_sg->offset, sizeof(u32)))
48 			return true;
49 		if (!IS_ALIGNED(dst_sg->offset, sizeof(u32)))
50 			return true;
51 		src_sg = sg_next(src_sg);
52 		dst_sg = sg_next(dst_sg);
53 	}
54 
55 	return false;
56 }
57 
meson_cipher_do_fallback(struct skcipher_request * areq)58 static int meson_cipher_do_fallback(struct skcipher_request *areq)
59 {
60 	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq);
61 	struct meson_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm);
62 	struct meson_cipher_req_ctx *rctx = skcipher_request_ctx(areq);
63 	int err;
64 #ifdef CONFIG_CRYPTO_DEV_AMLOGIC_GXL_DEBUG
65 	struct skcipher_alg *alg = crypto_skcipher_alg(tfm);
66 	struct meson_alg_template *algt;
67 
68 	algt = container_of(alg, struct meson_alg_template, alg.skcipher.base);
69 	algt->stat_fb++;
70 #endif
71 	skcipher_request_set_tfm(&rctx->fallback_req, op->fallback_tfm);
72 	skcipher_request_set_callback(&rctx->fallback_req, areq->base.flags,
73 				      areq->base.complete, areq->base.data);
74 	skcipher_request_set_crypt(&rctx->fallback_req, areq->src, areq->dst,
75 				   areq->cryptlen, areq->iv);
76 
77 	if (rctx->op_dir == MESON_DECRYPT)
78 		err = crypto_skcipher_decrypt(&rctx->fallback_req);
79 	else
80 		err = crypto_skcipher_encrypt(&rctx->fallback_req);
81 	return err;
82 }
83 
meson_cipher(struct skcipher_request * areq)84 static int meson_cipher(struct skcipher_request *areq)
85 {
86 	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq);
87 	struct meson_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm);
88 	struct meson_cipher_req_ctx *rctx = skcipher_request_ctx(areq);
89 	struct meson_dev *mc = op->mc;
90 	struct skcipher_alg *alg = crypto_skcipher_alg(tfm);
91 	struct meson_alg_template *algt;
92 	int flow = rctx->flow;
93 	unsigned int todo, eat, len;
94 	struct scatterlist *src_sg = areq->src;
95 	struct scatterlist *dst_sg = areq->dst;
96 	struct meson_desc *desc;
97 	int nr_sgs, nr_sgd;
98 	int i, err = 0;
99 	unsigned int keyivlen, ivsize, offset, tloffset;
100 	dma_addr_t phykeyiv;
101 	void *backup_iv = NULL, *bkeyiv;
102 	u32 v;
103 
104 	algt = container_of(alg, struct meson_alg_template, alg.skcipher.base);
105 
106 	dev_dbg(mc->dev, "%s %s %u %x IV(%u) key=%u flow=%d\n", __func__,
107 		crypto_tfm_alg_name(areq->base.tfm),
108 		areq->cryptlen,
109 		rctx->op_dir, crypto_skcipher_ivsize(tfm),
110 		op->keylen, flow);
111 
112 #ifdef CONFIG_CRYPTO_DEV_AMLOGIC_GXL_DEBUG
113 	algt->stat_req++;
114 	mc->chanlist[flow].stat_req++;
115 #endif
116 
117 	/*
118 	 * The hardware expect a list of meson_desc structures.
119 	 * The 2 first structures store key
120 	 * The third stores IV
121 	 */
122 	bkeyiv = kzalloc(48, GFP_KERNEL | GFP_DMA);
123 	if (!bkeyiv)
124 		return -ENOMEM;
125 
126 	memcpy(bkeyiv, op->key, op->keylen);
127 	keyivlen = op->keylen;
128 
129 	ivsize = crypto_skcipher_ivsize(tfm);
130 	if (areq->iv && ivsize > 0) {
131 		if (ivsize > areq->cryptlen) {
132 			dev_err(mc->dev, "invalid ivsize=%d vs len=%d\n", ivsize, areq->cryptlen);
133 			err = -EINVAL;
134 			goto theend;
135 		}
136 		memcpy(bkeyiv + 32, areq->iv, ivsize);
137 		keyivlen = 48;
138 		if (rctx->op_dir == MESON_DECRYPT) {
139 			backup_iv = kzalloc(ivsize, GFP_KERNEL);
140 			if (!backup_iv) {
141 				err = -ENOMEM;
142 				goto theend;
143 			}
144 			offset = areq->cryptlen - ivsize;
145 			scatterwalk_map_and_copy(backup_iv, areq->src, offset,
146 						 ivsize, 0);
147 		}
148 	}
149 	if (keyivlen == 24)
150 		keyivlen = 32;
151 
152 	phykeyiv = dma_map_single(mc->dev, bkeyiv, keyivlen,
153 				  DMA_TO_DEVICE);
154 	err = dma_mapping_error(mc->dev, phykeyiv);
155 	if (err) {
156 		dev_err(mc->dev, "Cannot DMA MAP KEY IV\n");
157 		goto theend;
158 	}
159 
160 	tloffset = 0;
161 	eat = 0;
162 	i = 0;
163 	while (keyivlen > eat) {
164 		desc = &mc->chanlist[flow].tl[tloffset];
165 		memset(desc, 0, sizeof(struct meson_desc));
166 		todo = min(keyivlen - eat, 16u);
167 		desc->t_src = cpu_to_le32(phykeyiv + i * 16);
168 		desc->t_dst = cpu_to_le32(i * 16);
169 		v = (MODE_KEY << 20) | DESC_OWN | 16;
170 		desc->t_status = cpu_to_le32(v);
171 
172 		eat += todo;
173 		i++;
174 		tloffset++;
175 	}
176 
177 	if (areq->src == areq->dst) {
178 		nr_sgs = dma_map_sg(mc->dev, areq->src, sg_nents(areq->src),
179 				    DMA_BIDIRECTIONAL);
180 		if (!nr_sgs) {
181 			dev_err(mc->dev, "Invalid SG count %d\n", nr_sgs);
182 			err = -EINVAL;
183 			goto theend;
184 		}
185 		nr_sgd = nr_sgs;
186 	} else {
187 		nr_sgs = dma_map_sg(mc->dev, areq->src, sg_nents(areq->src),
188 				    DMA_TO_DEVICE);
189 		if (!nr_sgs || nr_sgs > MAXDESC - 3) {
190 			dev_err(mc->dev, "Invalid SG count %d\n", nr_sgs);
191 			err = -EINVAL;
192 			goto theend;
193 		}
194 		nr_sgd = dma_map_sg(mc->dev, areq->dst, sg_nents(areq->dst),
195 				    DMA_FROM_DEVICE);
196 		if (!nr_sgd || nr_sgd > MAXDESC - 3) {
197 			dev_err(mc->dev, "Invalid SG count %d\n", nr_sgd);
198 			err = -EINVAL;
199 			goto theend;
200 		}
201 	}
202 
203 	src_sg = areq->src;
204 	dst_sg = areq->dst;
205 	len = areq->cryptlen;
206 	while (src_sg) {
207 		desc = &mc->chanlist[flow].tl[tloffset];
208 		memset(desc, 0, sizeof(struct meson_desc));
209 
210 		desc->t_src = cpu_to_le32(sg_dma_address(src_sg));
211 		desc->t_dst = cpu_to_le32(sg_dma_address(dst_sg));
212 		todo = min(len, sg_dma_len(src_sg));
213 		v = (op->keymode << 20) | DESC_OWN | todo | (algt->blockmode << 26);
214 		if (rctx->op_dir)
215 			v |= DESC_ENCRYPTION;
216 		len -= todo;
217 
218 		if (!sg_next(src_sg))
219 			v |= DESC_LAST;
220 		desc->t_status = cpu_to_le32(v);
221 		tloffset++;
222 		src_sg = sg_next(src_sg);
223 		dst_sg = sg_next(dst_sg);
224 	}
225 
226 	reinit_completion(&mc->chanlist[flow].complete);
227 	mc->chanlist[flow].status = 0;
228 	writel(mc->chanlist[flow].t_phy | 2, mc->base + (flow << 2));
229 	wait_for_completion_interruptible_timeout(&mc->chanlist[flow].complete,
230 						  msecs_to_jiffies(500));
231 	if (mc->chanlist[flow].status == 0) {
232 		dev_err(mc->dev, "DMA timeout for flow %d\n", flow);
233 		err = -EINVAL;
234 	}
235 
236 	dma_unmap_single(mc->dev, phykeyiv, keyivlen, DMA_TO_DEVICE);
237 
238 	if (areq->src == areq->dst) {
239 		dma_unmap_sg(mc->dev, areq->src, sg_nents(areq->src), DMA_BIDIRECTIONAL);
240 	} else {
241 		dma_unmap_sg(mc->dev, areq->src, sg_nents(areq->src), DMA_TO_DEVICE);
242 		dma_unmap_sg(mc->dev, areq->dst, sg_nents(areq->dst), DMA_FROM_DEVICE);
243 	}
244 
245 	if (areq->iv && ivsize > 0) {
246 		if (rctx->op_dir == MESON_DECRYPT) {
247 			memcpy(areq->iv, backup_iv, ivsize);
248 		} else {
249 			scatterwalk_map_and_copy(areq->iv, areq->dst,
250 						 areq->cryptlen - ivsize,
251 						 ivsize, 0);
252 		}
253 	}
254 theend:
255 	kfree_sensitive(bkeyiv);
256 	kfree_sensitive(backup_iv);
257 
258 	return err;
259 }
260 
meson_handle_cipher_request(struct crypto_engine * engine,void * areq)261 int meson_handle_cipher_request(struct crypto_engine *engine, void *areq)
262 {
263 	int err;
264 	struct skcipher_request *breq = container_of(areq, struct skcipher_request, base);
265 
266 	err = meson_cipher(breq);
267 	local_bh_disable();
268 	crypto_finalize_skcipher_request(engine, breq, err);
269 	local_bh_enable();
270 
271 	return 0;
272 }
273 
meson_skdecrypt(struct skcipher_request * areq)274 int meson_skdecrypt(struct skcipher_request *areq)
275 {
276 	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq);
277 	struct meson_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm);
278 	struct meson_cipher_req_ctx *rctx = skcipher_request_ctx(areq);
279 	struct crypto_engine *engine;
280 	int e;
281 
282 	rctx->op_dir = MESON_DECRYPT;
283 	if (meson_cipher_need_fallback(areq))
284 		return meson_cipher_do_fallback(areq);
285 	e = get_engine_number(op->mc);
286 	engine = op->mc->chanlist[e].engine;
287 	rctx->flow = e;
288 
289 	return crypto_transfer_skcipher_request_to_engine(engine, areq);
290 }
291 
meson_skencrypt(struct skcipher_request * areq)292 int meson_skencrypt(struct skcipher_request *areq)
293 {
294 	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq);
295 	struct meson_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm);
296 	struct meson_cipher_req_ctx *rctx = skcipher_request_ctx(areq);
297 	struct crypto_engine *engine;
298 	int e;
299 
300 	rctx->op_dir = MESON_ENCRYPT;
301 	if (meson_cipher_need_fallback(areq))
302 		return meson_cipher_do_fallback(areq);
303 	e = get_engine_number(op->mc);
304 	engine = op->mc->chanlist[e].engine;
305 	rctx->flow = e;
306 
307 	return crypto_transfer_skcipher_request_to_engine(engine, areq);
308 }
309 
meson_cipher_init(struct crypto_tfm * tfm)310 int meson_cipher_init(struct crypto_tfm *tfm)
311 {
312 	struct meson_cipher_tfm_ctx *op = crypto_tfm_ctx(tfm);
313 	struct meson_alg_template *algt;
314 	const char *name = crypto_tfm_alg_name(tfm);
315 	struct crypto_skcipher *sktfm = __crypto_skcipher_cast(tfm);
316 	struct skcipher_alg *alg = crypto_skcipher_alg(sktfm);
317 
318 	memset(op, 0, sizeof(struct meson_cipher_tfm_ctx));
319 
320 	algt = container_of(alg, struct meson_alg_template, alg.skcipher.base);
321 	op->mc = algt->mc;
322 
323 	op->fallback_tfm = crypto_alloc_skcipher(name, 0, CRYPTO_ALG_NEED_FALLBACK);
324 	if (IS_ERR(op->fallback_tfm)) {
325 		dev_err(op->mc->dev, "ERROR: Cannot allocate fallback for %s %ld\n",
326 			name, PTR_ERR(op->fallback_tfm));
327 		return PTR_ERR(op->fallback_tfm);
328 	}
329 
330 	sktfm->reqsize = sizeof(struct meson_cipher_req_ctx) +
331 			 crypto_skcipher_reqsize(op->fallback_tfm);
332 
333 	return 0;
334 }
335 
meson_cipher_exit(struct crypto_tfm * tfm)336 void meson_cipher_exit(struct crypto_tfm *tfm)
337 {
338 	struct meson_cipher_tfm_ctx *op = crypto_tfm_ctx(tfm);
339 
340 	kfree_sensitive(op->key);
341 	crypto_free_skcipher(op->fallback_tfm);
342 }
343 
meson_aes_setkey(struct crypto_skcipher * tfm,const u8 * key,unsigned int keylen)344 int meson_aes_setkey(struct crypto_skcipher *tfm, const u8 *key,
345 		     unsigned int keylen)
346 {
347 	struct meson_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm);
348 	struct meson_dev *mc = op->mc;
349 
350 	switch (keylen) {
351 	case 128 / 8:
352 		op->keymode = MODE_AES_128;
353 		break;
354 	case 192 / 8:
355 		op->keymode = MODE_AES_192;
356 		break;
357 	case 256 / 8:
358 		op->keymode = MODE_AES_256;
359 		break;
360 	default:
361 		dev_dbg(mc->dev, "ERROR: Invalid keylen %u\n", keylen);
362 		return -EINVAL;
363 	}
364 	kfree_sensitive(op->key);
365 	op->keylen = keylen;
366 	op->key = kmemdup(key, keylen, GFP_KERNEL | GFP_DMA);
367 	if (!op->key)
368 		return -ENOMEM;
369 
370 	return crypto_skcipher_setkey(op->fallback_tfm, key, keylen);
371 }
372