1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Crypto acceleration support for Rockchip RK3288
4  *
5  * Copyright (c) 2015, Fuzhou Rockchip Electronics Co., Ltd
6  *
7  * Author: Zain Wang <zain.wang@rock-chips.com>
8  *
9  * Some ideas are from marvell-cesa.c and s5p-sss.c driver.
10  */
11 
12 #include "rk3288_crypto.h"
13 #include <linux/dma-mapping.h>
14 #include <linux/module.h>
15 #include <linux/platform_device.h>
16 #include <linux/of.h>
17 #include <linux/clk.h>
18 #include <linux/crypto.h>
19 #include <linux/reset.h>
20 
21 static int rk_crypto_enable_clk(struct rk_crypto_info *dev)
22 {
23 	int err;
24 
25 	err = clk_prepare_enable(dev->sclk);
26 	if (err) {
27 		dev_err(dev->dev, "[%s:%d], Couldn't enable clock sclk\n",
28 			__func__, __LINE__);
29 		goto err_return;
30 	}
31 	err = clk_prepare_enable(dev->aclk);
32 	if (err) {
33 		dev_err(dev->dev, "[%s:%d], Couldn't enable clock aclk\n",
34 			__func__, __LINE__);
35 		goto err_aclk;
36 	}
37 	err = clk_prepare_enable(dev->hclk);
38 	if (err) {
39 		dev_err(dev->dev, "[%s:%d], Couldn't enable clock hclk\n",
40 			__func__, __LINE__);
41 		goto err_hclk;
42 	}
43 	err = clk_prepare_enable(dev->dmaclk);
44 	if (err) {
45 		dev_err(dev->dev, "[%s:%d], Couldn't enable clock dmaclk\n",
46 			__func__, __LINE__);
47 		goto err_dmaclk;
48 	}
49 	return err;
50 err_dmaclk:
51 	clk_disable_unprepare(dev->hclk);
52 err_hclk:
53 	clk_disable_unprepare(dev->aclk);
54 err_aclk:
55 	clk_disable_unprepare(dev->sclk);
56 err_return:
57 	return err;
58 }
59 
60 static void rk_crypto_disable_clk(struct rk_crypto_info *dev)
61 {
62 	clk_disable_unprepare(dev->dmaclk);
63 	clk_disable_unprepare(dev->hclk);
64 	clk_disable_unprepare(dev->aclk);
65 	clk_disable_unprepare(dev->sclk);
66 }
67 
68 static int check_alignment(struct scatterlist *sg_src,
69 			   struct scatterlist *sg_dst,
70 			   int align_mask)
71 {
72 	int in, out, align;
73 
74 	in = IS_ALIGNED((uint32_t)sg_src->offset, 4) &&
75 	     IS_ALIGNED((uint32_t)sg_src->length, align_mask);
76 	if (!sg_dst)
77 		return in;
78 	out = IS_ALIGNED((uint32_t)sg_dst->offset, 4) &&
79 	      IS_ALIGNED((uint32_t)sg_dst->length, align_mask);
80 	align = in && out;
81 
82 	return (align && (sg_src->length == sg_dst->length));
83 }
84 
85 static int rk_load_data(struct rk_crypto_info *dev,
86 			struct scatterlist *sg_src,
87 			struct scatterlist *sg_dst)
88 {
89 	unsigned int count;
90 
91 	dev->aligned = dev->aligned ?
92 		check_alignment(sg_src, sg_dst, dev->align_size) :
93 		dev->aligned;
94 	if (dev->aligned) {
95 		count = min(dev->left_bytes, sg_src->length);
96 		dev->left_bytes -= count;
97 
98 		if (!dma_map_sg(dev->dev, sg_src, 1, DMA_TO_DEVICE)) {
99 			dev_err(dev->dev, "[%s:%d] dma_map_sg(src)  error\n",
100 				__func__, __LINE__);
101 			return -EINVAL;
102 		}
103 		dev->addr_in = sg_dma_address(sg_src);
104 
105 		if (sg_dst) {
106 			if (!dma_map_sg(dev->dev, sg_dst, 1, DMA_FROM_DEVICE)) {
107 				dev_err(dev->dev,
108 					"[%s:%d] dma_map_sg(dst)  error\n",
109 					__func__, __LINE__);
110 				dma_unmap_sg(dev->dev, sg_src, 1,
111 					     DMA_TO_DEVICE);
112 				return -EINVAL;
113 			}
114 			dev->addr_out = sg_dma_address(sg_dst);
115 		}
116 	} else {
117 		count = (dev->left_bytes > PAGE_SIZE) ?
118 			PAGE_SIZE : dev->left_bytes;
119 
120 		if (!sg_pcopy_to_buffer(dev->first, dev->src_nents,
121 					dev->addr_vir, count,
122 					dev->total - dev->left_bytes)) {
123 			dev_err(dev->dev, "[%s:%d] pcopy err\n",
124 				__func__, __LINE__);
125 			return -EINVAL;
126 		}
127 		dev->left_bytes -= count;
128 		sg_init_one(&dev->sg_tmp, dev->addr_vir, count);
129 		if (!dma_map_sg(dev->dev, &dev->sg_tmp, 1, DMA_TO_DEVICE)) {
130 			dev_err(dev->dev, "[%s:%d] dma_map_sg(sg_tmp)  error\n",
131 				__func__, __LINE__);
132 			return -ENOMEM;
133 		}
134 		dev->addr_in = sg_dma_address(&dev->sg_tmp);
135 
136 		if (sg_dst) {
137 			if (!dma_map_sg(dev->dev, &dev->sg_tmp, 1,
138 					DMA_FROM_DEVICE)) {
139 				dev_err(dev->dev,
140 					"[%s:%d] dma_map_sg(sg_tmp)  error\n",
141 					__func__, __LINE__);
142 				dma_unmap_sg(dev->dev, &dev->sg_tmp, 1,
143 					     DMA_TO_DEVICE);
144 				return -ENOMEM;
145 			}
146 			dev->addr_out = sg_dma_address(&dev->sg_tmp);
147 		}
148 	}
149 	dev->count = count;
150 	return 0;
151 }
152 
153 static void rk_unload_data(struct rk_crypto_info *dev)
154 {
155 	struct scatterlist *sg_in, *sg_out;
156 
157 	sg_in = dev->aligned ? dev->sg_src : &dev->sg_tmp;
158 	dma_unmap_sg(dev->dev, sg_in, 1, DMA_TO_DEVICE);
159 
160 	if (dev->sg_dst) {
161 		sg_out = dev->aligned ? dev->sg_dst : &dev->sg_tmp;
162 		dma_unmap_sg(dev->dev, sg_out, 1, DMA_FROM_DEVICE);
163 	}
164 }
165 
166 static irqreturn_t rk_crypto_irq_handle(int irq, void *dev_id)
167 {
168 	struct rk_crypto_info *dev  = platform_get_drvdata(dev_id);
169 	u32 interrupt_status;
170 
171 	spin_lock(&dev->lock);
172 	interrupt_status = CRYPTO_READ(dev, RK_CRYPTO_INTSTS);
173 	CRYPTO_WRITE(dev, RK_CRYPTO_INTSTS, interrupt_status);
174 
175 	if (interrupt_status & 0x0a) {
176 		dev_warn(dev->dev, "DMA Error\n");
177 		dev->err = -EFAULT;
178 	}
179 	tasklet_schedule(&dev->done_task);
180 
181 	spin_unlock(&dev->lock);
182 	return IRQ_HANDLED;
183 }
184 
185 static int rk_crypto_enqueue(struct rk_crypto_info *dev,
186 			      struct crypto_async_request *async_req)
187 {
188 	unsigned long flags;
189 	int ret;
190 
191 	spin_lock_irqsave(&dev->lock, flags);
192 	ret = crypto_enqueue_request(&dev->queue, async_req);
193 	if (dev->busy) {
194 		spin_unlock_irqrestore(&dev->lock, flags);
195 		return ret;
196 	}
197 	dev->busy = true;
198 	spin_unlock_irqrestore(&dev->lock, flags);
199 	tasklet_schedule(&dev->queue_task);
200 
201 	return ret;
202 }
203 
204 static void rk_crypto_queue_task_cb(unsigned long data)
205 {
206 	struct rk_crypto_info *dev = (struct rk_crypto_info *)data;
207 	struct crypto_async_request *async_req, *backlog;
208 	unsigned long flags;
209 	int err = 0;
210 
211 	dev->err = 0;
212 	spin_lock_irqsave(&dev->lock, flags);
213 	backlog   = crypto_get_backlog(&dev->queue);
214 	async_req = crypto_dequeue_request(&dev->queue);
215 
216 	if (!async_req) {
217 		dev->busy = false;
218 		spin_unlock_irqrestore(&dev->lock, flags);
219 		return;
220 	}
221 	spin_unlock_irqrestore(&dev->lock, flags);
222 
223 	if (backlog) {
224 		backlog->complete(backlog, -EINPROGRESS);
225 		backlog = NULL;
226 	}
227 
228 	dev->async_req = async_req;
229 	err = dev->start(dev);
230 	if (err)
231 		dev->complete(dev->async_req, err);
232 }
233 
234 static void rk_crypto_done_task_cb(unsigned long data)
235 {
236 	struct rk_crypto_info *dev = (struct rk_crypto_info *)data;
237 
238 	if (dev->err) {
239 		dev->complete(dev->async_req, dev->err);
240 		return;
241 	}
242 
243 	dev->err = dev->update(dev);
244 	if (dev->err)
245 		dev->complete(dev->async_req, dev->err);
246 }
247 
248 static struct rk_crypto_tmp *rk_cipher_algs[] = {
249 	&rk_ecb_aes_alg,
250 	&rk_cbc_aes_alg,
251 	&rk_ecb_des_alg,
252 	&rk_cbc_des_alg,
253 	&rk_ecb_des3_ede_alg,
254 	&rk_cbc_des3_ede_alg,
255 	&rk_ahash_sha1,
256 	&rk_ahash_sha256,
257 	&rk_ahash_md5,
258 };
259 
260 static int rk_crypto_register(struct rk_crypto_info *crypto_info)
261 {
262 	unsigned int i, k;
263 	int err = 0;
264 
265 	for (i = 0; i < ARRAY_SIZE(rk_cipher_algs); i++) {
266 		rk_cipher_algs[i]->dev = crypto_info;
267 		if (rk_cipher_algs[i]->type == ALG_TYPE_CIPHER)
268 			err = crypto_register_skcipher(
269 					&rk_cipher_algs[i]->alg.skcipher);
270 		else
271 			err = crypto_register_ahash(
272 					&rk_cipher_algs[i]->alg.hash);
273 		if (err)
274 			goto err_cipher_algs;
275 	}
276 	return 0;
277 
278 err_cipher_algs:
279 	for (k = 0; k < i; k++) {
280 		if (rk_cipher_algs[i]->type == ALG_TYPE_CIPHER)
281 			crypto_unregister_skcipher(&rk_cipher_algs[k]->alg.skcipher);
282 		else
283 			crypto_unregister_ahash(&rk_cipher_algs[i]->alg.hash);
284 	}
285 	return err;
286 }
287 
288 static void rk_crypto_unregister(void)
289 {
290 	unsigned int i;
291 
292 	for (i = 0; i < ARRAY_SIZE(rk_cipher_algs); i++) {
293 		if (rk_cipher_algs[i]->type == ALG_TYPE_CIPHER)
294 			crypto_unregister_skcipher(&rk_cipher_algs[i]->alg.skcipher);
295 		else
296 			crypto_unregister_ahash(&rk_cipher_algs[i]->alg.hash);
297 	}
298 }
299 
300 static void rk_crypto_action(void *data)
301 {
302 	struct rk_crypto_info *crypto_info = data;
303 
304 	reset_control_assert(crypto_info->rst);
305 }
306 
307 static const struct of_device_id crypto_of_id_table[] = {
308 	{ .compatible = "rockchip,rk3288-crypto" },
309 	{}
310 };
311 MODULE_DEVICE_TABLE(of, crypto_of_id_table);
312 
313 static int rk_crypto_probe(struct platform_device *pdev)
314 {
315 	struct device *dev = &pdev->dev;
316 	struct rk_crypto_info *crypto_info;
317 	int err = 0;
318 
319 	crypto_info = devm_kzalloc(&pdev->dev,
320 				   sizeof(*crypto_info), GFP_KERNEL);
321 	if (!crypto_info) {
322 		err = -ENOMEM;
323 		goto err_crypto;
324 	}
325 
326 	crypto_info->rst = devm_reset_control_get(dev, "crypto-rst");
327 	if (IS_ERR(crypto_info->rst)) {
328 		err = PTR_ERR(crypto_info->rst);
329 		goto err_crypto;
330 	}
331 
332 	reset_control_assert(crypto_info->rst);
333 	usleep_range(10, 20);
334 	reset_control_deassert(crypto_info->rst);
335 
336 	err = devm_add_action_or_reset(dev, rk_crypto_action, crypto_info);
337 	if (err)
338 		goto err_crypto;
339 
340 	spin_lock_init(&crypto_info->lock);
341 
342 	crypto_info->reg = devm_platform_ioremap_resource(pdev, 0);
343 	if (IS_ERR(crypto_info->reg)) {
344 		err = PTR_ERR(crypto_info->reg);
345 		goto err_crypto;
346 	}
347 
348 	crypto_info->aclk = devm_clk_get(&pdev->dev, "aclk");
349 	if (IS_ERR(crypto_info->aclk)) {
350 		err = PTR_ERR(crypto_info->aclk);
351 		goto err_crypto;
352 	}
353 
354 	crypto_info->hclk = devm_clk_get(&pdev->dev, "hclk");
355 	if (IS_ERR(crypto_info->hclk)) {
356 		err = PTR_ERR(crypto_info->hclk);
357 		goto err_crypto;
358 	}
359 
360 	crypto_info->sclk = devm_clk_get(&pdev->dev, "sclk");
361 	if (IS_ERR(crypto_info->sclk)) {
362 		err = PTR_ERR(crypto_info->sclk);
363 		goto err_crypto;
364 	}
365 
366 	crypto_info->dmaclk = devm_clk_get(&pdev->dev, "apb_pclk");
367 	if (IS_ERR(crypto_info->dmaclk)) {
368 		err = PTR_ERR(crypto_info->dmaclk);
369 		goto err_crypto;
370 	}
371 
372 	crypto_info->irq = platform_get_irq(pdev, 0);
373 	if (crypto_info->irq < 0) {
374 		dev_warn(crypto_info->dev,
375 			 "control Interrupt is not available.\n");
376 		err = crypto_info->irq;
377 		goto err_crypto;
378 	}
379 
380 	err = devm_request_irq(&pdev->dev, crypto_info->irq,
381 			       rk_crypto_irq_handle, IRQF_SHARED,
382 			       "rk-crypto", pdev);
383 
384 	if (err) {
385 		dev_err(crypto_info->dev, "irq request failed.\n");
386 		goto err_crypto;
387 	}
388 
389 	crypto_info->dev = &pdev->dev;
390 	platform_set_drvdata(pdev, crypto_info);
391 
392 	tasklet_init(&crypto_info->queue_task,
393 		     rk_crypto_queue_task_cb, (unsigned long)crypto_info);
394 	tasklet_init(&crypto_info->done_task,
395 		     rk_crypto_done_task_cb, (unsigned long)crypto_info);
396 	crypto_init_queue(&crypto_info->queue, 50);
397 
398 	crypto_info->enable_clk = rk_crypto_enable_clk;
399 	crypto_info->disable_clk = rk_crypto_disable_clk;
400 	crypto_info->load_data = rk_load_data;
401 	crypto_info->unload_data = rk_unload_data;
402 	crypto_info->enqueue = rk_crypto_enqueue;
403 	crypto_info->busy = false;
404 
405 	err = rk_crypto_register(crypto_info);
406 	if (err) {
407 		dev_err(dev, "err in register alg");
408 		goto err_register_alg;
409 	}
410 
411 	dev_info(dev, "Crypto Accelerator successfully registered\n");
412 	return 0;
413 
414 err_register_alg:
415 	tasklet_kill(&crypto_info->queue_task);
416 	tasklet_kill(&crypto_info->done_task);
417 err_crypto:
418 	return err;
419 }
420 
421 static int rk_crypto_remove(struct platform_device *pdev)
422 {
423 	struct rk_crypto_info *crypto_tmp = platform_get_drvdata(pdev);
424 
425 	rk_crypto_unregister();
426 	tasklet_kill(&crypto_tmp->done_task);
427 	tasklet_kill(&crypto_tmp->queue_task);
428 	return 0;
429 }
430 
431 static struct platform_driver crypto_driver = {
432 	.probe		= rk_crypto_probe,
433 	.remove		= rk_crypto_remove,
434 	.driver		= {
435 		.name	= "rk3288-crypto",
436 		.of_match_table	= crypto_of_id_table,
437 	},
438 };
439 
440 module_platform_driver(crypto_driver);
441 
442 MODULE_AUTHOR("Zain Wang <zain.wang@rock-chips.com>");
443 MODULE_DESCRIPTION("Support for Rockchip's cryptographic engine");
444 MODULE_LICENSE("GPL");
445