1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Crypto acceleration support for Rockchip RK3288
4 *
5 * Copyright (c) 2015, Fuzhou Rockchip Electronics Co., Ltd
6 *
7 * Author: Zain Wang <zain.wang@rock-chips.com>
8 *
9 * Some ideas are from marvell/cesa.c and s5p-sss.c driver.
10 */
11
12 #include <asm/unaligned.h>
13 #include <crypto/internal/hash.h>
14 #include <linux/device.h>
15 #include <linux/err.h>
16 #include <linux/iopoll.h>
17 #include <linux/kernel.h>
18 #include <linux/module.h>
19 #include <linux/string.h>
20 #include "rk3288_crypto.h"
21
22 /*
23 * IC can not process zero message hash,
24 * so we put the fixed hash out when met zero message.
25 */
26
rk_ahash_need_fallback(struct ahash_request * req)27 static bool rk_ahash_need_fallback(struct ahash_request *req)
28 {
29 struct scatterlist *sg;
30
31 sg = req->src;
32 while (sg) {
33 if (!IS_ALIGNED(sg->offset, sizeof(u32))) {
34 return true;
35 }
36 if (sg->length % 4) {
37 return true;
38 }
39 sg = sg_next(sg);
40 }
41 return false;
42 }
43
rk_ahash_digest_fb(struct ahash_request * areq)44 static int rk_ahash_digest_fb(struct ahash_request *areq)
45 {
46 struct rk_ahash_rctx *rctx = ahash_request_ctx(areq);
47 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
48 struct rk_ahash_ctx *tfmctx = crypto_ahash_ctx(tfm);
49 struct ahash_alg *alg = crypto_ahash_alg(tfm);
50 struct rk_crypto_tmp *algt = container_of(alg, struct rk_crypto_tmp, alg.hash.base);
51
52 algt->stat_fb++;
53
54 ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm);
55 rctx->fallback_req.base.flags = areq->base.flags &
56 CRYPTO_TFM_REQ_MAY_SLEEP;
57
58 rctx->fallback_req.nbytes = areq->nbytes;
59 rctx->fallback_req.src = areq->src;
60 rctx->fallback_req.result = areq->result;
61
62 return crypto_ahash_digest(&rctx->fallback_req);
63 }
64
zero_message_process(struct ahash_request * req)65 static int zero_message_process(struct ahash_request *req)
66 {
67 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
68 int rk_digest_size = crypto_ahash_digestsize(tfm);
69
70 switch (rk_digest_size) {
71 case SHA1_DIGEST_SIZE:
72 memcpy(req->result, sha1_zero_message_hash, rk_digest_size);
73 break;
74 case SHA256_DIGEST_SIZE:
75 memcpy(req->result, sha256_zero_message_hash, rk_digest_size);
76 break;
77 case MD5_DIGEST_SIZE:
78 memcpy(req->result, md5_zero_message_hash, rk_digest_size);
79 break;
80 default:
81 return -EINVAL;
82 }
83
84 return 0;
85 }
86
rk_ahash_reg_init(struct ahash_request * req,struct rk_crypto_info * dev)87 static void rk_ahash_reg_init(struct ahash_request *req,
88 struct rk_crypto_info *dev)
89 {
90 struct rk_ahash_rctx *rctx = ahash_request_ctx(req);
91 int reg_status;
92
93 reg_status = CRYPTO_READ(dev, RK_CRYPTO_CTRL) |
94 RK_CRYPTO_HASH_FLUSH | _SBF(0xffff, 16);
95 CRYPTO_WRITE(dev, RK_CRYPTO_CTRL, reg_status);
96
97 reg_status = CRYPTO_READ(dev, RK_CRYPTO_CTRL);
98 reg_status &= (~RK_CRYPTO_HASH_FLUSH);
99 reg_status |= _SBF(0xffff, 16);
100 CRYPTO_WRITE(dev, RK_CRYPTO_CTRL, reg_status);
101
102 memset_io(dev->reg + RK_CRYPTO_HASH_DOUT_0, 0, 32);
103
104 CRYPTO_WRITE(dev, RK_CRYPTO_INTENA, RK_CRYPTO_HRDMA_ERR_ENA |
105 RK_CRYPTO_HRDMA_DONE_ENA);
106
107 CRYPTO_WRITE(dev, RK_CRYPTO_INTSTS, RK_CRYPTO_HRDMA_ERR_INT |
108 RK_CRYPTO_HRDMA_DONE_INT);
109
110 CRYPTO_WRITE(dev, RK_CRYPTO_HASH_CTRL, rctx->mode |
111 RK_CRYPTO_HASH_SWAP_DO);
112
113 CRYPTO_WRITE(dev, RK_CRYPTO_CONF, RK_CRYPTO_BYTESWAP_HRFIFO |
114 RK_CRYPTO_BYTESWAP_BRFIFO |
115 RK_CRYPTO_BYTESWAP_BTFIFO);
116
117 CRYPTO_WRITE(dev, RK_CRYPTO_HASH_MSG_LEN, req->nbytes);
118 }
119
rk_ahash_init(struct ahash_request * req)120 static int rk_ahash_init(struct ahash_request *req)
121 {
122 struct rk_ahash_rctx *rctx = ahash_request_ctx(req);
123 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
124 struct rk_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
125
126 ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm);
127 rctx->fallback_req.base.flags = req->base.flags &
128 CRYPTO_TFM_REQ_MAY_SLEEP;
129
130 return crypto_ahash_init(&rctx->fallback_req);
131 }
132
rk_ahash_update(struct ahash_request * req)133 static int rk_ahash_update(struct ahash_request *req)
134 {
135 struct rk_ahash_rctx *rctx = ahash_request_ctx(req);
136 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
137 struct rk_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
138
139 ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm);
140 rctx->fallback_req.base.flags = req->base.flags &
141 CRYPTO_TFM_REQ_MAY_SLEEP;
142 rctx->fallback_req.nbytes = req->nbytes;
143 rctx->fallback_req.src = req->src;
144
145 return crypto_ahash_update(&rctx->fallback_req);
146 }
147
rk_ahash_final(struct ahash_request * req)148 static int rk_ahash_final(struct ahash_request *req)
149 {
150 struct rk_ahash_rctx *rctx = ahash_request_ctx(req);
151 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
152 struct rk_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
153
154 ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm);
155 rctx->fallback_req.base.flags = req->base.flags &
156 CRYPTO_TFM_REQ_MAY_SLEEP;
157 rctx->fallback_req.result = req->result;
158
159 return crypto_ahash_final(&rctx->fallback_req);
160 }
161
rk_ahash_finup(struct ahash_request * req)162 static int rk_ahash_finup(struct ahash_request *req)
163 {
164 struct rk_ahash_rctx *rctx = ahash_request_ctx(req);
165 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
166 struct rk_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
167
168 ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm);
169 rctx->fallback_req.base.flags = req->base.flags &
170 CRYPTO_TFM_REQ_MAY_SLEEP;
171
172 rctx->fallback_req.nbytes = req->nbytes;
173 rctx->fallback_req.src = req->src;
174 rctx->fallback_req.result = req->result;
175
176 return crypto_ahash_finup(&rctx->fallback_req);
177 }
178
rk_ahash_import(struct ahash_request * req,const void * in)179 static int rk_ahash_import(struct ahash_request *req, const void *in)
180 {
181 struct rk_ahash_rctx *rctx = ahash_request_ctx(req);
182 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
183 struct rk_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
184
185 ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm);
186 rctx->fallback_req.base.flags = req->base.flags &
187 CRYPTO_TFM_REQ_MAY_SLEEP;
188
189 return crypto_ahash_import(&rctx->fallback_req, in);
190 }
191
rk_ahash_export(struct ahash_request * req,void * out)192 static int rk_ahash_export(struct ahash_request *req, void *out)
193 {
194 struct rk_ahash_rctx *rctx = ahash_request_ctx(req);
195 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
196 struct rk_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
197
198 ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm);
199 rctx->fallback_req.base.flags = req->base.flags &
200 CRYPTO_TFM_REQ_MAY_SLEEP;
201
202 return crypto_ahash_export(&rctx->fallback_req, out);
203 }
204
rk_ahash_digest(struct ahash_request * req)205 static int rk_ahash_digest(struct ahash_request *req)
206 {
207 struct rk_ahash_rctx *rctx = ahash_request_ctx(req);
208 struct rk_crypto_info *dev;
209 struct crypto_engine *engine;
210
211 if (rk_ahash_need_fallback(req))
212 return rk_ahash_digest_fb(req);
213
214 if (!req->nbytes)
215 return zero_message_process(req);
216
217 dev = get_rk_crypto();
218
219 rctx->dev = dev;
220 engine = dev->engine;
221
222 return crypto_transfer_hash_request_to_engine(engine, req);
223 }
224
crypto_ahash_dma_start(struct rk_crypto_info * dev,struct scatterlist * sg)225 static void crypto_ahash_dma_start(struct rk_crypto_info *dev, struct scatterlist *sg)
226 {
227 CRYPTO_WRITE(dev, RK_CRYPTO_HRDMAS, sg_dma_address(sg));
228 CRYPTO_WRITE(dev, RK_CRYPTO_HRDMAL, sg_dma_len(sg) / 4);
229 CRYPTO_WRITE(dev, RK_CRYPTO_CTRL, RK_CRYPTO_HASH_START |
230 (RK_CRYPTO_HASH_START << 16));
231 }
232
rk_hash_prepare(struct crypto_engine * engine,void * breq)233 static int rk_hash_prepare(struct crypto_engine *engine, void *breq)
234 {
235 struct ahash_request *areq = container_of(breq, struct ahash_request, base);
236 struct rk_ahash_rctx *rctx = ahash_request_ctx(areq);
237 struct rk_crypto_info *rkc = rctx->dev;
238 int ret;
239
240 ret = dma_map_sg(rkc->dev, areq->src, sg_nents(areq->src), DMA_TO_DEVICE);
241 if (ret <= 0)
242 return -EINVAL;
243
244 rctx->nrsg = ret;
245
246 return 0;
247 }
248
rk_hash_unprepare(struct crypto_engine * engine,void * breq)249 static void rk_hash_unprepare(struct crypto_engine *engine, void *breq)
250 {
251 struct ahash_request *areq = container_of(breq, struct ahash_request, base);
252 struct rk_ahash_rctx *rctx = ahash_request_ctx(areq);
253 struct rk_crypto_info *rkc = rctx->dev;
254
255 dma_unmap_sg(rkc->dev, areq->src, rctx->nrsg, DMA_TO_DEVICE);
256 }
257
rk_hash_run(struct crypto_engine * engine,void * breq)258 static int rk_hash_run(struct crypto_engine *engine, void *breq)
259 {
260 struct ahash_request *areq = container_of(breq, struct ahash_request, base);
261 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
262 struct rk_ahash_rctx *rctx = ahash_request_ctx(areq);
263 struct ahash_alg *alg = crypto_ahash_alg(tfm);
264 struct rk_crypto_tmp *algt = container_of(alg, struct rk_crypto_tmp, alg.hash.base);
265 struct scatterlist *sg = areq->src;
266 struct rk_crypto_info *rkc = rctx->dev;
267 int err;
268 int i;
269 u32 v;
270
271 err = pm_runtime_resume_and_get(rkc->dev);
272 if (err)
273 return err;
274
275 err = rk_hash_prepare(engine, breq);
276 if (err)
277 goto theend;
278
279 rctx->mode = 0;
280
281 algt->stat_req++;
282 rkc->nreq++;
283
284 switch (crypto_ahash_digestsize(tfm)) {
285 case SHA1_DIGEST_SIZE:
286 rctx->mode = RK_CRYPTO_HASH_SHA1;
287 break;
288 case SHA256_DIGEST_SIZE:
289 rctx->mode = RK_CRYPTO_HASH_SHA256;
290 break;
291 case MD5_DIGEST_SIZE:
292 rctx->mode = RK_CRYPTO_HASH_MD5;
293 break;
294 default:
295 err = -EINVAL;
296 goto theend;
297 }
298
299 rk_ahash_reg_init(areq, rkc);
300
301 while (sg) {
302 reinit_completion(&rkc->complete);
303 rkc->status = 0;
304 crypto_ahash_dma_start(rkc, sg);
305 wait_for_completion_interruptible_timeout(&rkc->complete,
306 msecs_to_jiffies(2000));
307 if (!rkc->status) {
308 dev_err(rkc->dev, "DMA timeout\n");
309 err = -EFAULT;
310 goto theend;
311 }
312 sg = sg_next(sg);
313 }
314
315 /*
316 * it will take some time to process date after last dma
317 * transmission.
318 *
319 * waiting time is relative with the last date len,
320 * so cannot set a fixed time here.
321 * 10us makes system not call here frequently wasting
322 * efficiency, and make it response quickly when dma
323 * complete.
324 */
325 readl_poll_timeout(rkc->reg + RK_CRYPTO_HASH_STS, v, v == 0, 10, 1000);
326
327 for (i = 0; i < crypto_ahash_digestsize(tfm) / 4; i++) {
328 v = readl(rkc->reg + RK_CRYPTO_HASH_DOUT_0 + i * 4);
329 put_unaligned_le32(v, areq->result + i * 4);
330 }
331
332 theend:
333 pm_runtime_put_autosuspend(rkc->dev);
334
335 rk_hash_unprepare(engine, breq);
336
337 local_bh_disable();
338 crypto_finalize_hash_request(engine, breq, err);
339 local_bh_enable();
340
341 return 0;
342 }
343
rk_hash_init_tfm(struct crypto_ahash * tfm)344 static int rk_hash_init_tfm(struct crypto_ahash *tfm)
345 {
346 struct rk_ahash_ctx *tctx = crypto_ahash_ctx(tfm);
347 const char *alg_name = crypto_ahash_alg_name(tfm);
348 struct ahash_alg *alg = crypto_ahash_alg(tfm);
349 struct rk_crypto_tmp *algt = container_of(alg, struct rk_crypto_tmp, alg.hash.base);
350
351 /* for fallback */
352 tctx->fallback_tfm = crypto_alloc_ahash(alg_name, 0,
353 CRYPTO_ALG_NEED_FALLBACK);
354 if (IS_ERR(tctx->fallback_tfm)) {
355 dev_err(algt->dev->dev, "Could not load fallback driver.\n");
356 return PTR_ERR(tctx->fallback_tfm);
357 }
358
359 crypto_ahash_set_reqsize(tfm,
360 sizeof(struct rk_ahash_rctx) +
361 crypto_ahash_reqsize(tctx->fallback_tfm));
362
363 return 0;
364 }
365
rk_hash_exit_tfm(struct crypto_ahash * tfm)366 static void rk_hash_exit_tfm(struct crypto_ahash *tfm)
367 {
368 struct rk_ahash_ctx *tctx = crypto_ahash_ctx(tfm);
369
370 crypto_free_ahash(tctx->fallback_tfm);
371 }
372
373 struct rk_crypto_tmp rk_ahash_sha1 = {
374 .type = CRYPTO_ALG_TYPE_AHASH,
375 .alg.hash.base = {
376 .init = rk_ahash_init,
377 .update = rk_ahash_update,
378 .final = rk_ahash_final,
379 .finup = rk_ahash_finup,
380 .export = rk_ahash_export,
381 .import = rk_ahash_import,
382 .digest = rk_ahash_digest,
383 .init_tfm = rk_hash_init_tfm,
384 .exit_tfm = rk_hash_exit_tfm,
385 .halg = {
386 .digestsize = SHA1_DIGEST_SIZE,
387 .statesize = sizeof(struct sha1_state),
388 .base = {
389 .cra_name = "sha1",
390 .cra_driver_name = "rk-sha1",
391 .cra_priority = 300,
392 .cra_flags = CRYPTO_ALG_ASYNC |
393 CRYPTO_ALG_NEED_FALLBACK,
394 .cra_blocksize = SHA1_BLOCK_SIZE,
395 .cra_ctxsize = sizeof(struct rk_ahash_ctx),
396 .cra_alignmask = 3,
397 .cra_module = THIS_MODULE,
398 }
399 }
400 },
401 .alg.hash.op = {
402 .do_one_request = rk_hash_run,
403 },
404 };
405
406 struct rk_crypto_tmp rk_ahash_sha256 = {
407 .type = CRYPTO_ALG_TYPE_AHASH,
408 .alg.hash.base = {
409 .init = rk_ahash_init,
410 .update = rk_ahash_update,
411 .final = rk_ahash_final,
412 .finup = rk_ahash_finup,
413 .export = rk_ahash_export,
414 .import = rk_ahash_import,
415 .digest = rk_ahash_digest,
416 .init_tfm = rk_hash_init_tfm,
417 .exit_tfm = rk_hash_exit_tfm,
418 .halg = {
419 .digestsize = SHA256_DIGEST_SIZE,
420 .statesize = sizeof(struct sha256_state),
421 .base = {
422 .cra_name = "sha256",
423 .cra_driver_name = "rk-sha256",
424 .cra_priority = 300,
425 .cra_flags = CRYPTO_ALG_ASYNC |
426 CRYPTO_ALG_NEED_FALLBACK,
427 .cra_blocksize = SHA256_BLOCK_SIZE,
428 .cra_ctxsize = sizeof(struct rk_ahash_ctx),
429 .cra_alignmask = 3,
430 .cra_module = THIS_MODULE,
431 }
432 }
433 },
434 .alg.hash.op = {
435 .do_one_request = rk_hash_run,
436 },
437 };
438
439 struct rk_crypto_tmp rk_ahash_md5 = {
440 .type = CRYPTO_ALG_TYPE_AHASH,
441 .alg.hash.base = {
442 .init = rk_ahash_init,
443 .update = rk_ahash_update,
444 .final = rk_ahash_final,
445 .finup = rk_ahash_finup,
446 .export = rk_ahash_export,
447 .import = rk_ahash_import,
448 .digest = rk_ahash_digest,
449 .init_tfm = rk_hash_init_tfm,
450 .exit_tfm = rk_hash_exit_tfm,
451 .halg = {
452 .digestsize = MD5_DIGEST_SIZE,
453 .statesize = sizeof(struct md5_state),
454 .base = {
455 .cra_name = "md5",
456 .cra_driver_name = "rk-md5",
457 .cra_priority = 300,
458 .cra_flags = CRYPTO_ALG_ASYNC |
459 CRYPTO_ALG_NEED_FALLBACK,
460 .cra_blocksize = SHA1_BLOCK_SIZE,
461 .cra_ctxsize = sizeof(struct rk_ahash_ctx),
462 .cra_alignmask = 3,
463 .cra_module = THIS_MODULE,
464 }
465 }
466 },
467 .alg.hash.op = {
468 .do_one_request = rk_hash_run,
469 },
470 };
471