1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * AMD Cryptographic Coprocessor (CCP) driver
4 *
5 * Copyright (C) 2013-2019 Advanced Micro Devices, Inc.
6 *
7 * Author: Tom Lendacky <thomas.lendacky@amd.com>
8 * Author: Gary R Hook <gary.hook@amd.com>
9 */
10
11 #include <linux/dma-mapping.h>
12 #include <linux/module.h>
13 #include <linux/kernel.h>
14 #include <linux/interrupt.h>
15 #include <crypto/scatterwalk.h>
16 #include <crypto/des.h>
17 #include <linux/ccp.h>
18
19 #include "ccp-dev.h"
20
21 /* SHA initial context values */
22 static const __be32 ccp_sha1_init[SHA1_DIGEST_SIZE / sizeof(__be32)] = {
23 cpu_to_be32(SHA1_H0), cpu_to_be32(SHA1_H1),
24 cpu_to_be32(SHA1_H2), cpu_to_be32(SHA1_H3),
25 cpu_to_be32(SHA1_H4),
26 };
27
28 static const __be32 ccp_sha224_init[SHA256_DIGEST_SIZE / sizeof(__be32)] = {
29 cpu_to_be32(SHA224_H0), cpu_to_be32(SHA224_H1),
30 cpu_to_be32(SHA224_H2), cpu_to_be32(SHA224_H3),
31 cpu_to_be32(SHA224_H4), cpu_to_be32(SHA224_H5),
32 cpu_to_be32(SHA224_H6), cpu_to_be32(SHA224_H7),
33 };
34
35 static const __be32 ccp_sha256_init[SHA256_DIGEST_SIZE / sizeof(__be32)] = {
36 cpu_to_be32(SHA256_H0), cpu_to_be32(SHA256_H1),
37 cpu_to_be32(SHA256_H2), cpu_to_be32(SHA256_H3),
38 cpu_to_be32(SHA256_H4), cpu_to_be32(SHA256_H5),
39 cpu_to_be32(SHA256_H6), cpu_to_be32(SHA256_H7),
40 };
41
42 static const __be64 ccp_sha384_init[SHA512_DIGEST_SIZE / sizeof(__be64)] = {
43 cpu_to_be64(SHA384_H0), cpu_to_be64(SHA384_H1),
44 cpu_to_be64(SHA384_H2), cpu_to_be64(SHA384_H3),
45 cpu_to_be64(SHA384_H4), cpu_to_be64(SHA384_H5),
46 cpu_to_be64(SHA384_H6), cpu_to_be64(SHA384_H7),
47 };
48
49 static const __be64 ccp_sha512_init[SHA512_DIGEST_SIZE / sizeof(__be64)] = {
50 cpu_to_be64(SHA512_H0), cpu_to_be64(SHA512_H1),
51 cpu_to_be64(SHA512_H2), cpu_to_be64(SHA512_H3),
52 cpu_to_be64(SHA512_H4), cpu_to_be64(SHA512_H5),
53 cpu_to_be64(SHA512_H6), cpu_to_be64(SHA512_H7),
54 };
55
56 #define CCP_NEW_JOBID(ccp) ((ccp->vdata->version == CCP_VERSION(3, 0)) ? \
57 ccp_gen_jobid(ccp) : 0)
58
ccp_gen_jobid(struct ccp_device * ccp)59 static u32 ccp_gen_jobid(struct ccp_device *ccp)
60 {
61 return atomic_inc_return(&ccp->current_id) & CCP_JOBID_MASK;
62 }
63
ccp_sg_free(struct ccp_sg_workarea * wa)64 static void ccp_sg_free(struct ccp_sg_workarea *wa)
65 {
66 if (wa->dma_count)
67 dma_unmap_sg(wa->dma_dev, wa->dma_sg_head, wa->nents, wa->dma_dir);
68
69 wa->dma_count = 0;
70 }
71
ccp_init_sg_workarea(struct ccp_sg_workarea * wa,struct device * dev,struct scatterlist * sg,u64 len,enum dma_data_direction dma_dir)72 static int ccp_init_sg_workarea(struct ccp_sg_workarea *wa, struct device *dev,
73 struct scatterlist *sg, u64 len,
74 enum dma_data_direction dma_dir)
75 {
76 memset(wa, 0, sizeof(*wa));
77
78 wa->sg = sg;
79 if (!sg)
80 return 0;
81
82 wa->nents = sg_nents_for_len(sg, len);
83 if (wa->nents < 0)
84 return wa->nents;
85
86 wa->bytes_left = len;
87 wa->sg_used = 0;
88
89 if (len == 0)
90 return 0;
91
92 if (dma_dir == DMA_NONE)
93 return 0;
94
95 wa->dma_sg = sg;
96 wa->dma_sg_head = sg;
97 wa->dma_dev = dev;
98 wa->dma_dir = dma_dir;
99 wa->dma_count = dma_map_sg(dev, sg, wa->nents, dma_dir);
100 if (!wa->dma_count)
101 return -ENOMEM;
102
103 return 0;
104 }
105
ccp_update_sg_workarea(struct ccp_sg_workarea * wa,unsigned int len)106 static void ccp_update_sg_workarea(struct ccp_sg_workarea *wa, unsigned int len)
107 {
108 unsigned int nbytes = min_t(u64, len, wa->bytes_left);
109 unsigned int sg_combined_len = 0;
110
111 if (!wa->sg)
112 return;
113
114 wa->sg_used += nbytes;
115 wa->bytes_left -= nbytes;
116 if (wa->sg_used == sg_dma_len(wa->dma_sg)) {
117 /* Advance to the next DMA scatterlist entry */
118 wa->dma_sg = sg_next(wa->dma_sg);
119
120 /* In the case that the DMA mapped scatterlist has entries
121 * that have been merged, the non-DMA mapped scatterlist
122 * must be advanced multiple times for each merged entry.
123 * This ensures that the current non-DMA mapped entry
124 * corresponds to the current DMA mapped entry.
125 */
126 do {
127 sg_combined_len += wa->sg->length;
128 wa->sg = sg_next(wa->sg);
129 } while (wa->sg_used > sg_combined_len);
130
131 wa->sg_used = 0;
132 }
133 }
134
ccp_dm_free(struct ccp_dm_workarea * wa)135 static void ccp_dm_free(struct ccp_dm_workarea *wa)
136 {
137 if (wa->length <= CCP_DMAPOOL_MAX_SIZE) {
138 if (wa->address)
139 dma_pool_free(wa->dma_pool, wa->address,
140 wa->dma.address);
141 } else {
142 if (wa->dma.address)
143 dma_unmap_single(wa->dev, wa->dma.address, wa->length,
144 wa->dma.dir);
145 kfree(wa->address);
146 }
147
148 wa->address = NULL;
149 wa->dma.address = 0;
150 }
151
ccp_init_dm_workarea(struct ccp_dm_workarea * wa,struct ccp_cmd_queue * cmd_q,unsigned int len,enum dma_data_direction dir)152 static int ccp_init_dm_workarea(struct ccp_dm_workarea *wa,
153 struct ccp_cmd_queue *cmd_q,
154 unsigned int len,
155 enum dma_data_direction dir)
156 {
157 memset(wa, 0, sizeof(*wa));
158
159 if (!len)
160 return 0;
161
162 wa->dev = cmd_q->ccp->dev;
163 wa->length = len;
164
165 if (len <= CCP_DMAPOOL_MAX_SIZE) {
166 wa->dma_pool = cmd_q->dma_pool;
167
168 wa->address = dma_pool_zalloc(wa->dma_pool, GFP_KERNEL,
169 &wa->dma.address);
170 if (!wa->address)
171 return -ENOMEM;
172
173 wa->dma.length = CCP_DMAPOOL_MAX_SIZE;
174
175 } else {
176 wa->address = kzalloc(len, GFP_KERNEL);
177 if (!wa->address)
178 return -ENOMEM;
179
180 wa->dma.address = dma_map_single(wa->dev, wa->address, len,
181 dir);
182 if (dma_mapping_error(wa->dev, wa->dma.address)) {
183 kfree(wa->address);
184 wa->address = NULL;
185 return -ENOMEM;
186 }
187
188 wa->dma.length = len;
189 }
190 wa->dma.dir = dir;
191
192 return 0;
193 }
194
ccp_set_dm_area(struct ccp_dm_workarea * wa,unsigned int wa_offset,struct scatterlist * sg,unsigned int sg_offset,unsigned int len)195 static int ccp_set_dm_area(struct ccp_dm_workarea *wa, unsigned int wa_offset,
196 struct scatterlist *sg, unsigned int sg_offset,
197 unsigned int len)
198 {
199 WARN_ON(!wa->address);
200
201 if (len > (wa->length - wa_offset))
202 return -EINVAL;
203
204 scatterwalk_map_and_copy(wa->address + wa_offset, sg, sg_offset, len,
205 0);
206 return 0;
207 }
208
ccp_get_dm_area(struct ccp_dm_workarea * wa,unsigned int wa_offset,struct scatterlist * sg,unsigned int sg_offset,unsigned int len)209 static void ccp_get_dm_area(struct ccp_dm_workarea *wa, unsigned int wa_offset,
210 struct scatterlist *sg, unsigned int sg_offset,
211 unsigned int len)
212 {
213 WARN_ON(!wa->address);
214
215 scatterwalk_map_and_copy(wa->address + wa_offset, sg, sg_offset, len,
216 1);
217 }
218
ccp_reverse_set_dm_area(struct ccp_dm_workarea * wa,unsigned int wa_offset,struct scatterlist * sg,unsigned int sg_offset,unsigned int len)219 static int ccp_reverse_set_dm_area(struct ccp_dm_workarea *wa,
220 unsigned int wa_offset,
221 struct scatterlist *sg,
222 unsigned int sg_offset,
223 unsigned int len)
224 {
225 u8 *p, *q;
226 int rc;
227
228 rc = ccp_set_dm_area(wa, wa_offset, sg, sg_offset, len);
229 if (rc)
230 return rc;
231
232 p = wa->address + wa_offset;
233 q = p + len - 1;
234 while (p < q) {
235 *p = *p ^ *q;
236 *q = *p ^ *q;
237 *p = *p ^ *q;
238 p++;
239 q--;
240 }
241 return 0;
242 }
243
ccp_reverse_get_dm_area(struct ccp_dm_workarea * wa,unsigned int wa_offset,struct scatterlist * sg,unsigned int sg_offset,unsigned int len)244 static void ccp_reverse_get_dm_area(struct ccp_dm_workarea *wa,
245 unsigned int wa_offset,
246 struct scatterlist *sg,
247 unsigned int sg_offset,
248 unsigned int len)
249 {
250 u8 *p, *q;
251
252 p = wa->address + wa_offset;
253 q = p + len - 1;
254 while (p < q) {
255 *p = *p ^ *q;
256 *q = *p ^ *q;
257 *p = *p ^ *q;
258 p++;
259 q--;
260 }
261
262 ccp_get_dm_area(wa, wa_offset, sg, sg_offset, len);
263 }
264
ccp_free_data(struct ccp_data * data,struct ccp_cmd_queue * cmd_q)265 static void ccp_free_data(struct ccp_data *data, struct ccp_cmd_queue *cmd_q)
266 {
267 ccp_dm_free(&data->dm_wa);
268 ccp_sg_free(&data->sg_wa);
269 }
270
ccp_init_data(struct ccp_data * data,struct ccp_cmd_queue * cmd_q,struct scatterlist * sg,u64 sg_len,unsigned int dm_len,enum dma_data_direction dir)271 static int ccp_init_data(struct ccp_data *data, struct ccp_cmd_queue *cmd_q,
272 struct scatterlist *sg, u64 sg_len,
273 unsigned int dm_len,
274 enum dma_data_direction dir)
275 {
276 int ret;
277
278 memset(data, 0, sizeof(*data));
279
280 ret = ccp_init_sg_workarea(&data->sg_wa, cmd_q->ccp->dev, sg, sg_len,
281 dir);
282 if (ret)
283 goto e_err;
284
285 ret = ccp_init_dm_workarea(&data->dm_wa, cmd_q, dm_len, dir);
286 if (ret)
287 goto e_err;
288
289 return 0;
290
291 e_err:
292 ccp_free_data(data, cmd_q);
293
294 return ret;
295 }
296
ccp_queue_buf(struct ccp_data * data,unsigned int from)297 static unsigned int ccp_queue_buf(struct ccp_data *data, unsigned int from)
298 {
299 struct ccp_sg_workarea *sg_wa = &data->sg_wa;
300 struct ccp_dm_workarea *dm_wa = &data->dm_wa;
301 unsigned int buf_count, nbytes;
302
303 /* Clear the buffer if setting it */
304 if (!from)
305 memset(dm_wa->address, 0, dm_wa->length);
306
307 if (!sg_wa->sg)
308 return 0;
309
310 /* Perform the copy operation
311 * nbytes will always be <= UINT_MAX because dm_wa->length is
312 * an unsigned int
313 */
314 nbytes = min_t(u64, sg_wa->bytes_left, dm_wa->length);
315 scatterwalk_map_and_copy(dm_wa->address, sg_wa->sg, sg_wa->sg_used,
316 nbytes, from);
317
318 /* Update the structures and generate the count */
319 buf_count = 0;
320 while (sg_wa->bytes_left && (buf_count < dm_wa->length)) {
321 nbytes = min(sg_dma_len(sg_wa->dma_sg) - sg_wa->sg_used,
322 dm_wa->length - buf_count);
323 nbytes = min_t(u64, sg_wa->bytes_left, nbytes);
324
325 buf_count += nbytes;
326 ccp_update_sg_workarea(sg_wa, nbytes);
327 }
328
329 return buf_count;
330 }
331
ccp_fill_queue_buf(struct ccp_data * data)332 static unsigned int ccp_fill_queue_buf(struct ccp_data *data)
333 {
334 return ccp_queue_buf(data, 0);
335 }
336
ccp_empty_queue_buf(struct ccp_data * data)337 static unsigned int ccp_empty_queue_buf(struct ccp_data *data)
338 {
339 return ccp_queue_buf(data, 1);
340 }
341
ccp_prepare_data(struct ccp_data * src,struct ccp_data * dst,struct ccp_op * op,unsigned int block_size,bool blocksize_op)342 static void ccp_prepare_data(struct ccp_data *src, struct ccp_data *dst,
343 struct ccp_op *op, unsigned int block_size,
344 bool blocksize_op)
345 {
346 unsigned int sg_src_len, sg_dst_len, op_len;
347
348 /* The CCP can only DMA from/to one address each per operation. This
349 * requires that we find the smallest DMA area between the source
350 * and destination. The resulting len values will always be <= UINT_MAX
351 * because the dma length is an unsigned int.
352 */
353 sg_src_len = sg_dma_len(src->sg_wa.dma_sg) - src->sg_wa.sg_used;
354 sg_src_len = min_t(u64, src->sg_wa.bytes_left, sg_src_len);
355
356 if (dst) {
357 sg_dst_len = sg_dma_len(dst->sg_wa.dma_sg) - dst->sg_wa.sg_used;
358 sg_dst_len = min_t(u64, src->sg_wa.bytes_left, sg_dst_len);
359 op_len = min(sg_src_len, sg_dst_len);
360 } else {
361 op_len = sg_src_len;
362 }
363
364 /* The data operation length will be at least block_size in length
365 * or the smaller of available sg room remaining for the source or
366 * the destination
367 */
368 op_len = max(op_len, block_size);
369
370 /* Unless we have to buffer data, there's no reason to wait */
371 op->soc = 0;
372
373 if (sg_src_len < block_size) {
374 /* Not enough data in the sg element, so it
375 * needs to be buffered into a blocksize chunk
376 */
377 int cp_len = ccp_fill_queue_buf(src);
378
379 op->soc = 1;
380 op->src.u.dma.address = src->dm_wa.dma.address;
381 op->src.u.dma.offset = 0;
382 op->src.u.dma.length = (blocksize_op) ? block_size : cp_len;
383 } else {
384 /* Enough data in the sg element, but we need to
385 * adjust for any previously copied data
386 */
387 op->src.u.dma.address = sg_dma_address(src->sg_wa.dma_sg);
388 op->src.u.dma.offset = src->sg_wa.sg_used;
389 op->src.u.dma.length = op_len & ~(block_size - 1);
390
391 ccp_update_sg_workarea(&src->sg_wa, op->src.u.dma.length);
392 }
393
394 if (dst) {
395 if (sg_dst_len < block_size) {
396 /* Not enough room in the sg element or we're on the
397 * last piece of data (when using padding), so the
398 * output needs to be buffered into a blocksize chunk
399 */
400 op->soc = 1;
401 op->dst.u.dma.address = dst->dm_wa.dma.address;
402 op->dst.u.dma.offset = 0;
403 op->dst.u.dma.length = op->src.u.dma.length;
404 } else {
405 /* Enough room in the sg element, but we need to
406 * adjust for any previously used area
407 */
408 op->dst.u.dma.address = sg_dma_address(dst->sg_wa.dma_sg);
409 op->dst.u.dma.offset = dst->sg_wa.sg_used;
410 op->dst.u.dma.length = op->src.u.dma.length;
411 }
412 }
413 }
414
ccp_process_data(struct ccp_data * src,struct ccp_data * dst,struct ccp_op * op)415 static void ccp_process_data(struct ccp_data *src, struct ccp_data *dst,
416 struct ccp_op *op)
417 {
418 op->init = 0;
419
420 if (dst) {
421 if (op->dst.u.dma.address == dst->dm_wa.dma.address)
422 ccp_empty_queue_buf(dst);
423 else
424 ccp_update_sg_workarea(&dst->sg_wa,
425 op->dst.u.dma.length);
426 }
427 }
428
ccp_copy_to_from_sb(struct ccp_cmd_queue * cmd_q,struct ccp_dm_workarea * wa,u32 jobid,u32 sb,u32 byte_swap,bool from)429 static int ccp_copy_to_from_sb(struct ccp_cmd_queue *cmd_q,
430 struct ccp_dm_workarea *wa, u32 jobid, u32 sb,
431 u32 byte_swap, bool from)
432 {
433 struct ccp_op op;
434
435 memset(&op, 0, sizeof(op));
436
437 op.cmd_q = cmd_q;
438 op.jobid = jobid;
439 op.eom = 1;
440
441 if (from) {
442 op.soc = 1;
443 op.src.type = CCP_MEMTYPE_SB;
444 op.src.u.sb = sb;
445 op.dst.type = CCP_MEMTYPE_SYSTEM;
446 op.dst.u.dma.address = wa->dma.address;
447 op.dst.u.dma.length = wa->length;
448 } else {
449 op.src.type = CCP_MEMTYPE_SYSTEM;
450 op.src.u.dma.address = wa->dma.address;
451 op.src.u.dma.length = wa->length;
452 op.dst.type = CCP_MEMTYPE_SB;
453 op.dst.u.sb = sb;
454 }
455
456 op.u.passthru.byte_swap = byte_swap;
457
458 return cmd_q->ccp->vdata->perform->passthru(&op);
459 }
460
ccp_copy_to_sb(struct ccp_cmd_queue * cmd_q,struct ccp_dm_workarea * wa,u32 jobid,u32 sb,u32 byte_swap)461 static int ccp_copy_to_sb(struct ccp_cmd_queue *cmd_q,
462 struct ccp_dm_workarea *wa, u32 jobid, u32 sb,
463 u32 byte_swap)
464 {
465 return ccp_copy_to_from_sb(cmd_q, wa, jobid, sb, byte_swap, false);
466 }
467
ccp_copy_from_sb(struct ccp_cmd_queue * cmd_q,struct ccp_dm_workarea * wa,u32 jobid,u32 sb,u32 byte_swap)468 static int ccp_copy_from_sb(struct ccp_cmd_queue *cmd_q,
469 struct ccp_dm_workarea *wa, u32 jobid, u32 sb,
470 u32 byte_swap)
471 {
472 return ccp_copy_to_from_sb(cmd_q, wa, jobid, sb, byte_swap, true);
473 }
474
475 static noinline_for_stack int
ccp_run_aes_cmac_cmd(struct ccp_cmd_queue * cmd_q,struct ccp_cmd * cmd)476 ccp_run_aes_cmac_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
477 {
478 struct ccp_aes_engine *aes = &cmd->u.aes;
479 struct ccp_dm_workarea key, ctx;
480 struct ccp_data src;
481 struct ccp_op op;
482 unsigned int dm_offset;
483 int ret;
484
485 if (!((aes->key_len == AES_KEYSIZE_128) ||
486 (aes->key_len == AES_KEYSIZE_192) ||
487 (aes->key_len == AES_KEYSIZE_256)))
488 return -EINVAL;
489
490 if (aes->src_len & (AES_BLOCK_SIZE - 1))
491 return -EINVAL;
492
493 if (aes->iv_len != AES_BLOCK_SIZE)
494 return -EINVAL;
495
496 if (!aes->key || !aes->iv || !aes->src)
497 return -EINVAL;
498
499 if (aes->cmac_final) {
500 if (aes->cmac_key_len != AES_BLOCK_SIZE)
501 return -EINVAL;
502
503 if (!aes->cmac_key)
504 return -EINVAL;
505 }
506
507 BUILD_BUG_ON(CCP_AES_KEY_SB_COUNT != 1);
508 BUILD_BUG_ON(CCP_AES_CTX_SB_COUNT != 1);
509
510 ret = -EIO;
511 memset(&op, 0, sizeof(op));
512 op.cmd_q = cmd_q;
513 op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
514 op.sb_key = cmd_q->sb_key;
515 op.sb_ctx = cmd_q->sb_ctx;
516 op.init = 1;
517 op.u.aes.type = aes->type;
518 op.u.aes.mode = aes->mode;
519 op.u.aes.action = aes->action;
520
521 /* All supported key sizes fit in a single (32-byte) SB entry
522 * and must be in little endian format. Use the 256-bit byte
523 * swap passthru option to convert from big endian to little
524 * endian.
525 */
526 ret = ccp_init_dm_workarea(&key, cmd_q,
527 CCP_AES_KEY_SB_COUNT * CCP_SB_BYTES,
528 DMA_TO_DEVICE);
529 if (ret)
530 return ret;
531
532 dm_offset = CCP_SB_BYTES - aes->key_len;
533 ret = ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len);
534 if (ret)
535 goto e_key;
536 ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
537 CCP_PASSTHRU_BYTESWAP_256BIT);
538 if (ret) {
539 cmd->engine_error = cmd_q->cmd_error;
540 goto e_key;
541 }
542
543 /* The AES context fits in a single (32-byte) SB entry and
544 * must be in little endian format. Use the 256-bit byte swap
545 * passthru option to convert from big endian to little endian.
546 */
547 ret = ccp_init_dm_workarea(&ctx, cmd_q,
548 CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES,
549 DMA_BIDIRECTIONAL);
550 if (ret)
551 goto e_key;
552
553 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
554 ret = ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
555 if (ret)
556 goto e_ctx;
557 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
558 CCP_PASSTHRU_BYTESWAP_256BIT);
559 if (ret) {
560 cmd->engine_error = cmd_q->cmd_error;
561 goto e_ctx;
562 }
563
564 /* Send data to the CCP AES engine */
565 ret = ccp_init_data(&src, cmd_q, aes->src, aes->src_len,
566 AES_BLOCK_SIZE, DMA_TO_DEVICE);
567 if (ret)
568 goto e_ctx;
569
570 while (src.sg_wa.bytes_left) {
571 ccp_prepare_data(&src, NULL, &op, AES_BLOCK_SIZE, true);
572 if (aes->cmac_final && !src.sg_wa.bytes_left) {
573 op.eom = 1;
574
575 /* Push the K1/K2 key to the CCP now */
576 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid,
577 op.sb_ctx,
578 CCP_PASSTHRU_BYTESWAP_256BIT);
579 if (ret) {
580 cmd->engine_error = cmd_q->cmd_error;
581 goto e_src;
582 }
583
584 ret = ccp_set_dm_area(&ctx, 0, aes->cmac_key, 0,
585 aes->cmac_key_len);
586 if (ret)
587 goto e_src;
588 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
589 CCP_PASSTHRU_BYTESWAP_256BIT);
590 if (ret) {
591 cmd->engine_error = cmd_q->cmd_error;
592 goto e_src;
593 }
594 }
595
596 ret = cmd_q->ccp->vdata->perform->aes(&op);
597 if (ret) {
598 cmd->engine_error = cmd_q->cmd_error;
599 goto e_src;
600 }
601
602 ccp_process_data(&src, NULL, &op);
603 }
604
605 /* Retrieve the AES context - convert from LE to BE using
606 * 32-byte (256-bit) byteswapping
607 */
608 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
609 CCP_PASSTHRU_BYTESWAP_256BIT);
610 if (ret) {
611 cmd->engine_error = cmd_q->cmd_error;
612 goto e_src;
613 }
614
615 /* ...but we only need AES_BLOCK_SIZE bytes */
616 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
617 ccp_get_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
618
619 e_src:
620 ccp_free_data(&src, cmd_q);
621
622 e_ctx:
623 ccp_dm_free(&ctx);
624
625 e_key:
626 ccp_dm_free(&key);
627
628 return ret;
629 }
630
631 static noinline_for_stack int
ccp_run_aes_gcm_cmd(struct ccp_cmd_queue * cmd_q,struct ccp_cmd * cmd)632 ccp_run_aes_gcm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
633 {
634 struct ccp_aes_engine *aes = &cmd->u.aes;
635 struct ccp_dm_workarea key, ctx, final_wa, tag;
636 struct ccp_data src, dst;
637 struct ccp_data aad;
638 struct ccp_op op;
639 unsigned int dm_offset;
640 unsigned int authsize;
641 unsigned int jobid;
642 unsigned int ilen;
643 bool in_place = true; /* Default value */
644 __be64 *final;
645 int ret;
646
647 struct scatterlist *p_inp, sg_inp[2];
648 struct scatterlist *p_tag, sg_tag[2];
649 struct scatterlist *p_outp, sg_outp[2];
650 struct scatterlist *p_aad;
651
652 if (!aes->iv)
653 return -EINVAL;
654
655 if (!((aes->key_len == AES_KEYSIZE_128) ||
656 (aes->key_len == AES_KEYSIZE_192) ||
657 (aes->key_len == AES_KEYSIZE_256)))
658 return -EINVAL;
659
660 if (!aes->key) /* Gotta have a key SGL */
661 return -EINVAL;
662
663 /* Zero defaults to 16 bytes, the maximum size */
664 authsize = aes->authsize ? aes->authsize : AES_BLOCK_SIZE;
665 switch (authsize) {
666 case 16:
667 case 15:
668 case 14:
669 case 13:
670 case 12:
671 case 8:
672 case 4:
673 break;
674 default:
675 return -EINVAL;
676 }
677
678 /* First, decompose the source buffer into AAD & PT,
679 * and the destination buffer into AAD, CT & tag, or
680 * the input into CT & tag.
681 * It is expected that the input and output SGs will
682 * be valid, even if the AAD and input lengths are 0.
683 */
684 p_aad = aes->src;
685 p_inp = scatterwalk_ffwd(sg_inp, aes->src, aes->aad_len);
686 p_outp = scatterwalk_ffwd(sg_outp, aes->dst, aes->aad_len);
687 if (aes->action == CCP_AES_ACTION_ENCRYPT) {
688 ilen = aes->src_len;
689 p_tag = scatterwalk_ffwd(sg_tag, p_outp, ilen);
690 } else {
691 /* Input length for decryption includes tag */
692 ilen = aes->src_len - authsize;
693 p_tag = scatterwalk_ffwd(sg_tag, p_inp, ilen);
694 }
695
696 jobid = CCP_NEW_JOBID(cmd_q->ccp);
697
698 memset(&op, 0, sizeof(op));
699 op.cmd_q = cmd_q;
700 op.jobid = jobid;
701 op.sb_key = cmd_q->sb_key; /* Pre-allocated */
702 op.sb_ctx = cmd_q->sb_ctx; /* Pre-allocated */
703 op.init = 1;
704 op.u.aes.type = aes->type;
705
706 /* Copy the key to the LSB */
707 ret = ccp_init_dm_workarea(&key, cmd_q,
708 CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES,
709 DMA_TO_DEVICE);
710 if (ret)
711 return ret;
712
713 dm_offset = CCP_SB_BYTES - aes->key_len;
714 ret = ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len);
715 if (ret)
716 goto e_key;
717 ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
718 CCP_PASSTHRU_BYTESWAP_256BIT);
719 if (ret) {
720 cmd->engine_error = cmd_q->cmd_error;
721 goto e_key;
722 }
723
724 /* Copy the context (IV) to the LSB.
725 * There is an assumption here that the IV is 96 bits in length, plus
726 * a nonce of 32 bits. If no IV is present, use a zeroed buffer.
727 */
728 ret = ccp_init_dm_workarea(&ctx, cmd_q,
729 CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES,
730 DMA_BIDIRECTIONAL);
731 if (ret)
732 goto e_key;
733
734 dm_offset = CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES - aes->iv_len;
735 ret = ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
736 if (ret)
737 goto e_ctx;
738
739 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
740 CCP_PASSTHRU_BYTESWAP_256BIT);
741 if (ret) {
742 cmd->engine_error = cmd_q->cmd_error;
743 goto e_ctx;
744 }
745
746 op.init = 1;
747 if (aes->aad_len > 0) {
748 /* Step 1: Run a GHASH over the Additional Authenticated Data */
749 ret = ccp_init_data(&aad, cmd_q, p_aad, aes->aad_len,
750 AES_BLOCK_SIZE,
751 DMA_TO_DEVICE);
752 if (ret)
753 goto e_ctx;
754
755 op.u.aes.mode = CCP_AES_MODE_GHASH;
756 op.u.aes.action = CCP_AES_GHASHAAD;
757
758 while (aad.sg_wa.bytes_left) {
759 ccp_prepare_data(&aad, NULL, &op, AES_BLOCK_SIZE, true);
760
761 ret = cmd_q->ccp->vdata->perform->aes(&op);
762 if (ret) {
763 cmd->engine_error = cmd_q->cmd_error;
764 goto e_aad;
765 }
766
767 ccp_process_data(&aad, NULL, &op);
768 op.init = 0;
769 }
770 }
771
772 op.u.aes.mode = CCP_AES_MODE_GCTR;
773 op.u.aes.action = aes->action;
774
775 if (ilen > 0) {
776 /* Step 2: Run a GCTR over the plaintext */
777 in_place = (sg_virt(p_inp) == sg_virt(p_outp)) ? true : false;
778
779 ret = ccp_init_data(&src, cmd_q, p_inp, ilen,
780 AES_BLOCK_SIZE,
781 in_place ? DMA_BIDIRECTIONAL
782 : DMA_TO_DEVICE);
783 if (ret)
784 goto e_aad;
785
786 if (in_place) {
787 dst = src;
788 } else {
789 ret = ccp_init_data(&dst, cmd_q, p_outp, ilen,
790 AES_BLOCK_SIZE, DMA_FROM_DEVICE);
791 if (ret)
792 goto e_src;
793 }
794
795 op.soc = 0;
796 op.eom = 0;
797 op.init = 1;
798 while (src.sg_wa.bytes_left) {
799 ccp_prepare_data(&src, &dst, &op, AES_BLOCK_SIZE, true);
800 if (!src.sg_wa.bytes_left) {
801 unsigned int nbytes = ilen % AES_BLOCK_SIZE;
802
803 if (nbytes) {
804 op.eom = 1;
805 op.u.aes.size = (nbytes * 8) - 1;
806 }
807 }
808
809 ret = cmd_q->ccp->vdata->perform->aes(&op);
810 if (ret) {
811 cmd->engine_error = cmd_q->cmd_error;
812 goto e_dst;
813 }
814
815 ccp_process_data(&src, &dst, &op);
816 op.init = 0;
817 }
818 }
819
820 /* Step 3: Update the IV portion of the context with the original IV */
821 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
822 CCP_PASSTHRU_BYTESWAP_256BIT);
823 if (ret) {
824 cmd->engine_error = cmd_q->cmd_error;
825 goto e_dst;
826 }
827
828 ret = ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
829 if (ret)
830 goto e_dst;
831
832 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
833 CCP_PASSTHRU_BYTESWAP_256BIT);
834 if (ret) {
835 cmd->engine_error = cmd_q->cmd_error;
836 goto e_dst;
837 }
838
839 /* Step 4: Concatenate the lengths of the AAD and source, and
840 * hash that 16 byte buffer.
841 */
842 ret = ccp_init_dm_workarea(&final_wa, cmd_q, AES_BLOCK_SIZE,
843 DMA_BIDIRECTIONAL);
844 if (ret)
845 goto e_dst;
846 final = (__be64 *)final_wa.address;
847 final[0] = cpu_to_be64(aes->aad_len * 8);
848 final[1] = cpu_to_be64(ilen * 8);
849
850 memset(&op, 0, sizeof(op));
851 op.cmd_q = cmd_q;
852 op.jobid = jobid;
853 op.sb_key = cmd_q->sb_key; /* Pre-allocated */
854 op.sb_ctx = cmd_q->sb_ctx; /* Pre-allocated */
855 op.init = 1;
856 op.u.aes.type = aes->type;
857 op.u.aes.mode = CCP_AES_MODE_GHASH;
858 op.u.aes.action = CCP_AES_GHASHFINAL;
859 op.src.type = CCP_MEMTYPE_SYSTEM;
860 op.src.u.dma.address = final_wa.dma.address;
861 op.src.u.dma.length = AES_BLOCK_SIZE;
862 op.dst.type = CCP_MEMTYPE_SYSTEM;
863 op.dst.u.dma.address = final_wa.dma.address;
864 op.dst.u.dma.length = AES_BLOCK_SIZE;
865 op.eom = 1;
866 op.u.aes.size = 0;
867 ret = cmd_q->ccp->vdata->perform->aes(&op);
868 if (ret)
869 goto e_final_wa;
870
871 if (aes->action == CCP_AES_ACTION_ENCRYPT) {
872 /* Put the ciphered tag after the ciphertext. */
873 ccp_get_dm_area(&final_wa, 0, p_tag, 0, authsize);
874 } else {
875 /* Does this ciphered tag match the input? */
876 ret = ccp_init_dm_workarea(&tag, cmd_q, authsize,
877 DMA_BIDIRECTIONAL);
878 if (ret)
879 goto e_final_wa;
880 ret = ccp_set_dm_area(&tag, 0, p_tag, 0, authsize);
881 if (ret) {
882 ccp_dm_free(&tag);
883 goto e_final_wa;
884 }
885
886 ret = crypto_memneq(tag.address, final_wa.address,
887 authsize) ? -EBADMSG : 0;
888 ccp_dm_free(&tag);
889 }
890
891 e_final_wa:
892 ccp_dm_free(&final_wa);
893
894 e_dst:
895 if (ilen > 0 && !in_place)
896 ccp_free_data(&dst, cmd_q);
897
898 e_src:
899 if (ilen > 0)
900 ccp_free_data(&src, cmd_q);
901
902 e_aad:
903 if (aes->aad_len)
904 ccp_free_data(&aad, cmd_q);
905
906 e_ctx:
907 ccp_dm_free(&ctx);
908
909 e_key:
910 ccp_dm_free(&key);
911
912 return ret;
913 }
914
915 static noinline_for_stack int
ccp_run_aes_cmd(struct ccp_cmd_queue * cmd_q,struct ccp_cmd * cmd)916 ccp_run_aes_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
917 {
918 struct ccp_aes_engine *aes = &cmd->u.aes;
919 struct ccp_dm_workarea key, ctx;
920 struct ccp_data src, dst;
921 struct ccp_op op;
922 unsigned int dm_offset;
923 bool in_place = false;
924 int ret;
925
926 if (!((aes->key_len == AES_KEYSIZE_128) ||
927 (aes->key_len == AES_KEYSIZE_192) ||
928 (aes->key_len == AES_KEYSIZE_256)))
929 return -EINVAL;
930
931 if (((aes->mode == CCP_AES_MODE_ECB) ||
932 (aes->mode == CCP_AES_MODE_CBC)) &&
933 (aes->src_len & (AES_BLOCK_SIZE - 1)))
934 return -EINVAL;
935
936 if (!aes->key || !aes->src || !aes->dst)
937 return -EINVAL;
938
939 if (aes->mode != CCP_AES_MODE_ECB) {
940 if (aes->iv_len != AES_BLOCK_SIZE)
941 return -EINVAL;
942
943 if (!aes->iv)
944 return -EINVAL;
945 }
946
947 BUILD_BUG_ON(CCP_AES_KEY_SB_COUNT != 1);
948 BUILD_BUG_ON(CCP_AES_CTX_SB_COUNT != 1);
949
950 ret = -EIO;
951 memset(&op, 0, sizeof(op));
952 op.cmd_q = cmd_q;
953 op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
954 op.sb_key = cmd_q->sb_key;
955 op.sb_ctx = cmd_q->sb_ctx;
956 op.init = (aes->mode == CCP_AES_MODE_ECB) ? 0 : 1;
957 op.u.aes.type = aes->type;
958 op.u.aes.mode = aes->mode;
959 op.u.aes.action = aes->action;
960
961 /* All supported key sizes fit in a single (32-byte) SB entry
962 * and must be in little endian format. Use the 256-bit byte
963 * swap passthru option to convert from big endian to little
964 * endian.
965 */
966 ret = ccp_init_dm_workarea(&key, cmd_q,
967 CCP_AES_KEY_SB_COUNT * CCP_SB_BYTES,
968 DMA_TO_DEVICE);
969 if (ret)
970 return ret;
971
972 dm_offset = CCP_SB_BYTES - aes->key_len;
973 ret = ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len);
974 if (ret)
975 goto e_key;
976 ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
977 CCP_PASSTHRU_BYTESWAP_256BIT);
978 if (ret) {
979 cmd->engine_error = cmd_q->cmd_error;
980 goto e_key;
981 }
982
983 /* The AES context fits in a single (32-byte) SB entry and
984 * must be in little endian format. Use the 256-bit byte swap
985 * passthru option to convert from big endian to little endian.
986 */
987 ret = ccp_init_dm_workarea(&ctx, cmd_q,
988 CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES,
989 DMA_BIDIRECTIONAL);
990 if (ret)
991 goto e_key;
992
993 if (aes->mode != CCP_AES_MODE_ECB) {
994 /* Load the AES context - convert to LE */
995 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
996 ret = ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
997 if (ret)
998 goto e_ctx;
999 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1000 CCP_PASSTHRU_BYTESWAP_256BIT);
1001 if (ret) {
1002 cmd->engine_error = cmd_q->cmd_error;
1003 goto e_ctx;
1004 }
1005 }
1006 switch (aes->mode) {
1007 case CCP_AES_MODE_CFB: /* CFB128 only */
1008 case CCP_AES_MODE_CTR:
1009 op.u.aes.size = AES_BLOCK_SIZE * BITS_PER_BYTE - 1;
1010 break;
1011 default:
1012 op.u.aes.size = 0;
1013 }
1014
1015 /* Prepare the input and output data workareas. For in-place
1016 * operations we need to set the dma direction to BIDIRECTIONAL
1017 * and copy the src workarea to the dst workarea.
1018 */
1019 if (sg_virt(aes->src) == sg_virt(aes->dst))
1020 in_place = true;
1021
1022 ret = ccp_init_data(&src, cmd_q, aes->src, aes->src_len,
1023 AES_BLOCK_SIZE,
1024 in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
1025 if (ret)
1026 goto e_ctx;
1027
1028 if (in_place) {
1029 dst = src;
1030 } else {
1031 ret = ccp_init_data(&dst, cmd_q, aes->dst, aes->src_len,
1032 AES_BLOCK_SIZE, DMA_FROM_DEVICE);
1033 if (ret)
1034 goto e_src;
1035 }
1036
1037 /* Send data to the CCP AES engine */
1038 while (src.sg_wa.bytes_left) {
1039 ccp_prepare_data(&src, &dst, &op, AES_BLOCK_SIZE, true);
1040 if (!src.sg_wa.bytes_left) {
1041 op.eom = 1;
1042
1043 /* Since we don't retrieve the AES context in ECB
1044 * mode we have to wait for the operation to complete
1045 * on the last piece of data
1046 */
1047 if (aes->mode == CCP_AES_MODE_ECB)
1048 op.soc = 1;
1049 }
1050
1051 ret = cmd_q->ccp->vdata->perform->aes(&op);
1052 if (ret) {
1053 cmd->engine_error = cmd_q->cmd_error;
1054 goto e_dst;
1055 }
1056
1057 ccp_process_data(&src, &dst, &op);
1058 }
1059
1060 if (aes->mode != CCP_AES_MODE_ECB) {
1061 /* Retrieve the AES context - convert from LE to BE using
1062 * 32-byte (256-bit) byteswapping
1063 */
1064 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1065 CCP_PASSTHRU_BYTESWAP_256BIT);
1066 if (ret) {
1067 cmd->engine_error = cmd_q->cmd_error;
1068 goto e_dst;
1069 }
1070
1071 /* ...but we only need AES_BLOCK_SIZE bytes */
1072 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
1073 ccp_get_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
1074 }
1075
1076 e_dst:
1077 if (!in_place)
1078 ccp_free_data(&dst, cmd_q);
1079
1080 e_src:
1081 ccp_free_data(&src, cmd_q);
1082
1083 e_ctx:
1084 ccp_dm_free(&ctx);
1085
1086 e_key:
1087 ccp_dm_free(&key);
1088
1089 return ret;
1090 }
1091
1092 static noinline_for_stack int
ccp_run_xts_aes_cmd(struct ccp_cmd_queue * cmd_q,struct ccp_cmd * cmd)1093 ccp_run_xts_aes_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1094 {
1095 struct ccp_xts_aes_engine *xts = &cmd->u.xts;
1096 struct ccp_dm_workarea key, ctx;
1097 struct ccp_data src, dst;
1098 struct ccp_op op;
1099 unsigned int unit_size, dm_offset;
1100 bool in_place = false;
1101 unsigned int sb_count;
1102 enum ccp_aes_type aestype;
1103 int ret;
1104
1105 switch (xts->unit_size) {
1106 case CCP_XTS_AES_UNIT_SIZE_16:
1107 unit_size = 16;
1108 break;
1109 case CCP_XTS_AES_UNIT_SIZE_512:
1110 unit_size = 512;
1111 break;
1112 case CCP_XTS_AES_UNIT_SIZE_1024:
1113 unit_size = 1024;
1114 break;
1115 case CCP_XTS_AES_UNIT_SIZE_2048:
1116 unit_size = 2048;
1117 break;
1118 case CCP_XTS_AES_UNIT_SIZE_4096:
1119 unit_size = 4096;
1120 break;
1121
1122 default:
1123 return -EINVAL;
1124 }
1125
1126 if (xts->key_len == AES_KEYSIZE_128)
1127 aestype = CCP_AES_TYPE_128;
1128 else if (xts->key_len == AES_KEYSIZE_256)
1129 aestype = CCP_AES_TYPE_256;
1130 else
1131 return -EINVAL;
1132
1133 if (!xts->final && (xts->src_len & (AES_BLOCK_SIZE - 1)))
1134 return -EINVAL;
1135
1136 if (xts->iv_len != AES_BLOCK_SIZE)
1137 return -EINVAL;
1138
1139 if (!xts->key || !xts->iv || !xts->src || !xts->dst)
1140 return -EINVAL;
1141
1142 BUILD_BUG_ON(CCP_XTS_AES_KEY_SB_COUNT != 1);
1143 BUILD_BUG_ON(CCP_XTS_AES_CTX_SB_COUNT != 1);
1144
1145 ret = -EIO;
1146 memset(&op, 0, sizeof(op));
1147 op.cmd_q = cmd_q;
1148 op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
1149 op.sb_key = cmd_q->sb_key;
1150 op.sb_ctx = cmd_q->sb_ctx;
1151 op.init = 1;
1152 op.u.xts.type = aestype;
1153 op.u.xts.action = xts->action;
1154 op.u.xts.unit_size = xts->unit_size;
1155
1156 /* A version 3 device only supports 128-bit keys, which fits into a
1157 * single SB entry. A version 5 device uses a 512-bit vector, so two
1158 * SB entries.
1159 */
1160 if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0))
1161 sb_count = CCP_XTS_AES_KEY_SB_COUNT;
1162 else
1163 sb_count = CCP5_XTS_AES_KEY_SB_COUNT;
1164 ret = ccp_init_dm_workarea(&key, cmd_q,
1165 sb_count * CCP_SB_BYTES,
1166 DMA_TO_DEVICE);
1167 if (ret)
1168 return ret;
1169
1170 if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0)) {
1171 /* All supported key sizes must be in little endian format.
1172 * Use the 256-bit byte swap passthru option to convert from
1173 * big endian to little endian.
1174 */
1175 dm_offset = CCP_SB_BYTES - AES_KEYSIZE_128;
1176 ret = ccp_set_dm_area(&key, dm_offset, xts->key, 0, xts->key_len);
1177 if (ret)
1178 goto e_key;
1179 ret = ccp_set_dm_area(&key, 0, xts->key, xts->key_len, xts->key_len);
1180 if (ret)
1181 goto e_key;
1182 } else {
1183 /* Version 5 CCPs use a 512-bit space for the key: each portion
1184 * occupies 256 bits, or one entire slot, and is zero-padded.
1185 */
1186 unsigned int pad;
1187
1188 dm_offset = CCP_SB_BYTES;
1189 pad = dm_offset - xts->key_len;
1190 ret = ccp_set_dm_area(&key, pad, xts->key, 0, xts->key_len);
1191 if (ret)
1192 goto e_key;
1193 ret = ccp_set_dm_area(&key, dm_offset + pad, xts->key,
1194 xts->key_len, xts->key_len);
1195 if (ret)
1196 goto e_key;
1197 }
1198 ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
1199 CCP_PASSTHRU_BYTESWAP_256BIT);
1200 if (ret) {
1201 cmd->engine_error = cmd_q->cmd_error;
1202 goto e_key;
1203 }
1204
1205 /* The AES context fits in a single (32-byte) SB entry and
1206 * for XTS is already in little endian format so no byte swapping
1207 * is needed.
1208 */
1209 ret = ccp_init_dm_workarea(&ctx, cmd_q,
1210 CCP_XTS_AES_CTX_SB_COUNT * CCP_SB_BYTES,
1211 DMA_BIDIRECTIONAL);
1212 if (ret)
1213 goto e_key;
1214
1215 ret = ccp_set_dm_area(&ctx, 0, xts->iv, 0, xts->iv_len);
1216 if (ret)
1217 goto e_ctx;
1218 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1219 CCP_PASSTHRU_BYTESWAP_NOOP);
1220 if (ret) {
1221 cmd->engine_error = cmd_q->cmd_error;
1222 goto e_ctx;
1223 }
1224
1225 /* Prepare the input and output data workareas. For in-place
1226 * operations we need to set the dma direction to BIDIRECTIONAL
1227 * and copy the src workarea to the dst workarea.
1228 */
1229 if (sg_virt(xts->src) == sg_virt(xts->dst))
1230 in_place = true;
1231
1232 ret = ccp_init_data(&src, cmd_q, xts->src, xts->src_len,
1233 unit_size,
1234 in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
1235 if (ret)
1236 goto e_ctx;
1237
1238 if (in_place) {
1239 dst = src;
1240 } else {
1241 ret = ccp_init_data(&dst, cmd_q, xts->dst, xts->src_len,
1242 unit_size, DMA_FROM_DEVICE);
1243 if (ret)
1244 goto e_src;
1245 }
1246
1247 /* Send data to the CCP AES engine */
1248 while (src.sg_wa.bytes_left) {
1249 ccp_prepare_data(&src, &dst, &op, unit_size, true);
1250 if (!src.sg_wa.bytes_left)
1251 op.eom = 1;
1252
1253 ret = cmd_q->ccp->vdata->perform->xts_aes(&op);
1254 if (ret) {
1255 cmd->engine_error = cmd_q->cmd_error;
1256 goto e_dst;
1257 }
1258
1259 ccp_process_data(&src, &dst, &op);
1260 }
1261
1262 /* Retrieve the AES context - convert from LE to BE using
1263 * 32-byte (256-bit) byteswapping
1264 */
1265 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1266 CCP_PASSTHRU_BYTESWAP_256BIT);
1267 if (ret) {
1268 cmd->engine_error = cmd_q->cmd_error;
1269 goto e_dst;
1270 }
1271
1272 /* ...but we only need AES_BLOCK_SIZE bytes */
1273 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
1274 ccp_get_dm_area(&ctx, dm_offset, xts->iv, 0, xts->iv_len);
1275
1276 e_dst:
1277 if (!in_place)
1278 ccp_free_data(&dst, cmd_q);
1279
1280 e_src:
1281 ccp_free_data(&src, cmd_q);
1282
1283 e_ctx:
1284 ccp_dm_free(&ctx);
1285
1286 e_key:
1287 ccp_dm_free(&key);
1288
1289 return ret;
1290 }
1291
1292 static noinline_for_stack int
ccp_run_des3_cmd(struct ccp_cmd_queue * cmd_q,struct ccp_cmd * cmd)1293 ccp_run_des3_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1294 {
1295 struct ccp_des3_engine *des3 = &cmd->u.des3;
1296
1297 struct ccp_dm_workarea key, ctx;
1298 struct ccp_data src, dst;
1299 struct ccp_op op;
1300 unsigned int dm_offset;
1301 unsigned int len_singlekey;
1302 bool in_place = false;
1303 int ret;
1304
1305 /* Error checks */
1306 if (cmd_q->ccp->vdata->version < CCP_VERSION(5, 0))
1307 return -EINVAL;
1308
1309 if (!cmd_q->ccp->vdata->perform->des3)
1310 return -EINVAL;
1311
1312 if (des3->key_len != DES3_EDE_KEY_SIZE)
1313 return -EINVAL;
1314
1315 if (((des3->mode == CCP_DES3_MODE_ECB) ||
1316 (des3->mode == CCP_DES3_MODE_CBC)) &&
1317 (des3->src_len & (DES3_EDE_BLOCK_SIZE - 1)))
1318 return -EINVAL;
1319
1320 if (!des3->key || !des3->src || !des3->dst)
1321 return -EINVAL;
1322
1323 if (des3->mode != CCP_DES3_MODE_ECB) {
1324 if (des3->iv_len != DES3_EDE_BLOCK_SIZE)
1325 return -EINVAL;
1326
1327 if (!des3->iv)
1328 return -EINVAL;
1329 }
1330
1331 /* Zero out all the fields of the command desc */
1332 memset(&op, 0, sizeof(op));
1333
1334 /* Set up the Function field */
1335 op.cmd_q = cmd_q;
1336 op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
1337 op.sb_key = cmd_q->sb_key;
1338
1339 op.init = (des3->mode == CCP_DES3_MODE_ECB) ? 0 : 1;
1340 op.u.des3.type = des3->type;
1341 op.u.des3.mode = des3->mode;
1342 op.u.des3.action = des3->action;
1343
1344 /*
1345 * All supported key sizes fit in a single (32-byte) KSB entry and
1346 * (like AES) must be in little endian format. Use the 256-bit byte
1347 * swap passthru option to convert from big endian to little endian.
1348 */
1349 ret = ccp_init_dm_workarea(&key, cmd_q,
1350 CCP_DES3_KEY_SB_COUNT * CCP_SB_BYTES,
1351 DMA_TO_DEVICE);
1352 if (ret)
1353 return ret;
1354
1355 /*
1356 * The contents of the key triplet are in the reverse order of what
1357 * is required by the engine. Copy the 3 pieces individually to put
1358 * them where they belong.
1359 */
1360 dm_offset = CCP_SB_BYTES - des3->key_len; /* Basic offset */
1361
1362 len_singlekey = des3->key_len / 3;
1363 ret = ccp_set_dm_area(&key, dm_offset + 2 * len_singlekey,
1364 des3->key, 0, len_singlekey);
1365 if (ret)
1366 goto e_key;
1367 ret = ccp_set_dm_area(&key, dm_offset + len_singlekey,
1368 des3->key, len_singlekey, len_singlekey);
1369 if (ret)
1370 goto e_key;
1371 ret = ccp_set_dm_area(&key, dm_offset,
1372 des3->key, 2 * len_singlekey, len_singlekey);
1373 if (ret)
1374 goto e_key;
1375
1376 /* Copy the key to the SB */
1377 ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
1378 CCP_PASSTHRU_BYTESWAP_256BIT);
1379 if (ret) {
1380 cmd->engine_error = cmd_q->cmd_error;
1381 goto e_key;
1382 }
1383
1384 /*
1385 * The DES3 context fits in a single (32-byte) KSB entry and
1386 * must be in little endian format. Use the 256-bit byte swap
1387 * passthru option to convert from big endian to little endian.
1388 */
1389 if (des3->mode != CCP_DES3_MODE_ECB) {
1390 op.sb_ctx = cmd_q->sb_ctx;
1391
1392 ret = ccp_init_dm_workarea(&ctx, cmd_q,
1393 CCP_DES3_CTX_SB_COUNT * CCP_SB_BYTES,
1394 DMA_BIDIRECTIONAL);
1395 if (ret)
1396 goto e_key;
1397
1398 /* Load the context into the LSB */
1399 dm_offset = CCP_SB_BYTES - des3->iv_len;
1400 ret = ccp_set_dm_area(&ctx, dm_offset, des3->iv, 0,
1401 des3->iv_len);
1402 if (ret)
1403 goto e_ctx;
1404
1405 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1406 CCP_PASSTHRU_BYTESWAP_256BIT);
1407 if (ret) {
1408 cmd->engine_error = cmd_q->cmd_error;
1409 goto e_ctx;
1410 }
1411 }
1412
1413 /*
1414 * Prepare the input and output data workareas. For in-place
1415 * operations we need to set the dma direction to BIDIRECTIONAL
1416 * and copy the src workarea to the dst workarea.
1417 */
1418 if (sg_virt(des3->src) == sg_virt(des3->dst))
1419 in_place = true;
1420
1421 ret = ccp_init_data(&src, cmd_q, des3->src, des3->src_len,
1422 DES3_EDE_BLOCK_SIZE,
1423 in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
1424 if (ret)
1425 goto e_ctx;
1426
1427 if (in_place)
1428 dst = src;
1429 else {
1430 ret = ccp_init_data(&dst, cmd_q, des3->dst, des3->src_len,
1431 DES3_EDE_BLOCK_SIZE, DMA_FROM_DEVICE);
1432 if (ret)
1433 goto e_src;
1434 }
1435
1436 /* Send data to the CCP DES3 engine */
1437 while (src.sg_wa.bytes_left) {
1438 ccp_prepare_data(&src, &dst, &op, DES3_EDE_BLOCK_SIZE, true);
1439 if (!src.sg_wa.bytes_left) {
1440 op.eom = 1;
1441
1442 /* Since we don't retrieve the context in ECB mode
1443 * we have to wait for the operation to complete
1444 * on the last piece of data
1445 */
1446 op.soc = 0;
1447 }
1448
1449 ret = cmd_q->ccp->vdata->perform->des3(&op);
1450 if (ret) {
1451 cmd->engine_error = cmd_q->cmd_error;
1452 goto e_dst;
1453 }
1454
1455 ccp_process_data(&src, &dst, &op);
1456 }
1457
1458 if (des3->mode != CCP_DES3_MODE_ECB) {
1459 /* Retrieve the context and make BE */
1460 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1461 CCP_PASSTHRU_BYTESWAP_256BIT);
1462 if (ret) {
1463 cmd->engine_error = cmd_q->cmd_error;
1464 goto e_dst;
1465 }
1466
1467 /* ...but we only need the last DES3_EDE_BLOCK_SIZE bytes */
1468 ccp_get_dm_area(&ctx, dm_offset, des3->iv, 0,
1469 DES3_EDE_BLOCK_SIZE);
1470 }
1471 e_dst:
1472 if (!in_place)
1473 ccp_free_data(&dst, cmd_q);
1474
1475 e_src:
1476 ccp_free_data(&src, cmd_q);
1477
1478 e_ctx:
1479 if (des3->mode != CCP_DES3_MODE_ECB)
1480 ccp_dm_free(&ctx);
1481
1482 e_key:
1483 ccp_dm_free(&key);
1484
1485 return ret;
1486 }
1487
1488 static noinline_for_stack int
ccp_run_sha_cmd(struct ccp_cmd_queue * cmd_q,struct ccp_cmd * cmd)1489 ccp_run_sha_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1490 {
1491 struct ccp_sha_engine *sha = &cmd->u.sha;
1492 struct ccp_dm_workarea ctx;
1493 struct ccp_data src;
1494 struct ccp_op op;
1495 unsigned int ioffset, ooffset;
1496 unsigned int digest_size;
1497 int sb_count;
1498 const void *init;
1499 u64 block_size;
1500 int ctx_size;
1501 int ret;
1502
1503 switch (sha->type) {
1504 case CCP_SHA_TYPE_1:
1505 if (sha->ctx_len < SHA1_DIGEST_SIZE)
1506 return -EINVAL;
1507 block_size = SHA1_BLOCK_SIZE;
1508 break;
1509 case CCP_SHA_TYPE_224:
1510 if (sha->ctx_len < SHA224_DIGEST_SIZE)
1511 return -EINVAL;
1512 block_size = SHA224_BLOCK_SIZE;
1513 break;
1514 case CCP_SHA_TYPE_256:
1515 if (sha->ctx_len < SHA256_DIGEST_SIZE)
1516 return -EINVAL;
1517 block_size = SHA256_BLOCK_SIZE;
1518 break;
1519 case CCP_SHA_TYPE_384:
1520 if (cmd_q->ccp->vdata->version < CCP_VERSION(4, 0)
1521 || sha->ctx_len < SHA384_DIGEST_SIZE)
1522 return -EINVAL;
1523 block_size = SHA384_BLOCK_SIZE;
1524 break;
1525 case CCP_SHA_TYPE_512:
1526 if (cmd_q->ccp->vdata->version < CCP_VERSION(4, 0)
1527 || sha->ctx_len < SHA512_DIGEST_SIZE)
1528 return -EINVAL;
1529 block_size = SHA512_BLOCK_SIZE;
1530 break;
1531 default:
1532 return -EINVAL;
1533 }
1534
1535 if (!sha->ctx)
1536 return -EINVAL;
1537
1538 if (!sha->final && (sha->src_len & (block_size - 1)))
1539 return -EINVAL;
1540
1541 /* The version 3 device can't handle zero-length input */
1542 if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0)) {
1543
1544 if (!sha->src_len) {
1545 unsigned int digest_len;
1546 const u8 *sha_zero;
1547
1548 /* Not final, just return */
1549 if (!sha->final)
1550 return 0;
1551
1552 /* CCP can't do a zero length sha operation so the
1553 * caller must buffer the data.
1554 */
1555 if (sha->msg_bits)
1556 return -EINVAL;
1557
1558 /* The CCP cannot perform zero-length sha operations
1559 * so the caller is required to buffer data for the
1560 * final operation. However, a sha operation for a
1561 * message with a total length of zero is valid so
1562 * known values are required to supply the result.
1563 */
1564 switch (sha->type) {
1565 case CCP_SHA_TYPE_1:
1566 sha_zero = sha1_zero_message_hash;
1567 digest_len = SHA1_DIGEST_SIZE;
1568 break;
1569 case CCP_SHA_TYPE_224:
1570 sha_zero = sha224_zero_message_hash;
1571 digest_len = SHA224_DIGEST_SIZE;
1572 break;
1573 case CCP_SHA_TYPE_256:
1574 sha_zero = sha256_zero_message_hash;
1575 digest_len = SHA256_DIGEST_SIZE;
1576 break;
1577 default:
1578 return -EINVAL;
1579 }
1580
1581 scatterwalk_map_and_copy((void *)sha_zero, sha->ctx, 0,
1582 digest_len, 1);
1583
1584 return 0;
1585 }
1586 }
1587
1588 /* Set variables used throughout */
1589 switch (sha->type) {
1590 case CCP_SHA_TYPE_1:
1591 digest_size = SHA1_DIGEST_SIZE;
1592 init = (void *) ccp_sha1_init;
1593 ctx_size = SHA1_DIGEST_SIZE;
1594 sb_count = 1;
1595 if (cmd_q->ccp->vdata->version != CCP_VERSION(3, 0))
1596 ooffset = ioffset = CCP_SB_BYTES - SHA1_DIGEST_SIZE;
1597 else
1598 ooffset = ioffset = 0;
1599 break;
1600 case CCP_SHA_TYPE_224:
1601 digest_size = SHA224_DIGEST_SIZE;
1602 init = (void *) ccp_sha224_init;
1603 ctx_size = SHA256_DIGEST_SIZE;
1604 sb_count = 1;
1605 ioffset = 0;
1606 if (cmd_q->ccp->vdata->version != CCP_VERSION(3, 0))
1607 ooffset = CCP_SB_BYTES - SHA224_DIGEST_SIZE;
1608 else
1609 ooffset = 0;
1610 break;
1611 case CCP_SHA_TYPE_256:
1612 digest_size = SHA256_DIGEST_SIZE;
1613 init = (void *) ccp_sha256_init;
1614 ctx_size = SHA256_DIGEST_SIZE;
1615 sb_count = 1;
1616 ooffset = ioffset = 0;
1617 break;
1618 case CCP_SHA_TYPE_384:
1619 digest_size = SHA384_DIGEST_SIZE;
1620 init = (void *) ccp_sha384_init;
1621 ctx_size = SHA512_DIGEST_SIZE;
1622 sb_count = 2;
1623 ioffset = 0;
1624 ooffset = 2 * CCP_SB_BYTES - SHA384_DIGEST_SIZE;
1625 break;
1626 case CCP_SHA_TYPE_512:
1627 digest_size = SHA512_DIGEST_SIZE;
1628 init = (void *) ccp_sha512_init;
1629 ctx_size = SHA512_DIGEST_SIZE;
1630 sb_count = 2;
1631 ooffset = ioffset = 0;
1632 break;
1633 default:
1634 ret = -EINVAL;
1635 goto e_data;
1636 }
1637
1638 /* For zero-length plaintext the src pointer is ignored;
1639 * otherwise both parts must be valid
1640 */
1641 if (sha->src_len && !sha->src)
1642 return -EINVAL;
1643
1644 memset(&op, 0, sizeof(op));
1645 op.cmd_q = cmd_q;
1646 op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
1647 op.sb_ctx = cmd_q->sb_ctx; /* Pre-allocated */
1648 op.u.sha.type = sha->type;
1649 op.u.sha.msg_bits = sha->msg_bits;
1650
1651 /* For SHA1/224/256 the context fits in a single (32-byte) SB entry;
1652 * SHA384/512 require 2 adjacent SB slots, with the right half in the
1653 * first slot, and the left half in the second. Each portion must then
1654 * be in little endian format: use the 256-bit byte swap option.
1655 */
1656 ret = ccp_init_dm_workarea(&ctx, cmd_q, sb_count * CCP_SB_BYTES,
1657 DMA_BIDIRECTIONAL);
1658 if (ret)
1659 return ret;
1660 if (sha->first) {
1661 switch (sha->type) {
1662 case CCP_SHA_TYPE_1:
1663 case CCP_SHA_TYPE_224:
1664 case CCP_SHA_TYPE_256:
1665 memcpy(ctx.address + ioffset, init, ctx_size);
1666 break;
1667 case CCP_SHA_TYPE_384:
1668 case CCP_SHA_TYPE_512:
1669 memcpy(ctx.address + ctx_size / 2, init,
1670 ctx_size / 2);
1671 memcpy(ctx.address, init + ctx_size / 2,
1672 ctx_size / 2);
1673 break;
1674 default:
1675 ret = -EINVAL;
1676 goto e_ctx;
1677 }
1678 } else {
1679 /* Restore the context */
1680 ret = ccp_set_dm_area(&ctx, 0, sha->ctx, 0,
1681 sb_count * CCP_SB_BYTES);
1682 if (ret)
1683 goto e_ctx;
1684 }
1685
1686 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1687 CCP_PASSTHRU_BYTESWAP_256BIT);
1688 if (ret) {
1689 cmd->engine_error = cmd_q->cmd_error;
1690 goto e_ctx;
1691 }
1692
1693 if (sha->src) {
1694 /* Send data to the CCP SHA engine; block_size is set above */
1695 ret = ccp_init_data(&src, cmd_q, sha->src, sha->src_len,
1696 block_size, DMA_TO_DEVICE);
1697 if (ret)
1698 goto e_ctx;
1699
1700 while (src.sg_wa.bytes_left) {
1701 ccp_prepare_data(&src, NULL, &op, block_size, false);
1702 if (sha->final && !src.sg_wa.bytes_left)
1703 op.eom = 1;
1704
1705 ret = cmd_q->ccp->vdata->perform->sha(&op);
1706 if (ret) {
1707 cmd->engine_error = cmd_q->cmd_error;
1708 goto e_data;
1709 }
1710
1711 ccp_process_data(&src, NULL, &op);
1712 }
1713 } else {
1714 op.eom = 1;
1715 ret = cmd_q->ccp->vdata->perform->sha(&op);
1716 if (ret) {
1717 cmd->engine_error = cmd_q->cmd_error;
1718 goto e_data;
1719 }
1720 }
1721
1722 /* Retrieve the SHA context - convert from LE to BE using
1723 * 32-byte (256-bit) byteswapping to BE
1724 */
1725 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1726 CCP_PASSTHRU_BYTESWAP_256BIT);
1727 if (ret) {
1728 cmd->engine_error = cmd_q->cmd_error;
1729 goto e_data;
1730 }
1731
1732 if (sha->final) {
1733 /* Finishing up, so get the digest */
1734 switch (sha->type) {
1735 case CCP_SHA_TYPE_1:
1736 case CCP_SHA_TYPE_224:
1737 case CCP_SHA_TYPE_256:
1738 ccp_get_dm_area(&ctx, ooffset,
1739 sha->ctx, 0,
1740 digest_size);
1741 break;
1742 case CCP_SHA_TYPE_384:
1743 case CCP_SHA_TYPE_512:
1744 ccp_get_dm_area(&ctx, 0,
1745 sha->ctx, LSB_ITEM_SIZE - ooffset,
1746 LSB_ITEM_SIZE);
1747 ccp_get_dm_area(&ctx, LSB_ITEM_SIZE + ooffset,
1748 sha->ctx, 0,
1749 LSB_ITEM_SIZE - ooffset);
1750 break;
1751 default:
1752 ret = -EINVAL;
1753 goto e_data;
1754 }
1755 } else {
1756 /* Stash the context */
1757 ccp_get_dm_area(&ctx, 0, sha->ctx, 0,
1758 sb_count * CCP_SB_BYTES);
1759 }
1760
1761 if (sha->final && sha->opad) {
1762 /* HMAC operation, recursively perform final SHA */
1763 struct ccp_cmd hmac_cmd;
1764 struct scatterlist sg;
1765 u8 *hmac_buf;
1766
1767 if (sha->opad_len != block_size) {
1768 ret = -EINVAL;
1769 goto e_data;
1770 }
1771
1772 hmac_buf = kmalloc(block_size + digest_size, GFP_KERNEL);
1773 if (!hmac_buf) {
1774 ret = -ENOMEM;
1775 goto e_data;
1776 }
1777 sg_init_one(&sg, hmac_buf, block_size + digest_size);
1778
1779 scatterwalk_map_and_copy(hmac_buf, sha->opad, 0, block_size, 0);
1780 switch (sha->type) {
1781 case CCP_SHA_TYPE_1:
1782 case CCP_SHA_TYPE_224:
1783 case CCP_SHA_TYPE_256:
1784 memcpy(hmac_buf + block_size,
1785 ctx.address + ooffset,
1786 digest_size);
1787 break;
1788 case CCP_SHA_TYPE_384:
1789 case CCP_SHA_TYPE_512:
1790 memcpy(hmac_buf + block_size,
1791 ctx.address + LSB_ITEM_SIZE + ooffset,
1792 LSB_ITEM_SIZE);
1793 memcpy(hmac_buf + block_size +
1794 (LSB_ITEM_SIZE - ooffset),
1795 ctx.address,
1796 LSB_ITEM_SIZE);
1797 break;
1798 default:
1799 kfree(hmac_buf);
1800 ret = -EINVAL;
1801 goto e_data;
1802 }
1803
1804 memset(&hmac_cmd, 0, sizeof(hmac_cmd));
1805 hmac_cmd.engine = CCP_ENGINE_SHA;
1806 hmac_cmd.u.sha.type = sha->type;
1807 hmac_cmd.u.sha.ctx = sha->ctx;
1808 hmac_cmd.u.sha.ctx_len = sha->ctx_len;
1809 hmac_cmd.u.sha.src = &sg;
1810 hmac_cmd.u.sha.src_len = block_size + digest_size;
1811 hmac_cmd.u.sha.opad = NULL;
1812 hmac_cmd.u.sha.opad_len = 0;
1813 hmac_cmd.u.sha.first = 1;
1814 hmac_cmd.u.sha.final = 1;
1815 hmac_cmd.u.sha.msg_bits = (block_size + digest_size) << 3;
1816
1817 ret = ccp_run_sha_cmd(cmd_q, &hmac_cmd);
1818 if (ret)
1819 cmd->engine_error = hmac_cmd.engine_error;
1820
1821 kfree(hmac_buf);
1822 }
1823
1824 e_data:
1825 if (sha->src)
1826 ccp_free_data(&src, cmd_q);
1827
1828 e_ctx:
1829 ccp_dm_free(&ctx);
1830
1831 return ret;
1832 }
1833
1834 static noinline_for_stack int
ccp_run_rsa_cmd(struct ccp_cmd_queue * cmd_q,struct ccp_cmd * cmd)1835 ccp_run_rsa_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1836 {
1837 struct ccp_rsa_engine *rsa = &cmd->u.rsa;
1838 struct ccp_dm_workarea exp, src, dst;
1839 struct ccp_op op;
1840 unsigned int sb_count, i_len, o_len;
1841 int ret;
1842
1843 /* Check against the maximum allowable size, in bits */
1844 if (rsa->key_size > cmd_q->ccp->vdata->rsamax)
1845 return -EINVAL;
1846
1847 if (!rsa->exp || !rsa->mod || !rsa->src || !rsa->dst)
1848 return -EINVAL;
1849
1850 memset(&op, 0, sizeof(op));
1851 op.cmd_q = cmd_q;
1852 op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
1853
1854 /* The RSA modulus must precede the message being acted upon, so
1855 * it must be copied to a DMA area where the message and the
1856 * modulus can be concatenated. Therefore the input buffer
1857 * length required is twice the output buffer length (which
1858 * must be a multiple of 256-bits). Compute o_len, i_len in bytes.
1859 * Buffer sizes must be a multiple of 32 bytes; rounding up may be
1860 * required.
1861 */
1862 o_len = 32 * ((rsa->key_size + 255) / 256);
1863 i_len = o_len * 2;
1864
1865 sb_count = 0;
1866 if (cmd_q->ccp->vdata->version < CCP_VERSION(5, 0)) {
1867 /* sb_count is the number of storage block slots required
1868 * for the modulus.
1869 */
1870 sb_count = o_len / CCP_SB_BYTES;
1871 op.sb_key = cmd_q->ccp->vdata->perform->sballoc(cmd_q,
1872 sb_count);
1873 if (!op.sb_key)
1874 return -EIO;
1875 } else {
1876 /* A version 5 device allows a modulus size that will not fit
1877 * in the LSB, so the command will transfer it from memory.
1878 * Set the sb key to the default, even though it's not used.
1879 */
1880 op.sb_key = cmd_q->sb_key;
1881 }
1882
1883 /* The RSA exponent must be in little endian format. Reverse its
1884 * byte order.
1885 */
1886 ret = ccp_init_dm_workarea(&exp, cmd_q, o_len, DMA_TO_DEVICE);
1887 if (ret)
1888 goto e_sb;
1889
1890 ret = ccp_reverse_set_dm_area(&exp, 0, rsa->exp, 0, rsa->exp_len);
1891 if (ret)
1892 goto e_exp;
1893
1894 if (cmd_q->ccp->vdata->version < CCP_VERSION(5, 0)) {
1895 /* Copy the exponent to the local storage block, using
1896 * as many 32-byte blocks as were allocated above. It's
1897 * already little endian, so no further change is required.
1898 */
1899 ret = ccp_copy_to_sb(cmd_q, &exp, op.jobid, op.sb_key,
1900 CCP_PASSTHRU_BYTESWAP_NOOP);
1901 if (ret) {
1902 cmd->engine_error = cmd_q->cmd_error;
1903 goto e_exp;
1904 }
1905 } else {
1906 /* The exponent can be retrieved from memory via DMA. */
1907 op.exp.u.dma.address = exp.dma.address;
1908 op.exp.u.dma.offset = 0;
1909 }
1910
1911 /* Concatenate the modulus and the message. Both the modulus and
1912 * the operands must be in little endian format. Since the input
1913 * is in big endian format it must be converted.
1914 */
1915 ret = ccp_init_dm_workarea(&src, cmd_q, i_len, DMA_TO_DEVICE);
1916 if (ret)
1917 goto e_exp;
1918
1919 ret = ccp_reverse_set_dm_area(&src, 0, rsa->mod, 0, rsa->mod_len);
1920 if (ret)
1921 goto e_src;
1922 ret = ccp_reverse_set_dm_area(&src, o_len, rsa->src, 0, rsa->src_len);
1923 if (ret)
1924 goto e_src;
1925
1926 /* Prepare the output area for the operation */
1927 ret = ccp_init_dm_workarea(&dst, cmd_q, o_len, DMA_FROM_DEVICE);
1928 if (ret)
1929 goto e_src;
1930
1931 op.soc = 1;
1932 op.src.u.dma.address = src.dma.address;
1933 op.src.u.dma.offset = 0;
1934 op.src.u.dma.length = i_len;
1935 op.dst.u.dma.address = dst.dma.address;
1936 op.dst.u.dma.offset = 0;
1937 op.dst.u.dma.length = o_len;
1938
1939 op.u.rsa.mod_size = rsa->key_size;
1940 op.u.rsa.input_len = i_len;
1941
1942 ret = cmd_q->ccp->vdata->perform->rsa(&op);
1943 if (ret) {
1944 cmd->engine_error = cmd_q->cmd_error;
1945 goto e_dst;
1946 }
1947
1948 ccp_reverse_get_dm_area(&dst, 0, rsa->dst, 0, rsa->mod_len);
1949
1950 e_dst:
1951 ccp_dm_free(&dst);
1952
1953 e_src:
1954 ccp_dm_free(&src);
1955
1956 e_exp:
1957 ccp_dm_free(&exp);
1958
1959 e_sb:
1960 if (sb_count)
1961 cmd_q->ccp->vdata->perform->sbfree(cmd_q, op.sb_key, sb_count);
1962
1963 return ret;
1964 }
1965
1966 static noinline_for_stack int
ccp_run_passthru_cmd(struct ccp_cmd_queue * cmd_q,struct ccp_cmd * cmd)1967 ccp_run_passthru_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1968 {
1969 struct ccp_passthru_engine *pt = &cmd->u.passthru;
1970 struct ccp_dm_workarea mask;
1971 struct ccp_data src, dst;
1972 struct ccp_op op;
1973 bool in_place = false;
1974 unsigned int i;
1975 int ret = 0;
1976
1977 if (!pt->final && (pt->src_len & (CCP_PASSTHRU_BLOCKSIZE - 1)))
1978 return -EINVAL;
1979
1980 if (!pt->src || !pt->dst)
1981 return -EINVAL;
1982
1983 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
1984 if (pt->mask_len != CCP_PASSTHRU_MASKSIZE)
1985 return -EINVAL;
1986 if (!pt->mask)
1987 return -EINVAL;
1988 }
1989
1990 BUILD_BUG_ON(CCP_PASSTHRU_SB_COUNT != 1);
1991
1992 memset(&op, 0, sizeof(op));
1993 op.cmd_q = cmd_q;
1994 op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
1995
1996 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
1997 /* Load the mask */
1998 op.sb_key = cmd_q->sb_key;
1999
2000 ret = ccp_init_dm_workarea(&mask, cmd_q,
2001 CCP_PASSTHRU_SB_COUNT *
2002 CCP_SB_BYTES,
2003 DMA_TO_DEVICE);
2004 if (ret)
2005 return ret;
2006
2007 ret = ccp_set_dm_area(&mask, 0, pt->mask, 0, pt->mask_len);
2008 if (ret)
2009 goto e_mask;
2010 ret = ccp_copy_to_sb(cmd_q, &mask, op.jobid, op.sb_key,
2011 CCP_PASSTHRU_BYTESWAP_NOOP);
2012 if (ret) {
2013 cmd->engine_error = cmd_q->cmd_error;
2014 goto e_mask;
2015 }
2016 }
2017
2018 /* Prepare the input and output data workareas. For in-place
2019 * operations we need to set the dma direction to BIDIRECTIONAL
2020 * and copy the src workarea to the dst workarea.
2021 */
2022 if (sg_virt(pt->src) == sg_virt(pt->dst))
2023 in_place = true;
2024
2025 ret = ccp_init_data(&src, cmd_q, pt->src, pt->src_len,
2026 CCP_PASSTHRU_MASKSIZE,
2027 in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
2028 if (ret)
2029 goto e_mask;
2030
2031 if (in_place) {
2032 dst = src;
2033 } else {
2034 ret = ccp_init_data(&dst, cmd_q, pt->dst, pt->src_len,
2035 CCP_PASSTHRU_MASKSIZE, DMA_FROM_DEVICE);
2036 if (ret)
2037 goto e_src;
2038 }
2039
2040 /* Send data to the CCP Passthru engine
2041 * Because the CCP engine works on a single source and destination
2042 * dma address at a time, each entry in the source scatterlist
2043 * (after the dma_map_sg call) must be less than or equal to the
2044 * (remaining) length in the destination scatterlist entry and the
2045 * length must be a multiple of CCP_PASSTHRU_BLOCKSIZE
2046 */
2047 dst.sg_wa.sg_used = 0;
2048 for (i = 1; i <= src.sg_wa.dma_count; i++) {
2049 if (!dst.sg_wa.sg ||
2050 (sg_dma_len(dst.sg_wa.sg) < sg_dma_len(src.sg_wa.sg))) {
2051 ret = -EINVAL;
2052 goto e_dst;
2053 }
2054
2055 if (i == src.sg_wa.dma_count) {
2056 op.eom = 1;
2057 op.soc = 1;
2058 }
2059
2060 op.src.type = CCP_MEMTYPE_SYSTEM;
2061 op.src.u.dma.address = sg_dma_address(src.sg_wa.sg);
2062 op.src.u.dma.offset = 0;
2063 op.src.u.dma.length = sg_dma_len(src.sg_wa.sg);
2064
2065 op.dst.type = CCP_MEMTYPE_SYSTEM;
2066 op.dst.u.dma.address = sg_dma_address(dst.sg_wa.sg);
2067 op.dst.u.dma.offset = dst.sg_wa.sg_used;
2068 op.dst.u.dma.length = op.src.u.dma.length;
2069
2070 ret = cmd_q->ccp->vdata->perform->passthru(&op);
2071 if (ret) {
2072 cmd->engine_error = cmd_q->cmd_error;
2073 goto e_dst;
2074 }
2075
2076 dst.sg_wa.sg_used += sg_dma_len(src.sg_wa.sg);
2077 if (dst.sg_wa.sg_used == sg_dma_len(dst.sg_wa.sg)) {
2078 dst.sg_wa.sg = sg_next(dst.sg_wa.sg);
2079 dst.sg_wa.sg_used = 0;
2080 }
2081 src.sg_wa.sg = sg_next(src.sg_wa.sg);
2082 }
2083
2084 e_dst:
2085 if (!in_place)
2086 ccp_free_data(&dst, cmd_q);
2087
2088 e_src:
2089 ccp_free_data(&src, cmd_q);
2090
2091 e_mask:
2092 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP)
2093 ccp_dm_free(&mask);
2094
2095 return ret;
2096 }
2097
2098 static noinline_for_stack int
ccp_run_passthru_nomap_cmd(struct ccp_cmd_queue * cmd_q,struct ccp_cmd * cmd)2099 ccp_run_passthru_nomap_cmd(struct ccp_cmd_queue *cmd_q,
2100 struct ccp_cmd *cmd)
2101 {
2102 struct ccp_passthru_nomap_engine *pt = &cmd->u.passthru_nomap;
2103 struct ccp_dm_workarea mask;
2104 struct ccp_op op;
2105 int ret;
2106
2107 if (!pt->final && (pt->src_len & (CCP_PASSTHRU_BLOCKSIZE - 1)))
2108 return -EINVAL;
2109
2110 if (!pt->src_dma || !pt->dst_dma)
2111 return -EINVAL;
2112
2113 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
2114 if (pt->mask_len != CCP_PASSTHRU_MASKSIZE)
2115 return -EINVAL;
2116 if (!pt->mask)
2117 return -EINVAL;
2118 }
2119
2120 BUILD_BUG_ON(CCP_PASSTHRU_SB_COUNT != 1);
2121
2122 memset(&op, 0, sizeof(op));
2123 op.cmd_q = cmd_q;
2124 op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
2125
2126 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
2127 /* Load the mask */
2128 op.sb_key = cmd_q->sb_key;
2129
2130 mask.length = pt->mask_len;
2131 mask.dma.address = pt->mask;
2132 mask.dma.length = pt->mask_len;
2133
2134 ret = ccp_copy_to_sb(cmd_q, &mask, op.jobid, op.sb_key,
2135 CCP_PASSTHRU_BYTESWAP_NOOP);
2136 if (ret) {
2137 cmd->engine_error = cmd_q->cmd_error;
2138 return ret;
2139 }
2140 }
2141
2142 /* Send data to the CCP Passthru engine */
2143 op.eom = 1;
2144 op.soc = 1;
2145
2146 op.src.type = CCP_MEMTYPE_SYSTEM;
2147 op.src.u.dma.address = pt->src_dma;
2148 op.src.u.dma.offset = 0;
2149 op.src.u.dma.length = pt->src_len;
2150
2151 op.dst.type = CCP_MEMTYPE_SYSTEM;
2152 op.dst.u.dma.address = pt->dst_dma;
2153 op.dst.u.dma.offset = 0;
2154 op.dst.u.dma.length = pt->src_len;
2155
2156 ret = cmd_q->ccp->vdata->perform->passthru(&op);
2157 if (ret)
2158 cmd->engine_error = cmd_q->cmd_error;
2159
2160 return ret;
2161 }
2162
ccp_run_ecc_mm_cmd(struct ccp_cmd_queue * cmd_q,struct ccp_cmd * cmd)2163 static int ccp_run_ecc_mm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
2164 {
2165 struct ccp_ecc_engine *ecc = &cmd->u.ecc;
2166 struct ccp_dm_workarea src, dst;
2167 struct ccp_op op;
2168 int ret;
2169 u8 *save;
2170
2171 if (!ecc->u.mm.operand_1 ||
2172 (ecc->u.mm.operand_1_len > CCP_ECC_MODULUS_BYTES))
2173 return -EINVAL;
2174
2175 if (ecc->function != CCP_ECC_FUNCTION_MINV_384BIT)
2176 if (!ecc->u.mm.operand_2 ||
2177 (ecc->u.mm.operand_2_len > CCP_ECC_MODULUS_BYTES))
2178 return -EINVAL;
2179
2180 if (!ecc->u.mm.result ||
2181 (ecc->u.mm.result_len < CCP_ECC_MODULUS_BYTES))
2182 return -EINVAL;
2183
2184 memset(&op, 0, sizeof(op));
2185 op.cmd_q = cmd_q;
2186 op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
2187
2188 /* Concatenate the modulus and the operands. Both the modulus and
2189 * the operands must be in little endian format. Since the input
2190 * is in big endian format it must be converted and placed in a
2191 * fixed length buffer.
2192 */
2193 ret = ccp_init_dm_workarea(&src, cmd_q, CCP_ECC_SRC_BUF_SIZE,
2194 DMA_TO_DEVICE);
2195 if (ret)
2196 return ret;
2197
2198 /* Save the workarea address since it is updated in order to perform
2199 * the concatenation
2200 */
2201 save = src.address;
2202
2203 /* Copy the ECC modulus */
2204 ret = ccp_reverse_set_dm_area(&src, 0, ecc->mod, 0, ecc->mod_len);
2205 if (ret)
2206 goto e_src;
2207 src.address += CCP_ECC_OPERAND_SIZE;
2208
2209 /* Copy the first operand */
2210 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.mm.operand_1, 0,
2211 ecc->u.mm.operand_1_len);
2212 if (ret)
2213 goto e_src;
2214 src.address += CCP_ECC_OPERAND_SIZE;
2215
2216 if (ecc->function != CCP_ECC_FUNCTION_MINV_384BIT) {
2217 /* Copy the second operand */
2218 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.mm.operand_2, 0,
2219 ecc->u.mm.operand_2_len);
2220 if (ret)
2221 goto e_src;
2222 src.address += CCP_ECC_OPERAND_SIZE;
2223 }
2224
2225 /* Restore the workarea address */
2226 src.address = save;
2227
2228 /* Prepare the output area for the operation */
2229 ret = ccp_init_dm_workarea(&dst, cmd_q, CCP_ECC_DST_BUF_SIZE,
2230 DMA_FROM_DEVICE);
2231 if (ret)
2232 goto e_src;
2233
2234 op.soc = 1;
2235 op.src.u.dma.address = src.dma.address;
2236 op.src.u.dma.offset = 0;
2237 op.src.u.dma.length = src.length;
2238 op.dst.u.dma.address = dst.dma.address;
2239 op.dst.u.dma.offset = 0;
2240 op.dst.u.dma.length = dst.length;
2241
2242 op.u.ecc.function = cmd->u.ecc.function;
2243
2244 ret = cmd_q->ccp->vdata->perform->ecc(&op);
2245 if (ret) {
2246 cmd->engine_error = cmd_q->cmd_error;
2247 goto e_dst;
2248 }
2249
2250 ecc->ecc_result = le16_to_cpup(
2251 (const __le16 *)(dst.address + CCP_ECC_RESULT_OFFSET));
2252 if (!(ecc->ecc_result & CCP_ECC_RESULT_SUCCESS)) {
2253 ret = -EIO;
2254 goto e_dst;
2255 }
2256
2257 /* Save the ECC result */
2258 ccp_reverse_get_dm_area(&dst, 0, ecc->u.mm.result, 0,
2259 CCP_ECC_MODULUS_BYTES);
2260
2261 e_dst:
2262 ccp_dm_free(&dst);
2263
2264 e_src:
2265 ccp_dm_free(&src);
2266
2267 return ret;
2268 }
2269
ccp_run_ecc_pm_cmd(struct ccp_cmd_queue * cmd_q,struct ccp_cmd * cmd)2270 static int ccp_run_ecc_pm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
2271 {
2272 struct ccp_ecc_engine *ecc = &cmd->u.ecc;
2273 struct ccp_dm_workarea src, dst;
2274 struct ccp_op op;
2275 int ret;
2276 u8 *save;
2277
2278 if (!ecc->u.pm.point_1.x ||
2279 (ecc->u.pm.point_1.x_len > CCP_ECC_MODULUS_BYTES) ||
2280 !ecc->u.pm.point_1.y ||
2281 (ecc->u.pm.point_1.y_len > CCP_ECC_MODULUS_BYTES))
2282 return -EINVAL;
2283
2284 if (ecc->function == CCP_ECC_FUNCTION_PADD_384BIT) {
2285 if (!ecc->u.pm.point_2.x ||
2286 (ecc->u.pm.point_2.x_len > CCP_ECC_MODULUS_BYTES) ||
2287 !ecc->u.pm.point_2.y ||
2288 (ecc->u.pm.point_2.y_len > CCP_ECC_MODULUS_BYTES))
2289 return -EINVAL;
2290 } else {
2291 if (!ecc->u.pm.domain_a ||
2292 (ecc->u.pm.domain_a_len > CCP_ECC_MODULUS_BYTES))
2293 return -EINVAL;
2294
2295 if (ecc->function == CCP_ECC_FUNCTION_PMUL_384BIT)
2296 if (!ecc->u.pm.scalar ||
2297 (ecc->u.pm.scalar_len > CCP_ECC_MODULUS_BYTES))
2298 return -EINVAL;
2299 }
2300
2301 if (!ecc->u.pm.result.x ||
2302 (ecc->u.pm.result.x_len < CCP_ECC_MODULUS_BYTES) ||
2303 !ecc->u.pm.result.y ||
2304 (ecc->u.pm.result.y_len < CCP_ECC_MODULUS_BYTES))
2305 return -EINVAL;
2306
2307 memset(&op, 0, sizeof(op));
2308 op.cmd_q = cmd_q;
2309 op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
2310
2311 /* Concatenate the modulus and the operands. Both the modulus and
2312 * the operands must be in little endian format. Since the input
2313 * is in big endian format it must be converted and placed in a
2314 * fixed length buffer.
2315 */
2316 ret = ccp_init_dm_workarea(&src, cmd_q, CCP_ECC_SRC_BUF_SIZE,
2317 DMA_TO_DEVICE);
2318 if (ret)
2319 return ret;
2320
2321 /* Save the workarea address since it is updated in order to perform
2322 * the concatenation
2323 */
2324 save = src.address;
2325
2326 /* Copy the ECC modulus */
2327 ret = ccp_reverse_set_dm_area(&src, 0, ecc->mod, 0, ecc->mod_len);
2328 if (ret)
2329 goto e_src;
2330 src.address += CCP_ECC_OPERAND_SIZE;
2331
2332 /* Copy the first point X and Y coordinate */
2333 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_1.x, 0,
2334 ecc->u.pm.point_1.x_len);
2335 if (ret)
2336 goto e_src;
2337 src.address += CCP_ECC_OPERAND_SIZE;
2338 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_1.y, 0,
2339 ecc->u.pm.point_1.y_len);
2340 if (ret)
2341 goto e_src;
2342 src.address += CCP_ECC_OPERAND_SIZE;
2343
2344 /* Set the first point Z coordinate to 1 */
2345 *src.address = 0x01;
2346 src.address += CCP_ECC_OPERAND_SIZE;
2347
2348 if (ecc->function == CCP_ECC_FUNCTION_PADD_384BIT) {
2349 /* Copy the second point X and Y coordinate */
2350 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_2.x, 0,
2351 ecc->u.pm.point_2.x_len);
2352 if (ret)
2353 goto e_src;
2354 src.address += CCP_ECC_OPERAND_SIZE;
2355 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_2.y, 0,
2356 ecc->u.pm.point_2.y_len);
2357 if (ret)
2358 goto e_src;
2359 src.address += CCP_ECC_OPERAND_SIZE;
2360
2361 /* Set the second point Z coordinate to 1 */
2362 *src.address = 0x01;
2363 src.address += CCP_ECC_OPERAND_SIZE;
2364 } else {
2365 /* Copy the Domain "a" parameter */
2366 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.domain_a, 0,
2367 ecc->u.pm.domain_a_len);
2368 if (ret)
2369 goto e_src;
2370 src.address += CCP_ECC_OPERAND_SIZE;
2371
2372 if (ecc->function == CCP_ECC_FUNCTION_PMUL_384BIT) {
2373 /* Copy the scalar value */
2374 ret = ccp_reverse_set_dm_area(&src, 0,
2375 ecc->u.pm.scalar, 0,
2376 ecc->u.pm.scalar_len);
2377 if (ret)
2378 goto e_src;
2379 src.address += CCP_ECC_OPERAND_SIZE;
2380 }
2381 }
2382
2383 /* Restore the workarea address */
2384 src.address = save;
2385
2386 /* Prepare the output area for the operation */
2387 ret = ccp_init_dm_workarea(&dst, cmd_q, CCP_ECC_DST_BUF_SIZE,
2388 DMA_FROM_DEVICE);
2389 if (ret)
2390 goto e_src;
2391
2392 op.soc = 1;
2393 op.src.u.dma.address = src.dma.address;
2394 op.src.u.dma.offset = 0;
2395 op.src.u.dma.length = src.length;
2396 op.dst.u.dma.address = dst.dma.address;
2397 op.dst.u.dma.offset = 0;
2398 op.dst.u.dma.length = dst.length;
2399
2400 op.u.ecc.function = cmd->u.ecc.function;
2401
2402 ret = cmd_q->ccp->vdata->perform->ecc(&op);
2403 if (ret) {
2404 cmd->engine_error = cmd_q->cmd_error;
2405 goto e_dst;
2406 }
2407
2408 ecc->ecc_result = le16_to_cpup(
2409 (const __le16 *)(dst.address + CCP_ECC_RESULT_OFFSET));
2410 if (!(ecc->ecc_result & CCP_ECC_RESULT_SUCCESS)) {
2411 ret = -EIO;
2412 goto e_dst;
2413 }
2414
2415 /* Save the workarea address since it is updated as we walk through
2416 * to copy the point math result
2417 */
2418 save = dst.address;
2419
2420 /* Save the ECC result X and Y coordinates */
2421 ccp_reverse_get_dm_area(&dst, 0, ecc->u.pm.result.x, 0,
2422 CCP_ECC_MODULUS_BYTES);
2423 dst.address += CCP_ECC_OUTPUT_SIZE;
2424 ccp_reverse_get_dm_area(&dst, 0, ecc->u.pm.result.y, 0,
2425 CCP_ECC_MODULUS_BYTES);
2426
2427 /* Restore the workarea address */
2428 dst.address = save;
2429
2430 e_dst:
2431 ccp_dm_free(&dst);
2432
2433 e_src:
2434 ccp_dm_free(&src);
2435
2436 return ret;
2437 }
2438
2439 static noinline_for_stack int
ccp_run_ecc_cmd(struct ccp_cmd_queue * cmd_q,struct ccp_cmd * cmd)2440 ccp_run_ecc_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
2441 {
2442 struct ccp_ecc_engine *ecc = &cmd->u.ecc;
2443
2444 ecc->ecc_result = 0;
2445
2446 if (!ecc->mod ||
2447 (ecc->mod_len > CCP_ECC_MODULUS_BYTES))
2448 return -EINVAL;
2449
2450 switch (ecc->function) {
2451 case CCP_ECC_FUNCTION_MMUL_384BIT:
2452 case CCP_ECC_FUNCTION_MADD_384BIT:
2453 case CCP_ECC_FUNCTION_MINV_384BIT:
2454 return ccp_run_ecc_mm_cmd(cmd_q, cmd);
2455
2456 case CCP_ECC_FUNCTION_PADD_384BIT:
2457 case CCP_ECC_FUNCTION_PMUL_384BIT:
2458 case CCP_ECC_FUNCTION_PDBL_384BIT:
2459 return ccp_run_ecc_pm_cmd(cmd_q, cmd);
2460
2461 default:
2462 return -EINVAL;
2463 }
2464 }
2465
ccp_run_cmd(struct ccp_cmd_queue * cmd_q,struct ccp_cmd * cmd)2466 int ccp_run_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
2467 {
2468 int ret;
2469
2470 cmd->engine_error = 0;
2471 cmd_q->cmd_error = 0;
2472 cmd_q->int_rcvd = 0;
2473 cmd_q->free_slots = cmd_q->ccp->vdata->perform->get_free_slots(cmd_q);
2474
2475 switch (cmd->engine) {
2476 case CCP_ENGINE_AES:
2477 switch (cmd->u.aes.mode) {
2478 case CCP_AES_MODE_CMAC:
2479 ret = ccp_run_aes_cmac_cmd(cmd_q, cmd);
2480 break;
2481 case CCP_AES_MODE_GCM:
2482 ret = ccp_run_aes_gcm_cmd(cmd_q, cmd);
2483 break;
2484 default:
2485 ret = ccp_run_aes_cmd(cmd_q, cmd);
2486 break;
2487 }
2488 break;
2489 case CCP_ENGINE_XTS_AES_128:
2490 ret = ccp_run_xts_aes_cmd(cmd_q, cmd);
2491 break;
2492 case CCP_ENGINE_DES3:
2493 ret = ccp_run_des3_cmd(cmd_q, cmd);
2494 break;
2495 case CCP_ENGINE_SHA:
2496 ret = ccp_run_sha_cmd(cmd_q, cmd);
2497 break;
2498 case CCP_ENGINE_RSA:
2499 ret = ccp_run_rsa_cmd(cmd_q, cmd);
2500 break;
2501 case CCP_ENGINE_PASSTHRU:
2502 if (cmd->flags & CCP_CMD_PASSTHRU_NO_DMA_MAP)
2503 ret = ccp_run_passthru_nomap_cmd(cmd_q, cmd);
2504 else
2505 ret = ccp_run_passthru_cmd(cmd_q, cmd);
2506 break;
2507 case CCP_ENGINE_ECC:
2508 ret = ccp_run_ecc_cmd(cmd_q, cmd);
2509 break;
2510 default:
2511 ret = -EINVAL;
2512 }
2513
2514 return ret;
2515 }
2516