1d2912cb1SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only
263b94509STom Lendacky /*
363b94509STom Lendacky * AMD Cryptographic Coprocessor (CCP) driver
463b94509STom Lendacky *
5499df967SHook, Gary * Copyright (C) 2013-2019 Advanced Micro Devices, Inc.
663b94509STom Lendacky *
763b94509STom Lendacky * Author: Tom Lendacky <thomas.lendacky@amd.com>
8a43eb985SGary R Hook * Author: Gary R Hook <gary.hook@amd.com>
963b94509STom Lendacky */
1063b94509STom Lendacky
110c3dc787SHerbert Xu #include <linux/dma-mapping.h>
1263b94509STom Lendacky #include <linux/module.h>
1363b94509STom Lendacky #include <linux/kernel.h>
1463b94509STom Lendacky #include <linux/interrupt.h>
1563b94509STom Lendacky #include <crypto/scatterwalk.h>
16990672d4SGary R Hook #include <crypto/des.h>
17ea0375afSGary R Hook #include <linux/ccp.h>
1863b94509STom Lendacky
1963b94509STom Lendacky #include "ccp-dev.h"
2063b94509STom Lendacky
21c11baa02STom Lendacky /* SHA initial context values */
224b394a23SGary R Hook static const __be32 ccp_sha1_init[SHA1_DIGEST_SIZE / sizeof(__be32)] = {
23c11baa02STom Lendacky cpu_to_be32(SHA1_H0), cpu_to_be32(SHA1_H1),
24c11baa02STom Lendacky cpu_to_be32(SHA1_H2), cpu_to_be32(SHA1_H3),
254b394a23SGary R Hook cpu_to_be32(SHA1_H4),
26c11baa02STom Lendacky };
27c11baa02STom Lendacky
284b394a23SGary R Hook static const __be32 ccp_sha224_init[SHA256_DIGEST_SIZE / sizeof(__be32)] = {
29c11baa02STom Lendacky cpu_to_be32(SHA224_H0), cpu_to_be32(SHA224_H1),
30c11baa02STom Lendacky cpu_to_be32(SHA224_H2), cpu_to_be32(SHA224_H3),
31c11baa02STom Lendacky cpu_to_be32(SHA224_H4), cpu_to_be32(SHA224_H5),
32c11baa02STom Lendacky cpu_to_be32(SHA224_H6), cpu_to_be32(SHA224_H7),
33c11baa02STom Lendacky };
34c11baa02STom Lendacky
354b394a23SGary R Hook static const __be32 ccp_sha256_init[SHA256_DIGEST_SIZE / sizeof(__be32)] = {
36c11baa02STom Lendacky cpu_to_be32(SHA256_H0), cpu_to_be32(SHA256_H1),
37c11baa02STom Lendacky cpu_to_be32(SHA256_H2), cpu_to_be32(SHA256_H3),
38c11baa02STom Lendacky cpu_to_be32(SHA256_H4), cpu_to_be32(SHA256_H5),
39c11baa02STom Lendacky cpu_to_be32(SHA256_H6), cpu_to_be32(SHA256_H7),
40c11baa02STom Lendacky };
41c11baa02STom Lendacky
42ccebcf3fSGary R Hook static const __be64 ccp_sha384_init[SHA512_DIGEST_SIZE / sizeof(__be64)] = {
43ccebcf3fSGary R Hook cpu_to_be64(SHA384_H0), cpu_to_be64(SHA384_H1),
44ccebcf3fSGary R Hook cpu_to_be64(SHA384_H2), cpu_to_be64(SHA384_H3),
45ccebcf3fSGary R Hook cpu_to_be64(SHA384_H4), cpu_to_be64(SHA384_H5),
46ccebcf3fSGary R Hook cpu_to_be64(SHA384_H6), cpu_to_be64(SHA384_H7),
47ccebcf3fSGary R Hook };
48ccebcf3fSGary R Hook
49ccebcf3fSGary R Hook static const __be64 ccp_sha512_init[SHA512_DIGEST_SIZE / sizeof(__be64)] = {
50ccebcf3fSGary R Hook cpu_to_be64(SHA512_H0), cpu_to_be64(SHA512_H1),
51ccebcf3fSGary R Hook cpu_to_be64(SHA512_H2), cpu_to_be64(SHA512_H3),
52ccebcf3fSGary R Hook cpu_to_be64(SHA512_H4), cpu_to_be64(SHA512_H5),
53ccebcf3fSGary R Hook cpu_to_be64(SHA512_H6), cpu_to_be64(SHA512_H7),
54ccebcf3fSGary R Hook };
55ccebcf3fSGary R Hook
564b394a23SGary R Hook #define CCP_NEW_JOBID(ccp) ((ccp->vdata->version == CCP_VERSION(3, 0)) ? \
574b394a23SGary R Hook ccp_gen_jobid(ccp) : 0)
584b394a23SGary R Hook
ccp_gen_jobid(struct ccp_device * ccp)5963b94509STom Lendacky static u32 ccp_gen_jobid(struct ccp_device *ccp)
6063b94509STom Lendacky {
6163b94509STom Lendacky return atomic_inc_return(&ccp->current_id) & CCP_JOBID_MASK;
6263b94509STom Lendacky }
6363b94509STom Lendacky
ccp_sg_free(struct ccp_sg_workarea * wa)6463b94509STom Lendacky static void ccp_sg_free(struct ccp_sg_workarea *wa)
6563b94509STom Lendacky {
6663b94509STom Lendacky if (wa->dma_count)
678a302808SJohn Allen dma_unmap_sg(wa->dma_dev, wa->dma_sg_head, wa->nents, wa->dma_dir);
6863b94509STom Lendacky
6963b94509STom Lendacky wa->dma_count = 0;
7063b94509STom Lendacky }
7163b94509STom Lendacky
ccp_init_sg_workarea(struct ccp_sg_workarea * wa,struct device * dev,struct scatterlist * sg,u64 len,enum dma_data_direction dma_dir)7263b94509STom Lendacky static int ccp_init_sg_workarea(struct ccp_sg_workarea *wa, struct device *dev,
7381a59f00STom Lendacky struct scatterlist *sg, u64 len,
7463b94509STom Lendacky enum dma_data_direction dma_dir)
7563b94509STom Lendacky {
7663b94509STom Lendacky memset(wa, 0, sizeof(*wa));
7763b94509STom Lendacky
7863b94509STom Lendacky wa->sg = sg;
7963b94509STom Lendacky if (!sg)
8063b94509STom Lendacky return 0;
8163b94509STom Lendacky
82fb43f694STom Lendacky wa->nents = sg_nents_for_len(sg, len);
83fb43f694STom Lendacky if (wa->nents < 0)
84fb43f694STom Lendacky return wa->nents;
85fb43f694STom Lendacky
8663b94509STom Lendacky wa->bytes_left = len;
8763b94509STom Lendacky wa->sg_used = 0;
8863b94509STom Lendacky
8963b94509STom Lendacky if (len == 0)
9063b94509STom Lendacky return 0;
9163b94509STom Lendacky
9263b94509STom Lendacky if (dma_dir == DMA_NONE)
9363b94509STom Lendacky return 0;
9463b94509STom Lendacky
9563b94509STom Lendacky wa->dma_sg = sg;
968a302808SJohn Allen wa->dma_sg_head = sg;
9763b94509STom Lendacky wa->dma_dev = dev;
9863b94509STom Lendacky wa->dma_dir = dma_dir;
9963b94509STom Lendacky wa->dma_count = dma_map_sg(dev, sg, wa->nents, dma_dir);
10063b94509STom Lendacky if (!wa->dma_count)
10163b94509STom Lendacky return -ENOMEM;
10263b94509STom Lendacky
10363b94509STom Lendacky return 0;
10463b94509STom Lendacky }
10563b94509STom Lendacky
ccp_update_sg_workarea(struct ccp_sg_workarea * wa,unsigned int len)10663b94509STom Lendacky static void ccp_update_sg_workarea(struct ccp_sg_workarea *wa, unsigned int len)
10763b94509STom Lendacky {
10881a59f00STom Lendacky unsigned int nbytes = min_t(u64, len, wa->bytes_left);
1098a302808SJohn Allen unsigned int sg_combined_len = 0;
11063b94509STom Lendacky
11163b94509STom Lendacky if (!wa->sg)
11263b94509STom Lendacky return;
11363b94509STom Lendacky
11463b94509STom Lendacky wa->sg_used += nbytes;
11563b94509STom Lendacky wa->bytes_left -= nbytes;
1168a302808SJohn Allen if (wa->sg_used == sg_dma_len(wa->dma_sg)) {
1178a302808SJohn Allen /* Advance to the next DMA scatterlist entry */
1188a302808SJohn Allen wa->dma_sg = sg_next(wa->dma_sg);
1198a302808SJohn Allen
1208a302808SJohn Allen /* In the case that the DMA mapped scatterlist has entries
1218a302808SJohn Allen * that have been merged, the non-DMA mapped scatterlist
1228a302808SJohn Allen * must be advanced multiple times for each merged entry.
1238a302808SJohn Allen * This ensures that the current non-DMA mapped entry
1248a302808SJohn Allen * corresponds to the current DMA mapped entry.
1258a302808SJohn Allen */
1268a302808SJohn Allen do {
1278a302808SJohn Allen sg_combined_len += wa->sg->length;
12863b94509STom Lendacky wa->sg = sg_next(wa->sg);
1298a302808SJohn Allen } while (wa->sg_used > sg_combined_len);
1308a302808SJohn Allen
13163b94509STom Lendacky wa->sg_used = 0;
13263b94509STom Lendacky }
13363b94509STom Lendacky }
13463b94509STom Lendacky
ccp_dm_free(struct ccp_dm_workarea * wa)13563b94509STom Lendacky static void ccp_dm_free(struct ccp_dm_workarea *wa)
13663b94509STom Lendacky {
13763b94509STom Lendacky if (wa->length <= CCP_DMAPOOL_MAX_SIZE) {
13863b94509STom Lendacky if (wa->address)
13963b94509STom Lendacky dma_pool_free(wa->dma_pool, wa->address,
14063b94509STom Lendacky wa->dma.address);
14163b94509STom Lendacky } else {
14263b94509STom Lendacky if (wa->dma.address)
14363b94509STom Lendacky dma_unmap_single(wa->dev, wa->dma.address, wa->length,
14463b94509STom Lendacky wa->dma.dir);
14563b94509STom Lendacky kfree(wa->address);
14663b94509STom Lendacky }
14763b94509STom Lendacky
14863b94509STom Lendacky wa->address = NULL;
14963b94509STom Lendacky wa->dma.address = 0;
15063b94509STom Lendacky }
15163b94509STom Lendacky
ccp_init_dm_workarea(struct ccp_dm_workarea * wa,struct ccp_cmd_queue * cmd_q,unsigned int len,enum dma_data_direction dir)15263b94509STom Lendacky static int ccp_init_dm_workarea(struct ccp_dm_workarea *wa,
15363b94509STom Lendacky struct ccp_cmd_queue *cmd_q,
15463b94509STom Lendacky unsigned int len,
15563b94509STom Lendacky enum dma_data_direction dir)
15663b94509STom Lendacky {
15763b94509STom Lendacky memset(wa, 0, sizeof(*wa));
15863b94509STom Lendacky
15963b94509STom Lendacky if (!len)
16063b94509STom Lendacky return 0;
16163b94509STom Lendacky
16263b94509STom Lendacky wa->dev = cmd_q->ccp->dev;
16363b94509STom Lendacky wa->length = len;
16463b94509STom Lendacky
16563b94509STom Lendacky if (len <= CCP_DMAPOOL_MAX_SIZE) {
16663b94509STom Lendacky wa->dma_pool = cmd_q->dma_pool;
16763b94509STom Lendacky
168bfb5eb08SChuhong Yuan wa->address = dma_pool_zalloc(wa->dma_pool, GFP_KERNEL,
16963b94509STom Lendacky &wa->dma.address);
17063b94509STom Lendacky if (!wa->address)
17163b94509STom Lendacky return -ENOMEM;
17263b94509STom Lendacky
17363b94509STom Lendacky wa->dma.length = CCP_DMAPOOL_MAX_SIZE;
17463b94509STom Lendacky
17563b94509STom Lendacky } else {
17663b94509STom Lendacky wa->address = kzalloc(len, GFP_KERNEL);
17763b94509STom Lendacky if (!wa->address)
17863b94509STom Lendacky return -ENOMEM;
17963b94509STom Lendacky
18063b94509STom Lendacky wa->dma.address = dma_map_single(wa->dev, wa->address, len,
18163b94509STom Lendacky dir);
182*1e9d7072SDinghao Liu if (dma_mapping_error(wa->dev, wa->dma.address)) {
183*1e9d7072SDinghao Liu kfree(wa->address);
184*1e9d7072SDinghao Liu wa->address = NULL;
18563b94509STom Lendacky return -ENOMEM;
186*1e9d7072SDinghao Liu }
18763b94509STom Lendacky
18863b94509STom Lendacky wa->dma.length = len;
18963b94509STom Lendacky }
19063b94509STom Lendacky wa->dma.dir = dir;
19163b94509STom Lendacky
19263b94509STom Lendacky return 0;
19363b94509STom Lendacky }
19463b94509STom Lendacky
ccp_set_dm_area(struct ccp_dm_workarea * wa,unsigned int wa_offset,struct scatterlist * sg,unsigned int sg_offset,unsigned int len)195b698a9f4SGary R Hook static int ccp_set_dm_area(struct ccp_dm_workarea *wa, unsigned int wa_offset,
19663b94509STom Lendacky struct scatterlist *sg, unsigned int sg_offset,
19763b94509STom Lendacky unsigned int len)
19863b94509STom Lendacky {
19963b94509STom Lendacky WARN_ON(!wa->address);
20063b94509STom Lendacky
201b698a9f4SGary R Hook if (len > (wa->length - wa_offset))
202b698a9f4SGary R Hook return -EINVAL;
203b698a9f4SGary R Hook
20463b94509STom Lendacky scatterwalk_map_and_copy(wa->address + wa_offset, sg, sg_offset, len,
20563b94509STom Lendacky 0);
206b698a9f4SGary R Hook return 0;
20763b94509STom Lendacky }
20863b94509STom Lendacky
ccp_get_dm_area(struct ccp_dm_workarea * wa,unsigned int wa_offset,struct scatterlist * sg,unsigned int sg_offset,unsigned int len)20963b94509STom Lendacky static void ccp_get_dm_area(struct ccp_dm_workarea *wa, unsigned int wa_offset,
21063b94509STom Lendacky struct scatterlist *sg, unsigned int sg_offset,
21163b94509STom Lendacky unsigned int len)
21263b94509STom Lendacky {
21363b94509STom Lendacky WARN_ON(!wa->address);
21463b94509STom Lendacky
21563b94509STom Lendacky scatterwalk_map_and_copy(wa->address + wa_offset, sg, sg_offset, len,
21663b94509STom Lendacky 1);
21763b94509STom Lendacky }
21863b94509STom Lendacky
ccp_reverse_set_dm_area(struct ccp_dm_workarea * wa,unsigned int wa_offset,struct scatterlist * sg,unsigned int sg_offset,unsigned int len)219355eba5dSTom Lendacky static int ccp_reverse_set_dm_area(struct ccp_dm_workarea *wa,
22083d650abSGary R Hook unsigned int wa_offset,
22163b94509STom Lendacky struct scatterlist *sg,
22283d650abSGary R Hook unsigned int sg_offset,
22383d650abSGary R Hook unsigned int len)
22463b94509STom Lendacky {
22583d650abSGary R Hook u8 *p, *q;
226b698a9f4SGary R Hook int rc;
22763b94509STom Lendacky
228b698a9f4SGary R Hook rc = ccp_set_dm_area(wa, wa_offset, sg, sg_offset, len);
229b698a9f4SGary R Hook if (rc)
230b698a9f4SGary R Hook return rc;
23163b94509STom Lendacky
23283d650abSGary R Hook p = wa->address + wa_offset;
23383d650abSGary R Hook q = p + len - 1;
23483d650abSGary R Hook while (p < q) {
23583d650abSGary R Hook *p = *p ^ *q;
23683d650abSGary R Hook *q = *p ^ *q;
23783d650abSGary R Hook *p = *p ^ *q;
23883d650abSGary R Hook p++;
23983d650abSGary R Hook q--;
24063b94509STom Lendacky }
241355eba5dSTom Lendacky return 0;
24263b94509STom Lendacky }
24363b94509STom Lendacky
ccp_reverse_get_dm_area(struct ccp_dm_workarea * wa,unsigned int wa_offset,struct scatterlist * sg,unsigned int sg_offset,unsigned int len)24463b94509STom Lendacky static void ccp_reverse_get_dm_area(struct ccp_dm_workarea *wa,
24583d650abSGary R Hook unsigned int wa_offset,
24663b94509STom Lendacky struct scatterlist *sg,
24783d650abSGary R Hook unsigned int sg_offset,
24863b94509STom Lendacky unsigned int len)
24963b94509STom Lendacky {
25083d650abSGary R Hook u8 *p, *q;
25163b94509STom Lendacky
25283d650abSGary R Hook p = wa->address + wa_offset;
25383d650abSGary R Hook q = p + len - 1;
25483d650abSGary R Hook while (p < q) {
25583d650abSGary R Hook *p = *p ^ *q;
25683d650abSGary R Hook *q = *p ^ *q;
25783d650abSGary R Hook *p = *p ^ *q;
25883d650abSGary R Hook p++;
25983d650abSGary R Hook q--;
26063b94509STom Lendacky }
26183d650abSGary R Hook
26283d650abSGary R Hook ccp_get_dm_area(wa, wa_offset, sg, sg_offset, len);
26363b94509STom Lendacky }
26463b94509STom Lendacky
ccp_free_data(struct ccp_data * data,struct ccp_cmd_queue * cmd_q)26563b94509STom Lendacky static void ccp_free_data(struct ccp_data *data, struct ccp_cmd_queue *cmd_q)
26663b94509STom Lendacky {
26763b94509STom Lendacky ccp_dm_free(&data->dm_wa);
26863b94509STom Lendacky ccp_sg_free(&data->sg_wa);
26963b94509STom Lendacky }
27063b94509STom Lendacky
ccp_init_data(struct ccp_data * data,struct ccp_cmd_queue * cmd_q,struct scatterlist * sg,u64 sg_len,unsigned int dm_len,enum dma_data_direction dir)27163b94509STom Lendacky static int ccp_init_data(struct ccp_data *data, struct ccp_cmd_queue *cmd_q,
27281a59f00STom Lendacky struct scatterlist *sg, u64 sg_len,
27363b94509STom Lendacky unsigned int dm_len,
27463b94509STom Lendacky enum dma_data_direction dir)
27563b94509STom Lendacky {
27663b94509STom Lendacky int ret;
27763b94509STom Lendacky
27863b94509STom Lendacky memset(data, 0, sizeof(*data));
27963b94509STom Lendacky
28063b94509STom Lendacky ret = ccp_init_sg_workarea(&data->sg_wa, cmd_q->ccp->dev, sg, sg_len,
28163b94509STom Lendacky dir);
28263b94509STom Lendacky if (ret)
28363b94509STom Lendacky goto e_err;
28463b94509STom Lendacky
28563b94509STom Lendacky ret = ccp_init_dm_workarea(&data->dm_wa, cmd_q, dm_len, dir);
28663b94509STom Lendacky if (ret)
28763b94509STom Lendacky goto e_err;
28863b94509STom Lendacky
28963b94509STom Lendacky return 0;
29063b94509STom Lendacky
29163b94509STom Lendacky e_err:
29263b94509STom Lendacky ccp_free_data(data, cmd_q);
29363b94509STom Lendacky
29463b94509STom Lendacky return ret;
29563b94509STom Lendacky }
29663b94509STom Lendacky
ccp_queue_buf(struct ccp_data * data,unsigned int from)29763b94509STom Lendacky static unsigned int ccp_queue_buf(struct ccp_data *data, unsigned int from)
29863b94509STom Lendacky {
29963b94509STom Lendacky struct ccp_sg_workarea *sg_wa = &data->sg_wa;
30063b94509STom Lendacky struct ccp_dm_workarea *dm_wa = &data->dm_wa;
30163b94509STom Lendacky unsigned int buf_count, nbytes;
30263b94509STom Lendacky
30363b94509STom Lendacky /* Clear the buffer if setting it */
30463b94509STom Lendacky if (!from)
30563b94509STom Lendacky memset(dm_wa->address, 0, dm_wa->length);
30663b94509STom Lendacky
30763b94509STom Lendacky if (!sg_wa->sg)
30863b94509STom Lendacky return 0;
30963b94509STom Lendacky
31081a59f00STom Lendacky /* Perform the copy operation
31181a59f00STom Lendacky * nbytes will always be <= UINT_MAX because dm_wa->length is
31281a59f00STom Lendacky * an unsigned int
31381a59f00STom Lendacky */
31481a59f00STom Lendacky nbytes = min_t(u64, sg_wa->bytes_left, dm_wa->length);
31563b94509STom Lendacky scatterwalk_map_and_copy(dm_wa->address, sg_wa->sg, sg_wa->sg_used,
31663b94509STom Lendacky nbytes, from);
31763b94509STom Lendacky
31863b94509STom Lendacky /* Update the structures and generate the count */
31963b94509STom Lendacky buf_count = 0;
32063b94509STom Lendacky while (sg_wa->bytes_left && (buf_count < dm_wa->length)) {
3218a302808SJohn Allen nbytes = min(sg_dma_len(sg_wa->dma_sg) - sg_wa->sg_used,
32281a59f00STom Lendacky dm_wa->length - buf_count);
32381a59f00STom Lendacky nbytes = min_t(u64, sg_wa->bytes_left, nbytes);
32463b94509STom Lendacky
32563b94509STom Lendacky buf_count += nbytes;
32663b94509STom Lendacky ccp_update_sg_workarea(sg_wa, nbytes);
32763b94509STom Lendacky }
32863b94509STom Lendacky
32963b94509STom Lendacky return buf_count;
33063b94509STom Lendacky }
33163b94509STom Lendacky
ccp_fill_queue_buf(struct ccp_data * data)33263b94509STom Lendacky static unsigned int ccp_fill_queue_buf(struct ccp_data *data)
33363b94509STom Lendacky {
33463b94509STom Lendacky return ccp_queue_buf(data, 0);
33563b94509STom Lendacky }
33663b94509STom Lendacky
ccp_empty_queue_buf(struct ccp_data * data)33763b94509STom Lendacky static unsigned int ccp_empty_queue_buf(struct ccp_data *data)
33863b94509STom Lendacky {
33963b94509STom Lendacky return ccp_queue_buf(data, 1);
34063b94509STom Lendacky }
34163b94509STom Lendacky
ccp_prepare_data(struct ccp_data * src,struct ccp_data * dst,struct ccp_op * op,unsigned int block_size,bool blocksize_op)34263b94509STom Lendacky static void ccp_prepare_data(struct ccp_data *src, struct ccp_data *dst,
34363b94509STom Lendacky struct ccp_op *op, unsigned int block_size,
34463b94509STom Lendacky bool blocksize_op)
34563b94509STom Lendacky {
34663b94509STom Lendacky unsigned int sg_src_len, sg_dst_len, op_len;
34763b94509STom Lendacky
34863b94509STom Lendacky /* The CCP can only DMA from/to one address each per operation. This
34963b94509STom Lendacky * requires that we find the smallest DMA area between the source
35081a59f00STom Lendacky * and destination. The resulting len values will always be <= UINT_MAX
35181a59f00STom Lendacky * because the dma length is an unsigned int.
35263b94509STom Lendacky */
3538a302808SJohn Allen sg_src_len = sg_dma_len(src->sg_wa.dma_sg) - src->sg_wa.sg_used;
35481a59f00STom Lendacky sg_src_len = min_t(u64, src->sg_wa.bytes_left, sg_src_len);
35563b94509STom Lendacky
35663b94509STom Lendacky if (dst) {
3578a302808SJohn Allen sg_dst_len = sg_dma_len(dst->sg_wa.dma_sg) - dst->sg_wa.sg_used;
35881a59f00STom Lendacky sg_dst_len = min_t(u64, src->sg_wa.bytes_left, sg_dst_len);
35963b94509STom Lendacky op_len = min(sg_src_len, sg_dst_len);
3608db88467STom Lendacky } else {
36163b94509STom Lendacky op_len = sg_src_len;
3628db88467STom Lendacky }
36363b94509STom Lendacky
36463b94509STom Lendacky /* The data operation length will be at least block_size in length
36563b94509STom Lendacky * or the smaller of available sg room remaining for the source or
36663b94509STom Lendacky * the destination
36763b94509STom Lendacky */
36863b94509STom Lendacky op_len = max(op_len, block_size);
36963b94509STom Lendacky
37063b94509STom Lendacky /* Unless we have to buffer data, there's no reason to wait */
37163b94509STom Lendacky op->soc = 0;
37263b94509STom Lendacky
37363b94509STom Lendacky if (sg_src_len < block_size) {
37463b94509STom Lendacky /* Not enough data in the sg element, so it
37563b94509STom Lendacky * needs to be buffered into a blocksize chunk
37663b94509STom Lendacky */
37763b94509STom Lendacky int cp_len = ccp_fill_queue_buf(src);
37863b94509STom Lendacky
37963b94509STom Lendacky op->soc = 1;
38063b94509STom Lendacky op->src.u.dma.address = src->dm_wa.dma.address;
38163b94509STom Lendacky op->src.u.dma.offset = 0;
38263b94509STom Lendacky op->src.u.dma.length = (blocksize_op) ? block_size : cp_len;
38363b94509STom Lendacky } else {
38463b94509STom Lendacky /* Enough data in the sg element, but we need to
38563b94509STom Lendacky * adjust for any previously copied data
38663b94509STom Lendacky */
3878a302808SJohn Allen op->src.u.dma.address = sg_dma_address(src->sg_wa.dma_sg);
38863b94509STom Lendacky op->src.u.dma.offset = src->sg_wa.sg_used;
38963b94509STom Lendacky op->src.u.dma.length = op_len & ~(block_size - 1);
39063b94509STom Lendacky
39163b94509STom Lendacky ccp_update_sg_workarea(&src->sg_wa, op->src.u.dma.length);
39263b94509STom Lendacky }
39363b94509STom Lendacky
39463b94509STom Lendacky if (dst) {
39563b94509STom Lendacky if (sg_dst_len < block_size) {
39663b94509STom Lendacky /* Not enough room in the sg element or we're on the
39763b94509STom Lendacky * last piece of data (when using padding), so the
39863b94509STom Lendacky * output needs to be buffered into a blocksize chunk
39963b94509STom Lendacky */
40063b94509STom Lendacky op->soc = 1;
40163b94509STom Lendacky op->dst.u.dma.address = dst->dm_wa.dma.address;
40263b94509STom Lendacky op->dst.u.dma.offset = 0;
40363b94509STom Lendacky op->dst.u.dma.length = op->src.u.dma.length;
40463b94509STom Lendacky } else {
40563b94509STom Lendacky /* Enough room in the sg element, but we need to
40663b94509STom Lendacky * adjust for any previously used area
40763b94509STom Lendacky */
4088a302808SJohn Allen op->dst.u.dma.address = sg_dma_address(dst->sg_wa.dma_sg);
40963b94509STom Lendacky op->dst.u.dma.offset = dst->sg_wa.sg_used;
41063b94509STom Lendacky op->dst.u.dma.length = op->src.u.dma.length;
41163b94509STom Lendacky }
41263b94509STom Lendacky }
41363b94509STom Lendacky }
41463b94509STom Lendacky
ccp_process_data(struct ccp_data * src,struct ccp_data * dst,struct ccp_op * op)41563b94509STom Lendacky static void ccp_process_data(struct ccp_data *src, struct ccp_data *dst,
41663b94509STom Lendacky struct ccp_op *op)
41763b94509STom Lendacky {
41863b94509STom Lendacky op->init = 0;
41963b94509STom Lendacky
42063b94509STom Lendacky if (dst) {
42163b94509STom Lendacky if (op->dst.u.dma.address == dst->dm_wa.dma.address)
42263b94509STom Lendacky ccp_empty_queue_buf(dst);
42363b94509STom Lendacky else
42463b94509STom Lendacky ccp_update_sg_workarea(&dst->sg_wa,
42563b94509STom Lendacky op->dst.u.dma.length);
42663b94509STom Lendacky }
42763b94509STom Lendacky }
42863b94509STom Lendacky
ccp_copy_to_from_sb(struct ccp_cmd_queue * cmd_q,struct ccp_dm_workarea * wa,u32 jobid,u32 sb,u32 byte_swap,bool from)429956ee21aSGary R Hook static int ccp_copy_to_from_sb(struct ccp_cmd_queue *cmd_q,
430956ee21aSGary R Hook struct ccp_dm_workarea *wa, u32 jobid, u32 sb,
43163b94509STom Lendacky u32 byte_swap, bool from)
43263b94509STom Lendacky {
43363b94509STom Lendacky struct ccp_op op;
43463b94509STom Lendacky
43563b94509STom Lendacky memset(&op, 0, sizeof(op));
43663b94509STom Lendacky
43763b94509STom Lendacky op.cmd_q = cmd_q;
43863b94509STom Lendacky op.jobid = jobid;
43963b94509STom Lendacky op.eom = 1;
44063b94509STom Lendacky
44163b94509STom Lendacky if (from) {
44263b94509STom Lendacky op.soc = 1;
443956ee21aSGary R Hook op.src.type = CCP_MEMTYPE_SB;
444956ee21aSGary R Hook op.src.u.sb = sb;
44563b94509STom Lendacky op.dst.type = CCP_MEMTYPE_SYSTEM;
44663b94509STom Lendacky op.dst.u.dma.address = wa->dma.address;
44763b94509STom Lendacky op.dst.u.dma.length = wa->length;
44863b94509STom Lendacky } else {
44963b94509STom Lendacky op.src.type = CCP_MEMTYPE_SYSTEM;
45063b94509STom Lendacky op.src.u.dma.address = wa->dma.address;
45163b94509STom Lendacky op.src.u.dma.length = wa->length;
452956ee21aSGary R Hook op.dst.type = CCP_MEMTYPE_SB;
453956ee21aSGary R Hook op.dst.u.sb = sb;
45463b94509STom Lendacky }
45563b94509STom Lendacky
45663b94509STom Lendacky op.u.passthru.byte_swap = byte_swap;
45763b94509STom Lendacky
458a43eb985SGary R Hook return cmd_q->ccp->vdata->perform->passthru(&op);
45963b94509STom Lendacky }
46063b94509STom Lendacky
ccp_copy_to_sb(struct ccp_cmd_queue * cmd_q,struct ccp_dm_workarea * wa,u32 jobid,u32 sb,u32 byte_swap)461956ee21aSGary R Hook static int ccp_copy_to_sb(struct ccp_cmd_queue *cmd_q,
462956ee21aSGary R Hook struct ccp_dm_workarea *wa, u32 jobid, u32 sb,
46363b94509STom Lendacky u32 byte_swap)
46463b94509STom Lendacky {
465956ee21aSGary R Hook return ccp_copy_to_from_sb(cmd_q, wa, jobid, sb, byte_swap, false);
46663b94509STom Lendacky }
46763b94509STom Lendacky
ccp_copy_from_sb(struct ccp_cmd_queue * cmd_q,struct ccp_dm_workarea * wa,u32 jobid,u32 sb,u32 byte_swap)468956ee21aSGary R Hook static int ccp_copy_from_sb(struct ccp_cmd_queue *cmd_q,
469956ee21aSGary R Hook struct ccp_dm_workarea *wa, u32 jobid, u32 sb,
47063b94509STom Lendacky u32 byte_swap)
47163b94509STom Lendacky {
472956ee21aSGary R Hook return ccp_copy_to_from_sb(cmd_q, wa, jobid, sb, byte_swap, true);
47363b94509STom Lendacky }
47463b94509STom Lendacky
47572c8117aSArnd Bergmann static noinline_for_stack int
ccp_run_aes_cmac_cmd(struct ccp_cmd_queue * cmd_q,struct ccp_cmd * cmd)47672c8117aSArnd Bergmann ccp_run_aes_cmac_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
47763b94509STom Lendacky {
47863b94509STom Lendacky struct ccp_aes_engine *aes = &cmd->u.aes;
47963b94509STom Lendacky struct ccp_dm_workarea key, ctx;
48063b94509STom Lendacky struct ccp_data src;
48163b94509STom Lendacky struct ccp_op op;
48263b94509STom Lendacky unsigned int dm_offset;
48363b94509STom Lendacky int ret;
48463b94509STom Lendacky
48563b94509STom Lendacky if (!((aes->key_len == AES_KEYSIZE_128) ||
48663b94509STom Lendacky (aes->key_len == AES_KEYSIZE_192) ||
48763b94509STom Lendacky (aes->key_len == AES_KEYSIZE_256)))
48863b94509STom Lendacky return -EINVAL;
48963b94509STom Lendacky
49063b94509STom Lendacky if (aes->src_len & (AES_BLOCK_SIZE - 1))
49163b94509STom Lendacky return -EINVAL;
49263b94509STom Lendacky
49363b94509STom Lendacky if (aes->iv_len != AES_BLOCK_SIZE)
49463b94509STom Lendacky return -EINVAL;
49563b94509STom Lendacky
49663b94509STom Lendacky if (!aes->key || !aes->iv || !aes->src)
49763b94509STom Lendacky return -EINVAL;
49863b94509STom Lendacky
49963b94509STom Lendacky if (aes->cmac_final) {
50063b94509STom Lendacky if (aes->cmac_key_len != AES_BLOCK_SIZE)
50163b94509STom Lendacky return -EINVAL;
50263b94509STom Lendacky
50363b94509STom Lendacky if (!aes->cmac_key)
50463b94509STom Lendacky return -EINVAL;
50563b94509STom Lendacky }
50663b94509STom Lendacky
507956ee21aSGary R Hook BUILD_BUG_ON(CCP_AES_KEY_SB_COUNT != 1);
508956ee21aSGary R Hook BUILD_BUG_ON(CCP_AES_CTX_SB_COUNT != 1);
50963b94509STom Lendacky
51063b94509STom Lendacky ret = -EIO;
51163b94509STom Lendacky memset(&op, 0, sizeof(op));
51263b94509STom Lendacky op.cmd_q = cmd_q;
5134b394a23SGary R Hook op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
514956ee21aSGary R Hook op.sb_key = cmd_q->sb_key;
515956ee21aSGary R Hook op.sb_ctx = cmd_q->sb_ctx;
51663b94509STom Lendacky op.init = 1;
51763b94509STom Lendacky op.u.aes.type = aes->type;
51863b94509STom Lendacky op.u.aes.mode = aes->mode;
51963b94509STom Lendacky op.u.aes.action = aes->action;
52063b94509STom Lendacky
521956ee21aSGary R Hook /* All supported key sizes fit in a single (32-byte) SB entry
52263b94509STom Lendacky * and must be in little endian format. Use the 256-bit byte
52363b94509STom Lendacky * swap passthru option to convert from big endian to little
52463b94509STom Lendacky * endian.
52563b94509STom Lendacky */
52663b94509STom Lendacky ret = ccp_init_dm_workarea(&key, cmd_q,
527956ee21aSGary R Hook CCP_AES_KEY_SB_COUNT * CCP_SB_BYTES,
52863b94509STom Lendacky DMA_TO_DEVICE);
52963b94509STom Lendacky if (ret)
53063b94509STom Lendacky return ret;
53163b94509STom Lendacky
532956ee21aSGary R Hook dm_offset = CCP_SB_BYTES - aes->key_len;
533b698a9f4SGary R Hook ret = ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len);
534b698a9f4SGary R Hook if (ret)
535b698a9f4SGary R Hook goto e_key;
536956ee21aSGary R Hook ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
53763b94509STom Lendacky CCP_PASSTHRU_BYTESWAP_256BIT);
53863b94509STom Lendacky if (ret) {
53963b94509STom Lendacky cmd->engine_error = cmd_q->cmd_error;
54063b94509STom Lendacky goto e_key;
54163b94509STom Lendacky }
54263b94509STom Lendacky
543956ee21aSGary R Hook /* The AES context fits in a single (32-byte) SB entry and
54463b94509STom Lendacky * must be in little endian format. Use the 256-bit byte swap
54563b94509STom Lendacky * passthru option to convert from big endian to little endian.
54663b94509STom Lendacky */
54763b94509STom Lendacky ret = ccp_init_dm_workarea(&ctx, cmd_q,
548956ee21aSGary R Hook CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES,
54963b94509STom Lendacky DMA_BIDIRECTIONAL);
55063b94509STom Lendacky if (ret)
55163b94509STom Lendacky goto e_key;
55263b94509STom Lendacky
553956ee21aSGary R Hook dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
554b698a9f4SGary R Hook ret = ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
555b698a9f4SGary R Hook if (ret)
556b698a9f4SGary R Hook goto e_ctx;
557956ee21aSGary R Hook ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
55863b94509STom Lendacky CCP_PASSTHRU_BYTESWAP_256BIT);
55963b94509STom Lendacky if (ret) {
56063b94509STom Lendacky cmd->engine_error = cmd_q->cmd_error;
56163b94509STom Lendacky goto e_ctx;
56263b94509STom Lendacky }
56363b94509STom Lendacky
56463b94509STom Lendacky /* Send data to the CCP AES engine */
56563b94509STom Lendacky ret = ccp_init_data(&src, cmd_q, aes->src, aes->src_len,
56663b94509STom Lendacky AES_BLOCK_SIZE, DMA_TO_DEVICE);
56763b94509STom Lendacky if (ret)
56863b94509STom Lendacky goto e_ctx;
56963b94509STom Lendacky
57063b94509STom Lendacky while (src.sg_wa.bytes_left) {
57163b94509STom Lendacky ccp_prepare_data(&src, NULL, &op, AES_BLOCK_SIZE, true);
57263b94509STom Lendacky if (aes->cmac_final && !src.sg_wa.bytes_left) {
57363b94509STom Lendacky op.eom = 1;
57463b94509STom Lendacky
57563b94509STom Lendacky /* Push the K1/K2 key to the CCP now */
576956ee21aSGary R Hook ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid,
577956ee21aSGary R Hook op.sb_ctx,
57863b94509STom Lendacky CCP_PASSTHRU_BYTESWAP_256BIT);
57963b94509STom Lendacky if (ret) {
58063b94509STom Lendacky cmd->engine_error = cmd_q->cmd_error;
58163b94509STom Lendacky goto e_src;
58263b94509STom Lendacky }
58363b94509STom Lendacky
584b698a9f4SGary R Hook ret = ccp_set_dm_area(&ctx, 0, aes->cmac_key, 0,
58563b94509STom Lendacky aes->cmac_key_len);
586b698a9f4SGary R Hook if (ret)
587b698a9f4SGary R Hook goto e_src;
588956ee21aSGary R Hook ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
58963b94509STom Lendacky CCP_PASSTHRU_BYTESWAP_256BIT);
59063b94509STom Lendacky if (ret) {
59163b94509STom Lendacky cmd->engine_error = cmd_q->cmd_error;
59263b94509STom Lendacky goto e_src;
59363b94509STom Lendacky }
59463b94509STom Lendacky }
59563b94509STom Lendacky
596a43eb985SGary R Hook ret = cmd_q->ccp->vdata->perform->aes(&op);
59763b94509STom Lendacky if (ret) {
59863b94509STom Lendacky cmd->engine_error = cmd_q->cmd_error;
59963b94509STom Lendacky goto e_src;
60063b94509STom Lendacky }
60163b94509STom Lendacky
60263b94509STom Lendacky ccp_process_data(&src, NULL, &op);
60363b94509STom Lendacky }
60463b94509STom Lendacky
60563b94509STom Lendacky /* Retrieve the AES context - convert from LE to BE using
60663b94509STom Lendacky * 32-byte (256-bit) byteswapping
60763b94509STom Lendacky */
608956ee21aSGary R Hook ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
60963b94509STom Lendacky CCP_PASSTHRU_BYTESWAP_256BIT);
61063b94509STom Lendacky if (ret) {
61163b94509STom Lendacky cmd->engine_error = cmd_q->cmd_error;
61263b94509STom Lendacky goto e_src;
61363b94509STom Lendacky }
61463b94509STom Lendacky
61563b94509STom Lendacky /* ...but we only need AES_BLOCK_SIZE bytes */
616956ee21aSGary R Hook dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
61763b94509STom Lendacky ccp_get_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
61863b94509STom Lendacky
61963b94509STom Lendacky e_src:
62063b94509STom Lendacky ccp_free_data(&src, cmd_q);
62163b94509STom Lendacky
62263b94509STom Lendacky e_ctx:
62363b94509STom Lendacky ccp_dm_free(&ctx);
62463b94509STom Lendacky
62563b94509STom Lendacky e_key:
62663b94509STom Lendacky ccp_dm_free(&key);
62763b94509STom Lendacky
62863b94509STom Lendacky return ret;
62963b94509STom Lendacky }
63063b94509STom Lendacky
63172c8117aSArnd Bergmann static noinline_for_stack int
ccp_run_aes_gcm_cmd(struct ccp_cmd_queue * cmd_q,struct ccp_cmd * cmd)63272c8117aSArnd Bergmann ccp_run_aes_gcm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
63336cf515bSGary R Hook {
63436cf515bSGary R Hook struct ccp_aes_engine *aes = &cmd->u.aes;
63536cf515bSGary R Hook struct ccp_dm_workarea key, ctx, final_wa, tag;
63636cf515bSGary R Hook struct ccp_data src, dst;
63736cf515bSGary R Hook struct ccp_data aad;
63836cf515bSGary R Hook struct ccp_op op;
63936cf515bSGary R Hook unsigned int dm_offset;
6409f00baf7SGary R Hook unsigned int authsize;
64120e833dcSHook, Gary unsigned int jobid;
64236cf515bSGary R Hook unsigned int ilen;
64336cf515bSGary R Hook bool in_place = true; /* Default value */
644d9dd5ef3SHerbert Xu __be64 *final;
64536cf515bSGary R Hook int ret;
64636cf515bSGary R Hook
64736cf515bSGary R Hook struct scatterlist *p_inp, sg_inp[2];
64836cf515bSGary R Hook struct scatterlist *p_tag, sg_tag[2];
64936cf515bSGary R Hook struct scatterlist *p_outp, sg_outp[2];
65036cf515bSGary R Hook struct scatterlist *p_aad;
65136cf515bSGary R Hook
65236cf515bSGary R Hook if (!aes->iv)
65336cf515bSGary R Hook return -EINVAL;
65436cf515bSGary R Hook
65536cf515bSGary R Hook if (!((aes->key_len == AES_KEYSIZE_128) ||
65636cf515bSGary R Hook (aes->key_len == AES_KEYSIZE_192) ||
65736cf515bSGary R Hook (aes->key_len == AES_KEYSIZE_256)))
65836cf515bSGary R Hook return -EINVAL;
65936cf515bSGary R Hook
66036cf515bSGary R Hook if (!aes->key) /* Gotta have a key SGL */
66136cf515bSGary R Hook return -EINVAL;
66236cf515bSGary R Hook
6639f00baf7SGary R Hook /* Zero defaults to 16 bytes, the maximum size */
6649f00baf7SGary R Hook authsize = aes->authsize ? aes->authsize : AES_BLOCK_SIZE;
6659f00baf7SGary R Hook switch (authsize) {
6669f00baf7SGary R Hook case 16:
6679f00baf7SGary R Hook case 15:
6689f00baf7SGary R Hook case 14:
6699f00baf7SGary R Hook case 13:
6709f00baf7SGary R Hook case 12:
6719f00baf7SGary R Hook case 8:
6729f00baf7SGary R Hook case 4:
6739f00baf7SGary R Hook break;
6749f00baf7SGary R Hook default:
6759f00baf7SGary R Hook return -EINVAL;
6769f00baf7SGary R Hook }
6779f00baf7SGary R Hook
67836cf515bSGary R Hook /* First, decompose the source buffer into AAD & PT,
67936cf515bSGary R Hook * and the destination buffer into AAD, CT & tag, or
68036cf515bSGary R Hook * the input into CT & tag.
68136cf515bSGary R Hook * It is expected that the input and output SGs will
68236cf515bSGary R Hook * be valid, even if the AAD and input lengths are 0.
68336cf515bSGary R Hook */
68436cf515bSGary R Hook p_aad = aes->src;
68536cf515bSGary R Hook p_inp = scatterwalk_ffwd(sg_inp, aes->src, aes->aad_len);
68636cf515bSGary R Hook p_outp = scatterwalk_ffwd(sg_outp, aes->dst, aes->aad_len);
68736cf515bSGary R Hook if (aes->action == CCP_AES_ACTION_ENCRYPT) {
68836cf515bSGary R Hook ilen = aes->src_len;
68936cf515bSGary R Hook p_tag = scatterwalk_ffwd(sg_tag, p_outp, ilen);
69036cf515bSGary R Hook } else {
69136cf515bSGary R Hook /* Input length for decryption includes tag */
6929f00baf7SGary R Hook ilen = aes->src_len - authsize;
69336cf515bSGary R Hook p_tag = scatterwalk_ffwd(sg_tag, p_inp, ilen);
69436cf515bSGary R Hook }
69536cf515bSGary R Hook
69620e833dcSHook, Gary jobid = CCP_NEW_JOBID(cmd_q->ccp);
69720e833dcSHook, Gary
69836cf515bSGary R Hook memset(&op, 0, sizeof(op));
69936cf515bSGary R Hook op.cmd_q = cmd_q;
70020e833dcSHook, Gary op.jobid = jobid;
70136cf515bSGary R Hook op.sb_key = cmd_q->sb_key; /* Pre-allocated */
70236cf515bSGary R Hook op.sb_ctx = cmd_q->sb_ctx; /* Pre-allocated */
70336cf515bSGary R Hook op.init = 1;
70436cf515bSGary R Hook op.u.aes.type = aes->type;
70536cf515bSGary R Hook
70636cf515bSGary R Hook /* Copy the key to the LSB */
70736cf515bSGary R Hook ret = ccp_init_dm_workarea(&key, cmd_q,
70836cf515bSGary R Hook CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES,
70936cf515bSGary R Hook DMA_TO_DEVICE);
71036cf515bSGary R Hook if (ret)
71136cf515bSGary R Hook return ret;
71236cf515bSGary R Hook
71336cf515bSGary R Hook dm_offset = CCP_SB_BYTES - aes->key_len;
714b698a9f4SGary R Hook ret = ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len);
715b698a9f4SGary R Hook if (ret)
716b698a9f4SGary R Hook goto e_key;
71736cf515bSGary R Hook ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
71836cf515bSGary R Hook CCP_PASSTHRU_BYTESWAP_256BIT);
71936cf515bSGary R Hook if (ret) {
72036cf515bSGary R Hook cmd->engine_error = cmd_q->cmd_error;
72136cf515bSGary R Hook goto e_key;
72236cf515bSGary R Hook }
72336cf515bSGary R Hook
72436cf515bSGary R Hook /* Copy the context (IV) to the LSB.
72536cf515bSGary R Hook * There is an assumption here that the IV is 96 bits in length, plus
72636cf515bSGary R Hook * a nonce of 32 bits. If no IV is present, use a zeroed buffer.
72736cf515bSGary R Hook */
72836cf515bSGary R Hook ret = ccp_init_dm_workarea(&ctx, cmd_q,
72936cf515bSGary R Hook CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES,
73036cf515bSGary R Hook DMA_BIDIRECTIONAL);
73136cf515bSGary R Hook if (ret)
73236cf515bSGary R Hook goto e_key;
73336cf515bSGary R Hook
73436cf515bSGary R Hook dm_offset = CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES - aes->iv_len;
735b698a9f4SGary R Hook ret = ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
736b698a9f4SGary R Hook if (ret)
737b698a9f4SGary R Hook goto e_ctx;
73836cf515bSGary R Hook
73936cf515bSGary R Hook ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
74036cf515bSGary R Hook CCP_PASSTHRU_BYTESWAP_256BIT);
74136cf515bSGary R Hook if (ret) {
74236cf515bSGary R Hook cmd->engine_error = cmd_q->cmd_error;
74336cf515bSGary R Hook goto e_ctx;
74436cf515bSGary R Hook }
74536cf515bSGary R Hook
74636cf515bSGary R Hook op.init = 1;
74736cf515bSGary R Hook if (aes->aad_len > 0) {
74836cf515bSGary R Hook /* Step 1: Run a GHASH over the Additional Authenticated Data */
74936cf515bSGary R Hook ret = ccp_init_data(&aad, cmd_q, p_aad, aes->aad_len,
75036cf515bSGary R Hook AES_BLOCK_SIZE,
75136cf515bSGary R Hook DMA_TO_DEVICE);
75236cf515bSGary R Hook if (ret)
75336cf515bSGary R Hook goto e_ctx;
75436cf515bSGary R Hook
75536cf515bSGary R Hook op.u.aes.mode = CCP_AES_MODE_GHASH;
75636cf515bSGary R Hook op.u.aes.action = CCP_AES_GHASHAAD;
75736cf515bSGary R Hook
75836cf515bSGary R Hook while (aad.sg_wa.bytes_left) {
75936cf515bSGary R Hook ccp_prepare_data(&aad, NULL, &op, AES_BLOCK_SIZE, true);
76036cf515bSGary R Hook
76136cf515bSGary R Hook ret = cmd_q->ccp->vdata->perform->aes(&op);
76236cf515bSGary R Hook if (ret) {
76336cf515bSGary R Hook cmd->engine_error = cmd_q->cmd_error;
76436cf515bSGary R Hook goto e_aad;
76536cf515bSGary R Hook }
76636cf515bSGary R Hook
76736cf515bSGary R Hook ccp_process_data(&aad, NULL, &op);
76836cf515bSGary R Hook op.init = 0;
76936cf515bSGary R Hook }
77036cf515bSGary R Hook }
77136cf515bSGary R Hook
77236cf515bSGary R Hook op.u.aes.mode = CCP_AES_MODE_GCTR;
77336cf515bSGary R Hook op.u.aes.action = aes->action;
77436cf515bSGary R Hook
77536cf515bSGary R Hook if (ilen > 0) {
77636cf515bSGary R Hook /* Step 2: Run a GCTR over the plaintext */
77736cf515bSGary R Hook in_place = (sg_virt(p_inp) == sg_virt(p_outp)) ? true : false;
77836cf515bSGary R Hook
77936cf515bSGary R Hook ret = ccp_init_data(&src, cmd_q, p_inp, ilen,
78036cf515bSGary R Hook AES_BLOCK_SIZE,
78136cf515bSGary R Hook in_place ? DMA_BIDIRECTIONAL
78236cf515bSGary R Hook : DMA_TO_DEVICE);
78336cf515bSGary R Hook if (ret)
784505d9dcbSDan Carpenter goto e_aad;
78536cf515bSGary R Hook
78636cf515bSGary R Hook if (in_place) {
78736cf515bSGary R Hook dst = src;
78836cf515bSGary R Hook } else {
78936cf515bSGary R Hook ret = ccp_init_data(&dst, cmd_q, p_outp, ilen,
79036cf515bSGary R Hook AES_BLOCK_SIZE, DMA_FROM_DEVICE);
79136cf515bSGary R Hook if (ret)
79236cf515bSGary R Hook goto e_src;
79336cf515bSGary R Hook }
79436cf515bSGary R Hook
79536cf515bSGary R Hook op.soc = 0;
79636cf515bSGary R Hook op.eom = 0;
79736cf515bSGary R Hook op.init = 1;
79836cf515bSGary R Hook while (src.sg_wa.bytes_left) {
79936cf515bSGary R Hook ccp_prepare_data(&src, &dst, &op, AES_BLOCK_SIZE, true);
80036cf515bSGary R Hook if (!src.sg_wa.bytes_left) {
801e2664ecbSGary R Hook unsigned int nbytes = ilen % AES_BLOCK_SIZE;
80236cf515bSGary R Hook
80336cf515bSGary R Hook if (nbytes) {
80436cf515bSGary R Hook op.eom = 1;
80536cf515bSGary R Hook op.u.aes.size = (nbytes * 8) - 1;
80636cf515bSGary R Hook }
80736cf515bSGary R Hook }
80836cf515bSGary R Hook
80936cf515bSGary R Hook ret = cmd_q->ccp->vdata->perform->aes(&op);
81036cf515bSGary R Hook if (ret) {
81136cf515bSGary R Hook cmd->engine_error = cmd_q->cmd_error;
81236cf515bSGary R Hook goto e_dst;
81336cf515bSGary R Hook }
81436cf515bSGary R Hook
81536cf515bSGary R Hook ccp_process_data(&src, &dst, &op);
81636cf515bSGary R Hook op.init = 0;
81736cf515bSGary R Hook }
81836cf515bSGary R Hook }
81936cf515bSGary R Hook
82036cf515bSGary R Hook /* Step 3: Update the IV portion of the context with the original IV */
82136cf515bSGary R Hook ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
82236cf515bSGary R Hook CCP_PASSTHRU_BYTESWAP_256BIT);
82336cf515bSGary R Hook if (ret) {
82436cf515bSGary R Hook cmd->engine_error = cmd_q->cmd_error;
82536cf515bSGary R Hook goto e_dst;
82636cf515bSGary R Hook }
82736cf515bSGary R Hook
828b698a9f4SGary R Hook ret = ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
829b698a9f4SGary R Hook if (ret)
830b698a9f4SGary R Hook goto e_dst;
83136cf515bSGary R Hook
83236cf515bSGary R Hook ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
83336cf515bSGary R Hook CCP_PASSTHRU_BYTESWAP_256BIT);
83436cf515bSGary R Hook if (ret) {
83536cf515bSGary R Hook cmd->engine_error = cmd_q->cmd_error;
83636cf515bSGary R Hook goto e_dst;
83736cf515bSGary R Hook }
83836cf515bSGary R Hook
83936cf515bSGary R Hook /* Step 4: Concatenate the lengths of the AAD and source, and
84036cf515bSGary R Hook * hash that 16 byte buffer.
84136cf515bSGary R Hook */
84236cf515bSGary R Hook ret = ccp_init_dm_workarea(&final_wa, cmd_q, AES_BLOCK_SIZE,
84336cf515bSGary R Hook DMA_BIDIRECTIONAL);
84436cf515bSGary R Hook if (ret)
84536cf515bSGary R Hook goto e_dst;
846d9dd5ef3SHerbert Xu final = (__be64 *)final_wa.address;
84736cf515bSGary R Hook final[0] = cpu_to_be64(aes->aad_len * 8);
84836cf515bSGary R Hook final[1] = cpu_to_be64(ilen * 8);
84936cf515bSGary R Hook
85020e833dcSHook, Gary memset(&op, 0, sizeof(op));
85120e833dcSHook, Gary op.cmd_q = cmd_q;
85220e833dcSHook, Gary op.jobid = jobid;
85320e833dcSHook, Gary op.sb_key = cmd_q->sb_key; /* Pre-allocated */
85420e833dcSHook, Gary op.sb_ctx = cmd_q->sb_ctx; /* Pre-allocated */
85520e833dcSHook, Gary op.init = 1;
85620e833dcSHook, Gary op.u.aes.type = aes->type;
85736cf515bSGary R Hook op.u.aes.mode = CCP_AES_MODE_GHASH;
85836cf515bSGary R Hook op.u.aes.action = CCP_AES_GHASHFINAL;
85936cf515bSGary R Hook op.src.type = CCP_MEMTYPE_SYSTEM;
86036cf515bSGary R Hook op.src.u.dma.address = final_wa.dma.address;
86136cf515bSGary R Hook op.src.u.dma.length = AES_BLOCK_SIZE;
86236cf515bSGary R Hook op.dst.type = CCP_MEMTYPE_SYSTEM;
86336cf515bSGary R Hook op.dst.u.dma.address = final_wa.dma.address;
86436cf515bSGary R Hook op.dst.u.dma.length = AES_BLOCK_SIZE;
86536cf515bSGary R Hook op.eom = 1;
86636cf515bSGary R Hook op.u.aes.size = 0;
86736cf515bSGary R Hook ret = cmd_q->ccp->vdata->perform->aes(&op);
86836cf515bSGary R Hook if (ret)
869505d9dcbSDan Carpenter goto e_final_wa;
87036cf515bSGary R Hook
87136cf515bSGary R Hook if (aes->action == CCP_AES_ACTION_ENCRYPT) {
87236cf515bSGary R Hook /* Put the ciphered tag after the ciphertext. */
8739f00baf7SGary R Hook ccp_get_dm_area(&final_wa, 0, p_tag, 0, authsize);
87436cf515bSGary R Hook } else {
87536cf515bSGary R Hook /* Does this ciphered tag match the input? */
8769f00baf7SGary R Hook ret = ccp_init_dm_workarea(&tag, cmd_q, authsize,
87736cf515bSGary R Hook DMA_BIDIRECTIONAL);
87836cf515bSGary R Hook if (ret)
879505d9dcbSDan Carpenter goto e_final_wa;
8809f00baf7SGary R Hook ret = ccp_set_dm_area(&tag, 0, p_tag, 0, authsize);
881505d9dcbSDan Carpenter if (ret) {
882505d9dcbSDan Carpenter ccp_dm_free(&tag);
883505d9dcbSDan Carpenter goto e_final_wa;
884505d9dcbSDan Carpenter }
88536cf515bSGary R Hook
886538a5a07SCfir Cohen ret = crypto_memneq(tag.address, final_wa.address,
8879f00baf7SGary R Hook authsize) ? -EBADMSG : 0;
88836cf515bSGary R Hook ccp_dm_free(&tag);
88936cf515bSGary R Hook }
89036cf515bSGary R Hook
891505d9dcbSDan Carpenter e_final_wa:
89236cf515bSGary R Hook ccp_dm_free(&final_wa);
89336cf515bSGary R Hook
89436cf515bSGary R Hook e_dst:
89525e44338SGary R Hook if (ilen > 0 && !in_place)
89636cf515bSGary R Hook ccp_free_data(&dst, cmd_q);
89736cf515bSGary R Hook
89836cf515bSGary R Hook e_src:
89925e44338SGary R Hook if (ilen > 0)
90036cf515bSGary R Hook ccp_free_data(&src, cmd_q);
90136cf515bSGary R Hook
90236cf515bSGary R Hook e_aad:
90336cf515bSGary R Hook if (aes->aad_len)
90436cf515bSGary R Hook ccp_free_data(&aad, cmd_q);
90536cf515bSGary R Hook
90636cf515bSGary R Hook e_ctx:
90736cf515bSGary R Hook ccp_dm_free(&ctx);
90836cf515bSGary R Hook
90936cf515bSGary R Hook e_key:
91036cf515bSGary R Hook ccp_dm_free(&key);
91136cf515bSGary R Hook
91236cf515bSGary R Hook return ret;
91336cf515bSGary R Hook }
91436cf515bSGary R Hook
91572c8117aSArnd Bergmann static noinline_for_stack int
ccp_run_aes_cmd(struct ccp_cmd_queue * cmd_q,struct ccp_cmd * cmd)91672c8117aSArnd Bergmann ccp_run_aes_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
91763b94509STom Lendacky {
91863b94509STom Lendacky struct ccp_aes_engine *aes = &cmd->u.aes;
91963b94509STom Lendacky struct ccp_dm_workarea key, ctx;
92063b94509STom Lendacky struct ccp_data src, dst;
92163b94509STom Lendacky struct ccp_op op;
92263b94509STom Lendacky unsigned int dm_offset;
92363b94509STom Lendacky bool in_place = false;
92463b94509STom Lendacky int ret;
92563b94509STom Lendacky
92663b94509STom Lendacky if (!((aes->key_len == AES_KEYSIZE_128) ||
92763b94509STom Lendacky (aes->key_len == AES_KEYSIZE_192) ||
92863b94509STom Lendacky (aes->key_len == AES_KEYSIZE_256)))
92963b94509STom Lendacky return -EINVAL;
93063b94509STom Lendacky
93163b94509STom Lendacky if (((aes->mode == CCP_AES_MODE_ECB) ||
932499df967SHook, Gary (aes->mode == CCP_AES_MODE_CBC)) &&
93363b94509STom Lendacky (aes->src_len & (AES_BLOCK_SIZE - 1)))
93463b94509STom Lendacky return -EINVAL;
93563b94509STom Lendacky
93663b94509STom Lendacky if (!aes->key || !aes->src || !aes->dst)
93763b94509STom Lendacky return -EINVAL;
93863b94509STom Lendacky
93963b94509STom Lendacky if (aes->mode != CCP_AES_MODE_ECB) {
94063b94509STom Lendacky if (aes->iv_len != AES_BLOCK_SIZE)
94163b94509STom Lendacky return -EINVAL;
94263b94509STom Lendacky
94363b94509STom Lendacky if (!aes->iv)
94463b94509STom Lendacky return -EINVAL;
94563b94509STom Lendacky }
94663b94509STom Lendacky
947956ee21aSGary R Hook BUILD_BUG_ON(CCP_AES_KEY_SB_COUNT != 1);
948956ee21aSGary R Hook BUILD_BUG_ON(CCP_AES_CTX_SB_COUNT != 1);
94963b94509STom Lendacky
95063b94509STom Lendacky ret = -EIO;
95163b94509STom Lendacky memset(&op, 0, sizeof(op));
95263b94509STom Lendacky op.cmd_q = cmd_q;
9534b394a23SGary R Hook op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
954956ee21aSGary R Hook op.sb_key = cmd_q->sb_key;
955956ee21aSGary R Hook op.sb_ctx = cmd_q->sb_ctx;
95663b94509STom Lendacky op.init = (aes->mode == CCP_AES_MODE_ECB) ? 0 : 1;
95763b94509STom Lendacky op.u.aes.type = aes->type;
95863b94509STom Lendacky op.u.aes.mode = aes->mode;
95963b94509STom Lendacky op.u.aes.action = aes->action;
96063b94509STom Lendacky
961956ee21aSGary R Hook /* All supported key sizes fit in a single (32-byte) SB entry
96263b94509STom Lendacky * and must be in little endian format. Use the 256-bit byte
96363b94509STom Lendacky * swap passthru option to convert from big endian to little
96463b94509STom Lendacky * endian.
96563b94509STom Lendacky */
96663b94509STom Lendacky ret = ccp_init_dm_workarea(&key, cmd_q,
967956ee21aSGary R Hook CCP_AES_KEY_SB_COUNT * CCP_SB_BYTES,
96863b94509STom Lendacky DMA_TO_DEVICE);
96963b94509STom Lendacky if (ret)
97063b94509STom Lendacky return ret;
97163b94509STom Lendacky
972956ee21aSGary R Hook dm_offset = CCP_SB_BYTES - aes->key_len;
973b698a9f4SGary R Hook ret = ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len);
974b698a9f4SGary R Hook if (ret)
975b698a9f4SGary R Hook goto e_key;
976956ee21aSGary R Hook ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
97763b94509STom Lendacky CCP_PASSTHRU_BYTESWAP_256BIT);
97863b94509STom Lendacky if (ret) {
97963b94509STom Lendacky cmd->engine_error = cmd_q->cmd_error;
98063b94509STom Lendacky goto e_key;
98163b94509STom Lendacky }
98263b94509STom Lendacky
983956ee21aSGary R Hook /* The AES context fits in a single (32-byte) SB entry and
98463b94509STom Lendacky * must be in little endian format. Use the 256-bit byte swap
98563b94509STom Lendacky * passthru option to convert from big endian to little endian.
98663b94509STom Lendacky */
98763b94509STom Lendacky ret = ccp_init_dm_workarea(&ctx, cmd_q,
988956ee21aSGary R Hook CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES,
98963b94509STom Lendacky DMA_BIDIRECTIONAL);
99063b94509STom Lendacky if (ret)
99163b94509STom Lendacky goto e_key;
99263b94509STom Lendacky
99363b94509STom Lendacky if (aes->mode != CCP_AES_MODE_ECB) {
9944b394a23SGary R Hook /* Load the AES context - convert to LE */
995956ee21aSGary R Hook dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
996b698a9f4SGary R Hook ret = ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
997b698a9f4SGary R Hook if (ret)
998b698a9f4SGary R Hook goto e_ctx;
999956ee21aSGary R Hook ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
100063b94509STom Lendacky CCP_PASSTHRU_BYTESWAP_256BIT);
100163b94509STom Lendacky if (ret) {
100263b94509STom Lendacky cmd->engine_error = cmd_q->cmd_error;
100363b94509STom Lendacky goto e_ctx;
100463b94509STom Lendacky }
100563b94509STom Lendacky }
1006f7cc02b3SGary R Hook switch (aes->mode) {
1007f7cc02b3SGary R Hook case CCP_AES_MODE_CFB: /* CFB128 only */
1008f7cc02b3SGary R Hook case CCP_AES_MODE_CTR:
1009f7cc02b3SGary R Hook op.u.aes.size = AES_BLOCK_SIZE * BITS_PER_BYTE - 1;
1010f7cc02b3SGary R Hook break;
1011f7cc02b3SGary R Hook default:
1012f7cc02b3SGary R Hook op.u.aes.size = 0;
1013f7cc02b3SGary R Hook }
101463b94509STom Lendacky
101563b94509STom Lendacky /* Prepare the input and output data workareas. For in-place
101663b94509STom Lendacky * operations we need to set the dma direction to BIDIRECTIONAL
101763b94509STom Lendacky * and copy the src workarea to the dst workarea.
101863b94509STom Lendacky */
101963b94509STom Lendacky if (sg_virt(aes->src) == sg_virt(aes->dst))
102063b94509STom Lendacky in_place = true;
102163b94509STom Lendacky
102263b94509STom Lendacky ret = ccp_init_data(&src, cmd_q, aes->src, aes->src_len,
102363b94509STom Lendacky AES_BLOCK_SIZE,
102463b94509STom Lendacky in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
102563b94509STom Lendacky if (ret)
102663b94509STom Lendacky goto e_ctx;
102763b94509STom Lendacky
10288db88467STom Lendacky if (in_place) {
102963b94509STom Lendacky dst = src;
10308db88467STom Lendacky } else {
103163b94509STom Lendacky ret = ccp_init_data(&dst, cmd_q, aes->dst, aes->src_len,
103263b94509STom Lendacky AES_BLOCK_SIZE, DMA_FROM_DEVICE);
103363b94509STom Lendacky if (ret)
103463b94509STom Lendacky goto e_src;
103563b94509STom Lendacky }
103663b94509STom Lendacky
103763b94509STom Lendacky /* Send data to the CCP AES engine */
103863b94509STom Lendacky while (src.sg_wa.bytes_left) {
103963b94509STom Lendacky ccp_prepare_data(&src, &dst, &op, AES_BLOCK_SIZE, true);
104063b94509STom Lendacky if (!src.sg_wa.bytes_left) {
104163b94509STom Lendacky op.eom = 1;
104263b94509STom Lendacky
104363b94509STom Lendacky /* Since we don't retrieve the AES context in ECB
104463b94509STom Lendacky * mode we have to wait for the operation to complete
104563b94509STom Lendacky * on the last piece of data
104663b94509STom Lendacky */
104763b94509STom Lendacky if (aes->mode == CCP_AES_MODE_ECB)
104863b94509STom Lendacky op.soc = 1;
104963b94509STom Lendacky }
105063b94509STom Lendacky
1051a43eb985SGary R Hook ret = cmd_q->ccp->vdata->perform->aes(&op);
105263b94509STom Lendacky if (ret) {
105363b94509STom Lendacky cmd->engine_error = cmd_q->cmd_error;
105463b94509STom Lendacky goto e_dst;
105563b94509STom Lendacky }
105663b94509STom Lendacky
105763b94509STom Lendacky ccp_process_data(&src, &dst, &op);
105863b94509STom Lendacky }
105963b94509STom Lendacky
106063b94509STom Lendacky if (aes->mode != CCP_AES_MODE_ECB) {
106163b94509STom Lendacky /* Retrieve the AES context - convert from LE to BE using
106263b94509STom Lendacky * 32-byte (256-bit) byteswapping
106363b94509STom Lendacky */
1064956ee21aSGary R Hook ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
106563b94509STom Lendacky CCP_PASSTHRU_BYTESWAP_256BIT);
106663b94509STom Lendacky if (ret) {
106763b94509STom Lendacky cmd->engine_error = cmd_q->cmd_error;
106863b94509STom Lendacky goto e_dst;
106963b94509STom Lendacky }
107063b94509STom Lendacky
107163b94509STom Lendacky /* ...but we only need AES_BLOCK_SIZE bytes */
1072956ee21aSGary R Hook dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
107363b94509STom Lendacky ccp_get_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
107463b94509STom Lendacky }
107563b94509STom Lendacky
107663b94509STom Lendacky e_dst:
107763b94509STom Lendacky if (!in_place)
107863b94509STom Lendacky ccp_free_data(&dst, cmd_q);
107963b94509STom Lendacky
108063b94509STom Lendacky e_src:
108163b94509STom Lendacky ccp_free_data(&src, cmd_q);
108263b94509STom Lendacky
108363b94509STom Lendacky e_ctx:
108463b94509STom Lendacky ccp_dm_free(&ctx);
108563b94509STom Lendacky
108663b94509STom Lendacky e_key:
108763b94509STom Lendacky ccp_dm_free(&key);
108863b94509STom Lendacky
108963b94509STom Lendacky return ret;
109063b94509STom Lendacky }
109163b94509STom Lendacky
109272c8117aSArnd Bergmann static noinline_for_stack int
ccp_run_xts_aes_cmd(struct ccp_cmd_queue * cmd_q,struct ccp_cmd * cmd)109372c8117aSArnd Bergmann ccp_run_xts_aes_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
109463b94509STom Lendacky {
109563b94509STom Lendacky struct ccp_xts_aes_engine *xts = &cmd->u.xts;
109663b94509STom Lendacky struct ccp_dm_workarea key, ctx;
109763b94509STom Lendacky struct ccp_data src, dst;
109863b94509STom Lendacky struct ccp_op op;
109963b94509STom Lendacky unsigned int unit_size, dm_offset;
110063b94509STom Lendacky bool in_place = false;
1101e652399eSGary R Hook unsigned int sb_count;
1102e652399eSGary R Hook enum ccp_aes_type aestype;
110363b94509STom Lendacky int ret;
110463b94509STom Lendacky
110563b94509STom Lendacky switch (xts->unit_size) {
110663b94509STom Lendacky case CCP_XTS_AES_UNIT_SIZE_16:
110763b94509STom Lendacky unit_size = 16;
110863b94509STom Lendacky break;
110963b94509STom Lendacky case CCP_XTS_AES_UNIT_SIZE_512:
111063b94509STom Lendacky unit_size = 512;
111163b94509STom Lendacky break;
111263b94509STom Lendacky case CCP_XTS_AES_UNIT_SIZE_1024:
111363b94509STom Lendacky unit_size = 1024;
111463b94509STom Lendacky break;
111563b94509STom Lendacky case CCP_XTS_AES_UNIT_SIZE_2048:
111663b94509STom Lendacky unit_size = 2048;
111763b94509STom Lendacky break;
111863b94509STom Lendacky case CCP_XTS_AES_UNIT_SIZE_4096:
111963b94509STom Lendacky unit_size = 4096;
112063b94509STom Lendacky break;
112163b94509STom Lendacky
112263b94509STom Lendacky default:
112363b94509STom Lendacky return -EINVAL;
112463b94509STom Lendacky }
112563b94509STom Lendacky
1126e652399eSGary R Hook if (xts->key_len == AES_KEYSIZE_128)
1127e652399eSGary R Hook aestype = CCP_AES_TYPE_128;
11285060ffc9SGary R Hook else if (xts->key_len == AES_KEYSIZE_256)
11295060ffc9SGary R Hook aestype = CCP_AES_TYPE_256;
1130e652399eSGary R Hook else
113163b94509STom Lendacky return -EINVAL;
113263b94509STom Lendacky
113363b94509STom Lendacky if (!xts->final && (xts->src_len & (AES_BLOCK_SIZE - 1)))
113463b94509STom Lendacky return -EINVAL;
113563b94509STom Lendacky
113663b94509STom Lendacky if (xts->iv_len != AES_BLOCK_SIZE)
113763b94509STom Lendacky return -EINVAL;
113863b94509STom Lendacky
113963b94509STom Lendacky if (!xts->key || !xts->iv || !xts->src || !xts->dst)
114063b94509STom Lendacky return -EINVAL;
114163b94509STom Lendacky
1142956ee21aSGary R Hook BUILD_BUG_ON(CCP_XTS_AES_KEY_SB_COUNT != 1);
1143956ee21aSGary R Hook BUILD_BUG_ON(CCP_XTS_AES_CTX_SB_COUNT != 1);
114463b94509STom Lendacky
114563b94509STom Lendacky ret = -EIO;
114663b94509STom Lendacky memset(&op, 0, sizeof(op));
114763b94509STom Lendacky op.cmd_q = cmd_q;
11484b394a23SGary R Hook op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
1149956ee21aSGary R Hook op.sb_key = cmd_q->sb_key;
1150956ee21aSGary R Hook op.sb_ctx = cmd_q->sb_ctx;
115163b94509STom Lendacky op.init = 1;
1152e652399eSGary R Hook op.u.xts.type = aestype;
115363b94509STom Lendacky op.u.xts.action = xts->action;
115463b94509STom Lendacky op.u.xts.unit_size = xts->unit_size;
115563b94509STom Lendacky
1156e652399eSGary R Hook /* A version 3 device only supports 128-bit keys, which fits into a
1157e652399eSGary R Hook * single SB entry. A version 5 device uses a 512-bit vector, so two
1158e652399eSGary R Hook * SB entries.
115963b94509STom Lendacky */
1160e652399eSGary R Hook if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0))
1161e652399eSGary R Hook sb_count = CCP_XTS_AES_KEY_SB_COUNT;
1162e652399eSGary R Hook else
1163e652399eSGary R Hook sb_count = CCP5_XTS_AES_KEY_SB_COUNT;
116463b94509STom Lendacky ret = ccp_init_dm_workarea(&key, cmd_q,
1165e652399eSGary R Hook sb_count * CCP_SB_BYTES,
116663b94509STom Lendacky DMA_TO_DEVICE);
116763b94509STom Lendacky if (ret)
116863b94509STom Lendacky return ret;
116963b94509STom Lendacky
1170e652399eSGary R Hook if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0)) {
1171e652399eSGary R Hook /* All supported key sizes must be in little endian format.
1172e652399eSGary R Hook * Use the 256-bit byte swap passthru option to convert from
1173e652399eSGary R Hook * big endian to little endian.
1174e652399eSGary R Hook */
1175956ee21aSGary R Hook dm_offset = CCP_SB_BYTES - AES_KEYSIZE_128;
1176b698a9f4SGary R Hook ret = ccp_set_dm_area(&key, dm_offset, xts->key, 0, xts->key_len);
1177b698a9f4SGary R Hook if (ret)
1178b698a9f4SGary R Hook goto e_key;
1179b698a9f4SGary R Hook ret = ccp_set_dm_area(&key, 0, xts->key, xts->key_len, xts->key_len);
1180b698a9f4SGary R Hook if (ret)
1181b698a9f4SGary R Hook goto e_key;
1182e652399eSGary R Hook } else {
1183e652399eSGary R Hook /* Version 5 CCPs use a 512-bit space for the key: each portion
1184e652399eSGary R Hook * occupies 256 bits, or one entire slot, and is zero-padded.
1185e652399eSGary R Hook */
1186e652399eSGary R Hook unsigned int pad;
1187e652399eSGary R Hook
1188e652399eSGary R Hook dm_offset = CCP_SB_BYTES;
1189e652399eSGary R Hook pad = dm_offset - xts->key_len;
1190b698a9f4SGary R Hook ret = ccp_set_dm_area(&key, pad, xts->key, 0, xts->key_len);
1191b698a9f4SGary R Hook if (ret)
1192b698a9f4SGary R Hook goto e_key;
1193b698a9f4SGary R Hook ret = ccp_set_dm_area(&key, dm_offset + pad, xts->key,
1194b698a9f4SGary R Hook xts->key_len, xts->key_len);
1195b698a9f4SGary R Hook if (ret)
1196b698a9f4SGary R Hook goto e_key;
1197e652399eSGary R Hook }
1198956ee21aSGary R Hook ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
119963b94509STom Lendacky CCP_PASSTHRU_BYTESWAP_256BIT);
120063b94509STom Lendacky if (ret) {
120163b94509STom Lendacky cmd->engine_error = cmd_q->cmd_error;
120263b94509STom Lendacky goto e_key;
120363b94509STom Lendacky }
120463b94509STom Lendacky
1205956ee21aSGary R Hook /* The AES context fits in a single (32-byte) SB entry and
120663b94509STom Lendacky * for XTS is already in little endian format so no byte swapping
120763b94509STom Lendacky * is needed.
120863b94509STom Lendacky */
120963b94509STom Lendacky ret = ccp_init_dm_workarea(&ctx, cmd_q,
1210956ee21aSGary R Hook CCP_XTS_AES_CTX_SB_COUNT * CCP_SB_BYTES,
121163b94509STom Lendacky DMA_BIDIRECTIONAL);
121263b94509STom Lendacky if (ret)
121363b94509STom Lendacky goto e_key;
121463b94509STom Lendacky
1215b698a9f4SGary R Hook ret = ccp_set_dm_area(&ctx, 0, xts->iv, 0, xts->iv_len);
1216b698a9f4SGary R Hook if (ret)
1217b698a9f4SGary R Hook goto e_ctx;
1218956ee21aSGary R Hook ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
121963b94509STom Lendacky CCP_PASSTHRU_BYTESWAP_NOOP);
122063b94509STom Lendacky if (ret) {
122163b94509STom Lendacky cmd->engine_error = cmd_q->cmd_error;
122263b94509STom Lendacky goto e_ctx;
122363b94509STom Lendacky }
122463b94509STom Lendacky
122563b94509STom Lendacky /* Prepare the input and output data workareas. For in-place
122663b94509STom Lendacky * operations we need to set the dma direction to BIDIRECTIONAL
122763b94509STom Lendacky * and copy the src workarea to the dst workarea.
122863b94509STom Lendacky */
122963b94509STom Lendacky if (sg_virt(xts->src) == sg_virt(xts->dst))
123063b94509STom Lendacky in_place = true;
123163b94509STom Lendacky
123263b94509STom Lendacky ret = ccp_init_data(&src, cmd_q, xts->src, xts->src_len,
123363b94509STom Lendacky unit_size,
123463b94509STom Lendacky in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
123563b94509STom Lendacky if (ret)
123663b94509STom Lendacky goto e_ctx;
123763b94509STom Lendacky
12388db88467STom Lendacky if (in_place) {
123963b94509STom Lendacky dst = src;
12408db88467STom Lendacky } else {
124163b94509STom Lendacky ret = ccp_init_data(&dst, cmd_q, xts->dst, xts->src_len,
124263b94509STom Lendacky unit_size, DMA_FROM_DEVICE);
124363b94509STom Lendacky if (ret)
124463b94509STom Lendacky goto e_src;
124563b94509STom Lendacky }
124663b94509STom Lendacky
124763b94509STom Lendacky /* Send data to the CCP AES engine */
124863b94509STom Lendacky while (src.sg_wa.bytes_left) {
124963b94509STom Lendacky ccp_prepare_data(&src, &dst, &op, unit_size, true);
125063b94509STom Lendacky if (!src.sg_wa.bytes_left)
125163b94509STom Lendacky op.eom = 1;
125263b94509STom Lendacky
1253a43eb985SGary R Hook ret = cmd_q->ccp->vdata->perform->xts_aes(&op);
125463b94509STom Lendacky if (ret) {
125563b94509STom Lendacky cmd->engine_error = cmd_q->cmd_error;
125663b94509STom Lendacky goto e_dst;
125763b94509STom Lendacky }
125863b94509STom Lendacky
125963b94509STom Lendacky ccp_process_data(&src, &dst, &op);
126063b94509STom Lendacky }
126163b94509STom Lendacky
126263b94509STom Lendacky /* Retrieve the AES context - convert from LE to BE using
126363b94509STom Lendacky * 32-byte (256-bit) byteswapping
126463b94509STom Lendacky */
1265956ee21aSGary R Hook ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
126663b94509STom Lendacky CCP_PASSTHRU_BYTESWAP_256BIT);
126763b94509STom Lendacky if (ret) {
126863b94509STom Lendacky cmd->engine_error = cmd_q->cmd_error;
126963b94509STom Lendacky goto e_dst;
127063b94509STom Lendacky }
127163b94509STom Lendacky
127263b94509STom Lendacky /* ...but we only need AES_BLOCK_SIZE bytes */
1273956ee21aSGary R Hook dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
127463b94509STom Lendacky ccp_get_dm_area(&ctx, dm_offset, xts->iv, 0, xts->iv_len);
127563b94509STom Lendacky
127663b94509STom Lendacky e_dst:
127763b94509STom Lendacky if (!in_place)
127863b94509STom Lendacky ccp_free_data(&dst, cmd_q);
127963b94509STom Lendacky
128063b94509STom Lendacky e_src:
128163b94509STom Lendacky ccp_free_data(&src, cmd_q);
128263b94509STom Lendacky
128363b94509STom Lendacky e_ctx:
128463b94509STom Lendacky ccp_dm_free(&ctx);
128563b94509STom Lendacky
128663b94509STom Lendacky e_key:
128763b94509STom Lendacky ccp_dm_free(&key);
128863b94509STom Lendacky
128963b94509STom Lendacky return ret;
129063b94509STom Lendacky }
129163b94509STom Lendacky
129272c8117aSArnd Bergmann static noinline_for_stack int
ccp_run_des3_cmd(struct ccp_cmd_queue * cmd_q,struct ccp_cmd * cmd)129372c8117aSArnd Bergmann ccp_run_des3_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1294990672d4SGary R Hook {
1295990672d4SGary R Hook struct ccp_des3_engine *des3 = &cmd->u.des3;
1296990672d4SGary R Hook
1297990672d4SGary R Hook struct ccp_dm_workarea key, ctx;
1298990672d4SGary R Hook struct ccp_data src, dst;
1299990672d4SGary R Hook struct ccp_op op;
1300990672d4SGary R Hook unsigned int dm_offset;
1301990672d4SGary R Hook unsigned int len_singlekey;
1302990672d4SGary R Hook bool in_place = false;
1303990672d4SGary R Hook int ret;
1304990672d4SGary R Hook
1305990672d4SGary R Hook /* Error checks */
130689646fddSHook, Gary if (cmd_q->ccp->vdata->version < CCP_VERSION(5, 0))
130789646fddSHook, Gary return -EINVAL;
130889646fddSHook, Gary
1309990672d4SGary R Hook if (!cmd_q->ccp->vdata->perform->des3)
1310990672d4SGary R Hook return -EINVAL;
1311990672d4SGary R Hook
1312990672d4SGary R Hook if (des3->key_len != DES3_EDE_KEY_SIZE)
1313990672d4SGary R Hook return -EINVAL;
1314990672d4SGary R Hook
1315990672d4SGary R Hook if (((des3->mode == CCP_DES3_MODE_ECB) ||
1316990672d4SGary R Hook (des3->mode == CCP_DES3_MODE_CBC)) &&
1317990672d4SGary R Hook (des3->src_len & (DES3_EDE_BLOCK_SIZE - 1)))
1318990672d4SGary R Hook return -EINVAL;
1319990672d4SGary R Hook
1320990672d4SGary R Hook if (!des3->key || !des3->src || !des3->dst)
1321990672d4SGary R Hook return -EINVAL;
1322990672d4SGary R Hook
1323990672d4SGary R Hook if (des3->mode != CCP_DES3_MODE_ECB) {
1324990672d4SGary R Hook if (des3->iv_len != DES3_EDE_BLOCK_SIZE)
1325990672d4SGary R Hook return -EINVAL;
1326990672d4SGary R Hook
1327990672d4SGary R Hook if (!des3->iv)
1328990672d4SGary R Hook return -EINVAL;
1329990672d4SGary R Hook }
1330990672d4SGary R Hook
1331990672d4SGary R Hook /* Zero out all the fields of the command desc */
1332990672d4SGary R Hook memset(&op, 0, sizeof(op));
1333990672d4SGary R Hook
1334990672d4SGary R Hook /* Set up the Function field */
1335990672d4SGary R Hook op.cmd_q = cmd_q;
1336990672d4SGary R Hook op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
1337990672d4SGary R Hook op.sb_key = cmd_q->sb_key;
1338990672d4SGary R Hook
1339990672d4SGary R Hook op.init = (des3->mode == CCP_DES3_MODE_ECB) ? 0 : 1;
1340990672d4SGary R Hook op.u.des3.type = des3->type;
1341990672d4SGary R Hook op.u.des3.mode = des3->mode;
1342990672d4SGary R Hook op.u.des3.action = des3->action;
1343990672d4SGary R Hook
1344990672d4SGary R Hook /*
1345990672d4SGary R Hook * All supported key sizes fit in a single (32-byte) KSB entry and
1346990672d4SGary R Hook * (like AES) must be in little endian format. Use the 256-bit byte
1347990672d4SGary R Hook * swap passthru option to convert from big endian to little endian.
1348990672d4SGary R Hook */
1349990672d4SGary R Hook ret = ccp_init_dm_workarea(&key, cmd_q,
1350990672d4SGary R Hook CCP_DES3_KEY_SB_COUNT * CCP_SB_BYTES,
1351990672d4SGary R Hook DMA_TO_DEVICE);
1352990672d4SGary R Hook if (ret)
1353990672d4SGary R Hook return ret;
1354990672d4SGary R Hook
1355990672d4SGary R Hook /*
1356990672d4SGary R Hook * The contents of the key triplet are in the reverse order of what
1357990672d4SGary R Hook * is required by the engine. Copy the 3 pieces individually to put
1358990672d4SGary R Hook * them where they belong.
1359990672d4SGary R Hook */
1360990672d4SGary R Hook dm_offset = CCP_SB_BYTES - des3->key_len; /* Basic offset */
1361990672d4SGary R Hook
1362990672d4SGary R Hook len_singlekey = des3->key_len / 3;
1363b698a9f4SGary R Hook ret = ccp_set_dm_area(&key, dm_offset + 2 * len_singlekey,
1364990672d4SGary R Hook des3->key, 0, len_singlekey);
1365b698a9f4SGary R Hook if (ret)
1366b698a9f4SGary R Hook goto e_key;
1367b698a9f4SGary R Hook ret = ccp_set_dm_area(&key, dm_offset + len_singlekey,
1368990672d4SGary R Hook des3->key, len_singlekey, len_singlekey);
1369b698a9f4SGary R Hook if (ret)
1370b698a9f4SGary R Hook goto e_key;
1371b698a9f4SGary R Hook ret = ccp_set_dm_area(&key, dm_offset,
1372990672d4SGary R Hook des3->key, 2 * len_singlekey, len_singlekey);
1373b698a9f4SGary R Hook if (ret)
1374b698a9f4SGary R Hook goto e_key;
1375990672d4SGary R Hook
1376990672d4SGary R Hook /* Copy the key to the SB */
1377990672d4SGary R Hook ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
1378990672d4SGary R Hook CCP_PASSTHRU_BYTESWAP_256BIT);
1379990672d4SGary R Hook if (ret) {
1380990672d4SGary R Hook cmd->engine_error = cmd_q->cmd_error;
1381990672d4SGary R Hook goto e_key;
1382990672d4SGary R Hook }
1383990672d4SGary R Hook
1384990672d4SGary R Hook /*
1385990672d4SGary R Hook * The DES3 context fits in a single (32-byte) KSB entry and
1386990672d4SGary R Hook * must be in little endian format. Use the 256-bit byte swap
1387990672d4SGary R Hook * passthru option to convert from big endian to little endian.
1388990672d4SGary R Hook */
1389990672d4SGary R Hook if (des3->mode != CCP_DES3_MODE_ECB) {
1390990672d4SGary R Hook op.sb_ctx = cmd_q->sb_ctx;
1391990672d4SGary R Hook
1392990672d4SGary R Hook ret = ccp_init_dm_workarea(&ctx, cmd_q,
1393990672d4SGary R Hook CCP_DES3_CTX_SB_COUNT * CCP_SB_BYTES,
1394990672d4SGary R Hook DMA_BIDIRECTIONAL);
1395990672d4SGary R Hook if (ret)
1396990672d4SGary R Hook goto e_key;
1397990672d4SGary R Hook
1398990672d4SGary R Hook /* Load the context into the LSB */
1399990672d4SGary R Hook dm_offset = CCP_SB_BYTES - des3->iv_len;
1400b698a9f4SGary R Hook ret = ccp_set_dm_area(&ctx, dm_offset, des3->iv, 0,
1401b698a9f4SGary R Hook des3->iv_len);
1402b698a9f4SGary R Hook if (ret)
1403b698a9f4SGary R Hook goto e_ctx;
1404990672d4SGary R Hook
1405990672d4SGary R Hook ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
140689646fddSHook, Gary CCP_PASSTHRU_BYTESWAP_256BIT);
1407990672d4SGary R Hook if (ret) {
1408990672d4SGary R Hook cmd->engine_error = cmd_q->cmd_error;
1409990672d4SGary R Hook goto e_ctx;
1410990672d4SGary R Hook }
1411990672d4SGary R Hook }
1412990672d4SGary R Hook
1413990672d4SGary R Hook /*
1414990672d4SGary R Hook * Prepare the input and output data workareas. For in-place
1415990672d4SGary R Hook * operations we need to set the dma direction to BIDIRECTIONAL
1416990672d4SGary R Hook * and copy the src workarea to the dst workarea.
1417990672d4SGary R Hook */
1418990672d4SGary R Hook if (sg_virt(des3->src) == sg_virt(des3->dst))
1419990672d4SGary R Hook in_place = true;
1420990672d4SGary R Hook
1421990672d4SGary R Hook ret = ccp_init_data(&src, cmd_q, des3->src, des3->src_len,
1422990672d4SGary R Hook DES3_EDE_BLOCK_SIZE,
1423990672d4SGary R Hook in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
1424990672d4SGary R Hook if (ret)
1425990672d4SGary R Hook goto e_ctx;
1426990672d4SGary R Hook
1427990672d4SGary R Hook if (in_place)
1428990672d4SGary R Hook dst = src;
1429990672d4SGary R Hook else {
1430990672d4SGary R Hook ret = ccp_init_data(&dst, cmd_q, des3->dst, des3->src_len,
1431990672d4SGary R Hook DES3_EDE_BLOCK_SIZE, DMA_FROM_DEVICE);
1432990672d4SGary R Hook if (ret)
1433990672d4SGary R Hook goto e_src;
1434990672d4SGary R Hook }
1435990672d4SGary R Hook
1436990672d4SGary R Hook /* Send data to the CCP DES3 engine */
1437990672d4SGary R Hook while (src.sg_wa.bytes_left) {
1438990672d4SGary R Hook ccp_prepare_data(&src, &dst, &op, DES3_EDE_BLOCK_SIZE, true);
1439990672d4SGary R Hook if (!src.sg_wa.bytes_left) {
1440990672d4SGary R Hook op.eom = 1;
1441990672d4SGary R Hook
1442990672d4SGary R Hook /* Since we don't retrieve the context in ECB mode
1443990672d4SGary R Hook * we have to wait for the operation to complete
1444990672d4SGary R Hook * on the last piece of data
1445990672d4SGary R Hook */
1446990672d4SGary R Hook op.soc = 0;
1447990672d4SGary R Hook }
1448990672d4SGary R Hook
1449990672d4SGary R Hook ret = cmd_q->ccp->vdata->perform->des3(&op);
1450990672d4SGary R Hook if (ret) {
1451990672d4SGary R Hook cmd->engine_error = cmd_q->cmd_error;
1452990672d4SGary R Hook goto e_dst;
1453990672d4SGary R Hook }
1454990672d4SGary R Hook
1455990672d4SGary R Hook ccp_process_data(&src, &dst, &op);
1456990672d4SGary R Hook }
1457990672d4SGary R Hook
1458990672d4SGary R Hook if (des3->mode != CCP_DES3_MODE_ECB) {
1459990672d4SGary R Hook /* Retrieve the context and make BE */
1460990672d4SGary R Hook ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1461990672d4SGary R Hook CCP_PASSTHRU_BYTESWAP_256BIT);
1462990672d4SGary R Hook if (ret) {
1463990672d4SGary R Hook cmd->engine_error = cmd_q->cmd_error;
1464990672d4SGary R Hook goto e_dst;
1465990672d4SGary R Hook }
1466990672d4SGary R Hook
1467990672d4SGary R Hook /* ...but we only need the last DES3_EDE_BLOCK_SIZE bytes */
1468990672d4SGary R Hook ccp_get_dm_area(&ctx, dm_offset, des3->iv, 0,
1469990672d4SGary R Hook DES3_EDE_BLOCK_SIZE);
1470990672d4SGary R Hook }
1471990672d4SGary R Hook e_dst:
1472990672d4SGary R Hook if (!in_place)
1473990672d4SGary R Hook ccp_free_data(&dst, cmd_q);
1474990672d4SGary R Hook
1475990672d4SGary R Hook e_src:
1476990672d4SGary R Hook ccp_free_data(&src, cmd_q);
1477990672d4SGary R Hook
1478990672d4SGary R Hook e_ctx:
1479990672d4SGary R Hook if (des3->mode != CCP_DES3_MODE_ECB)
1480990672d4SGary R Hook ccp_dm_free(&ctx);
1481990672d4SGary R Hook
1482990672d4SGary R Hook e_key:
1483990672d4SGary R Hook ccp_dm_free(&key);
1484990672d4SGary R Hook
1485990672d4SGary R Hook return ret;
1486990672d4SGary R Hook }
1487990672d4SGary R Hook
148872c8117aSArnd Bergmann static noinline_for_stack int
ccp_run_sha_cmd(struct ccp_cmd_queue * cmd_q,struct ccp_cmd * cmd)148972c8117aSArnd Bergmann ccp_run_sha_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
149063b94509STom Lendacky {
149163b94509STom Lendacky struct ccp_sha_engine *sha = &cmd->u.sha;
149263b94509STom Lendacky struct ccp_dm_workarea ctx;
149363b94509STom Lendacky struct ccp_data src;
149463b94509STom Lendacky struct ccp_op op;
14954b394a23SGary R Hook unsigned int ioffset, ooffset;
14964b394a23SGary R Hook unsigned int digest_size;
14974b394a23SGary R Hook int sb_count;
14984b394a23SGary R Hook const void *init;
14994b394a23SGary R Hook u64 block_size;
15004b394a23SGary R Hook int ctx_size;
150163b94509STom Lendacky int ret;
150263b94509STom Lendacky
15034b394a23SGary R Hook switch (sha->type) {
15044b394a23SGary R Hook case CCP_SHA_TYPE_1:
15054b394a23SGary R Hook if (sha->ctx_len < SHA1_DIGEST_SIZE)
150663b94509STom Lendacky return -EINVAL;
15074b394a23SGary R Hook block_size = SHA1_BLOCK_SIZE;
15084b394a23SGary R Hook break;
15094b394a23SGary R Hook case CCP_SHA_TYPE_224:
15104b394a23SGary R Hook if (sha->ctx_len < SHA224_DIGEST_SIZE)
15114b394a23SGary R Hook return -EINVAL;
15124b394a23SGary R Hook block_size = SHA224_BLOCK_SIZE;
15134b394a23SGary R Hook break;
15144b394a23SGary R Hook case CCP_SHA_TYPE_256:
15154b394a23SGary R Hook if (sha->ctx_len < SHA256_DIGEST_SIZE)
15164b394a23SGary R Hook return -EINVAL;
15174b394a23SGary R Hook block_size = SHA256_BLOCK_SIZE;
15184b394a23SGary R Hook break;
1519ccebcf3fSGary R Hook case CCP_SHA_TYPE_384:
1520ccebcf3fSGary R Hook if (cmd_q->ccp->vdata->version < CCP_VERSION(4, 0)
1521ccebcf3fSGary R Hook || sha->ctx_len < SHA384_DIGEST_SIZE)
1522ccebcf3fSGary R Hook return -EINVAL;
1523ccebcf3fSGary R Hook block_size = SHA384_BLOCK_SIZE;
1524ccebcf3fSGary R Hook break;
1525ccebcf3fSGary R Hook case CCP_SHA_TYPE_512:
1526ccebcf3fSGary R Hook if (cmd_q->ccp->vdata->version < CCP_VERSION(4, 0)
1527ccebcf3fSGary R Hook || sha->ctx_len < SHA512_DIGEST_SIZE)
1528ccebcf3fSGary R Hook return -EINVAL;
1529ccebcf3fSGary R Hook block_size = SHA512_BLOCK_SIZE;
1530ccebcf3fSGary R Hook break;
15314b394a23SGary R Hook default:
15324b394a23SGary R Hook return -EINVAL;
15334b394a23SGary R Hook }
153463b94509STom Lendacky
153563b94509STom Lendacky if (!sha->ctx)
153663b94509STom Lendacky return -EINVAL;
153763b94509STom Lendacky
15384b394a23SGary R Hook if (!sha->final && (sha->src_len & (block_size - 1)))
153963b94509STom Lendacky return -EINVAL;
154063b94509STom Lendacky
15414b394a23SGary R Hook /* The version 3 device can't handle zero-length input */
15424b394a23SGary R Hook if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0)) {
15434b394a23SGary R Hook
154463b94509STom Lendacky if (!sha->src_len) {
15454b394a23SGary R Hook unsigned int digest_len;
154663b94509STom Lendacky const u8 *sha_zero;
154763b94509STom Lendacky
154863b94509STom Lendacky /* Not final, just return */
154963b94509STom Lendacky if (!sha->final)
155063b94509STom Lendacky return 0;
155163b94509STom Lendacky
15524b394a23SGary R Hook /* CCP can't do a zero length sha operation so the
15534b394a23SGary R Hook * caller must buffer the data.
155463b94509STom Lendacky */
155563b94509STom Lendacky if (sha->msg_bits)
155663b94509STom Lendacky return -EINVAL;
155763b94509STom Lendacky
15584b394a23SGary R Hook /* The CCP cannot perform zero-length sha operations
15594b394a23SGary R Hook * so the caller is required to buffer data for the
15604b394a23SGary R Hook * final operation. However, a sha operation for a
15614b394a23SGary R Hook * message with a total length of zero is valid so
15624b394a23SGary R Hook * known values are required to supply the result.
156363b94509STom Lendacky */
156463b94509STom Lendacky switch (sha->type) {
156563b94509STom Lendacky case CCP_SHA_TYPE_1:
1566bdd75064SLABBE Corentin sha_zero = sha1_zero_message_hash;
15674b394a23SGary R Hook digest_len = SHA1_DIGEST_SIZE;
156863b94509STom Lendacky break;
156963b94509STom Lendacky case CCP_SHA_TYPE_224:
1570bdd75064SLABBE Corentin sha_zero = sha224_zero_message_hash;
15714b394a23SGary R Hook digest_len = SHA224_DIGEST_SIZE;
157263b94509STom Lendacky break;
157363b94509STom Lendacky case CCP_SHA_TYPE_256:
1574bdd75064SLABBE Corentin sha_zero = sha256_zero_message_hash;
15754b394a23SGary R Hook digest_len = SHA256_DIGEST_SIZE;
157663b94509STom Lendacky break;
157763b94509STom Lendacky default:
157863b94509STom Lendacky return -EINVAL;
157963b94509STom Lendacky }
158063b94509STom Lendacky
158163b94509STom Lendacky scatterwalk_map_and_copy((void *)sha_zero, sha->ctx, 0,
15824b394a23SGary R Hook digest_len, 1);
158363b94509STom Lendacky
158463b94509STom Lendacky return 0;
158563b94509STom Lendacky }
15864b394a23SGary R Hook }
158763b94509STom Lendacky
15884b394a23SGary R Hook /* Set variables used throughout */
15894b394a23SGary R Hook switch (sha->type) {
15904b394a23SGary R Hook case CCP_SHA_TYPE_1:
15914b394a23SGary R Hook digest_size = SHA1_DIGEST_SIZE;
15924b394a23SGary R Hook init = (void *) ccp_sha1_init;
15934b394a23SGary R Hook ctx_size = SHA1_DIGEST_SIZE;
15944b394a23SGary R Hook sb_count = 1;
15954b394a23SGary R Hook if (cmd_q->ccp->vdata->version != CCP_VERSION(3, 0))
15964b394a23SGary R Hook ooffset = ioffset = CCP_SB_BYTES - SHA1_DIGEST_SIZE;
15974b394a23SGary R Hook else
15984b394a23SGary R Hook ooffset = ioffset = 0;
15994b394a23SGary R Hook break;
16004b394a23SGary R Hook case CCP_SHA_TYPE_224:
16014b394a23SGary R Hook digest_size = SHA224_DIGEST_SIZE;
16024b394a23SGary R Hook init = (void *) ccp_sha224_init;
16034b394a23SGary R Hook ctx_size = SHA256_DIGEST_SIZE;
16044b394a23SGary R Hook sb_count = 1;
16054b394a23SGary R Hook ioffset = 0;
16064b394a23SGary R Hook if (cmd_q->ccp->vdata->version != CCP_VERSION(3, 0))
16074b394a23SGary R Hook ooffset = CCP_SB_BYTES - SHA224_DIGEST_SIZE;
16084b394a23SGary R Hook else
16094b394a23SGary R Hook ooffset = 0;
16104b394a23SGary R Hook break;
16114b394a23SGary R Hook case CCP_SHA_TYPE_256:
16124b394a23SGary R Hook digest_size = SHA256_DIGEST_SIZE;
16134b394a23SGary R Hook init = (void *) ccp_sha256_init;
16144b394a23SGary R Hook ctx_size = SHA256_DIGEST_SIZE;
16154b394a23SGary R Hook sb_count = 1;
16164b394a23SGary R Hook ooffset = ioffset = 0;
16174b394a23SGary R Hook break;
1618ccebcf3fSGary R Hook case CCP_SHA_TYPE_384:
1619ccebcf3fSGary R Hook digest_size = SHA384_DIGEST_SIZE;
1620ccebcf3fSGary R Hook init = (void *) ccp_sha384_init;
1621ccebcf3fSGary R Hook ctx_size = SHA512_DIGEST_SIZE;
1622ccebcf3fSGary R Hook sb_count = 2;
1623ccebcf3fSGary R Hook ioffset = 0;
1624ccebcf3fSGary R Hook ooffset = 2 * CCP_SB_BYTES - SHA384_DIGEST_SIZE;
1625ccebcf3fSGary R Hook break;
1626ccebcf3fSGary R Hook case CCP_SHA_TYPE_512:
1627ccebcf3fSGary R Hook digest_size = SHA512_DIGEST_SIZE;
1628ccebcf3fSGary R Hook init = (void *) ccp_sha512_init;
1629ccebcf3fSGary R Hook ctx_size = SHA512_DIGEST_SIZE;
1630ccebcf3fSGary R Hook sb_count = 2;
1631ccebcf3fSGary R Hook ooffset = ioffset = 0;
1632ccebcf3fSGary R Hook break;
16334b394a23SGary R Hook default:
16344b394a23SGary R Hook ret = -EINVAL;
16354b394a23SGary R Hook goto e_data;
16364b394a23SGary R Hook }
16374b394a23SGary R Hook
16384b394a23SGary R Hook /* For zero-length plaintext the src pointer is ignored;
16394b394a23SGary R Hook * otherwise both parts must be valid
16404b394a23SGary R Hook */
16414b394a23SGary R Hook if (sha->src_len && !sha->src)
164263b94509STom Lendacky return -EINVAL;
164363b94509STom Lendacky
164463b94509STom Lendacky memset(&op, 0, sizeof(op));
164563b94509STom Lendacky op.cmd_q = cmd_q;
16464b394a23SGary R Hook op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
16474b394a23SGary R Hook op.sb_ctx = cmd_q->sb_ctx; /* Pre-allocated */
164863b94509STom Lendacky op.u.sha.type = sha->type;
164963b94509STom Lendacky op.u.sha.msg_bits = sha->msg_bits;
165063b94509STom Lendacky
1651ccebcf3fSGary R Hook /* For SHA1/224/256 the context fits in a single (32-byte) SB entry;
1652ccebcf3fSGary R Hook * SHA384/512 require 2 adjacent SB slots, with the right half in the
1653ccebcf3fSGary R Hook * first slot, and the left half in the second. Each portion must then
1654ccebcf3fSGary R Hook * be in little endian format: use the 256-bit byte swap option.
1655ccebcf3fSGary R Hook */
16564b394a23SGary R Hook ret = ccp_init_dm_workarea(&ctx, cmd_q, sb_count * CCP_SB_BYTES,
165763b94509STom Lendacky DMA_BIDIRECTIONAL);
165863b94509STom Lendacky if (ret)
165963b94509STom Lendacky return ret;
1660c11baa02STom Lendacky if (sha->first) {
1661c11baa02STom Lendacky switch (sha->type) {
1662c11baa02STom Lendacky case CCP_SHA_TYPE_1:
1663c11baa02STom Lendacky case CCP_SHA_TYPE_224:
1664c11baa02STom Lendacky case CCP_SHA_TYPE_256:
16654b394a23SGary R Hook memcpy(ctx.address + ioffset, init, ctx_size);
1666c11baa02STom Lendacky break;
1667ccebcf3fSGary R Hook case CCP_SHA_TYPE_384:
1668ccebcf3fSGary R Hook case CCP_SHA_TYPE_512:
1669ccebcf3fSGary R Hook memcpy(ctx.address + ctx_size / 2, init,
1670ccebcf3fSGary R Hook ctx_size / 2);
1671ccebcf3fSGary R Hook memcpy(ctx.address, init + ctx_size / 2,
1672ccebcf3fSGary R Hook ctx_size / 2);
1673ccebcf3fSGary R Hook break;
1674c11baa02STom Lendacky default:
1675c11baa02STom Lendacky ret = -EINVAL;
1676c11baa02STom Lendacky goto e_ctx;
1677c11baa02STom Lendacky }
16788db88467STom Lendacky } else {
16794b394a23SGary R Hook /* Restore the context */
1680b698a9f4SGary R Hook ret = ccp_set_dm_area(&ctx, 0, sha->ctx, 0,
16814b394a23SGary R Hook sb_count * CCP_SB_BYTES);
1682b698a9f4SGary R Hook if (ret)
1683b698a9f4SGary R Hook goto e_ctx;
16848db88467STom Lendacky }
1685c11baa02STom Lendacky
1686956ee21aSGary R Hook ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
168763b94509STom Lendacky CCP_PASSTHRU_BYTESWAP_256BIT);
168863b94509STom Lendacky if (ret) {
168963b94509STom Lendacky cmd->engine_error = cmd_q->cmd_error;
169063b94509STom Lendacky goto e_ctx;
169163b94509STom Lendacky }
169263b94509STom Lendacky
16934b394a23SGary R Hook if (sha->src) {
16944b394a23SGary R Hook /* Send data to the CCP SHA engine; block_size is set above */
169563b94509STom Lendacky ret = ccp_init_data(&src, cmd_q, sha->src, sha->src_len,
16964b394a23SGary R Hook block_size, DMA_TO_DEVICE);
169763b94509STom Lendacky if (ret)
169863b94509STom Lendacky goto e_ctx;
169963b94509STom Lendacky
170063b94509STom Lendacky while (src.sg_wa.bytes_left) {
17014b394a23SGary R Hook ccp_prepare_data(&src, NULL, &op, block_size, false);
170263b94509STom Lendacky if (sha->final && !src.sg_wa.bytes_left)
170363b94509STom Lendacky op.eom = 1;
170463b94509STom Lendacky
1705a43eb985SGary R Hook ret = cmd_q->ccp->vdata->perform->sha(&op);
170663b94509STom Lendacky if (ret) {
170763b94509STom Lendacky cmd->engine_error = cmd_q->cmd_error;
170863b94509STom Lendacky goto e_data;
170963b94509STom Lendacky }
171063b94509STom Lendacky
171163b94509STom Lendacky ccp_process_data(&src, NULL, &op);
171263b94509STom Lendacky }
17134b394a23SGary R Hook } else {
17144b394a23SGary R Hook op.eom = 1;
17154b394a23SGary R Hook ret = cmd_q->ccp->vdata->perform->sha(&op);
17164b394a23SGary R Hook if (ret) {
17174b394a23SGary R Hook cmd->engine_error = cmd_q->cmd_error;
17184b394a23SGary R Hook goto e_data;
17194b394a23SGary R Hook }
17204b394a23SGary R Hook }
172163b94509STom Lendacky
172263b94509STom Lendacky /* Retrieve the SHA context - convert from LE to BE using
172363b94509STom Lendacky * 32-byte (256-bit) byteswapping to BE
172463b94509STom Lendacky */
1725956ee21aSGary R Hook ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
172663b94509STom Lendacky CCP_PASSTHRU_BYTESWAP_256BIT);
172763b94509STom Lendacky if (ret) {
172863b94509STom Lendacky cmd->engine_error = cmd_q->cmd_error;
172963b94509STom Lendacky goto e_data;
173063b94509STom Lendacky }
173163b94509STom Lendacky
17324b394a23SGary R Hook if (sha->final) {
17334b394a23SGary R Hook /* Finishing up, so get the digest */
17344b394a23SGary R Hook switch (sha->type) {
17354b394a23SGary R Hook case CCP_SHA_TYPE_1:
17364b394a23SGary R Hook case CCP_SHA_TYPE_224:
17374b394a23SGary R Hook case CCP_SHA_TYPE_256:
17384b394a23SGary R Hook ccp_get_dm_area(&ctx, ooffset,
17394b394a23SGary R Hook sha->ctx, 0,
17404b394a23SGary R Hook digest_size);
17414b394a23SGary R Hook break;
1742ccebcf3fSGary R Hook case CCP_SHA_TYPE_384:
1743ccebcf3fSGary R Hook case CCP_SHA_TYPE_512:
1744ccebcf3fSGary R Hook ccp_get_dm_area(&ctx, 0,
1745ccebcf3fSGary R Hook sha->ctx, LSB_ITEM_SIZE - ooffset,
1746ccebcf3fSGary R Hook LSB_ITEM_SIZE);
1747ccebcf3fSGary R Hook ccp_get_dm_area(&ctx, LSB_ITEM_SIZE + ooffset,
1748ccebcf3fSGary R Hook sha->ctx, 0,
1749ccebcf3fSGary R Hook LSB_ITEM_SIZE - ooffset);
1750ccebcf3fSGary R Hook break;
17514b394a23SGary R Hook default:
17524b394a23SGary R Hook ret = -EINVAL;
1753e356c49cSPavel Machek goto e_data;
17544b394a23SGary R Hook }
17554b394a23SGary R Hook } else {
17564b394a23SGary R Hook /* Stash the context */
17574b394a23SGary R Hook ccp_get_dm_area(&ctx, 0, sha->ctx, 0,
17584b394a23SGary R Hook sb_count * CCP_SB_BYTES);
17594b394a23SGary R Hook }
176063b94509STom Lendacky
1761c11baa02STom Lendacky if (sha->final && sha->opad) {
1762c11baa02STom Lendacky /* HMAC operation, recursively perform final SHA */
1763c11baa02STom Lendacky struct ccp_cmd hmac_cmd;
1764c11baa02STom Lendacky struct scatterlist sg;
1765c11baa02STom Lendacky u8 *hmac_buf;
1766c11baa02STom Lendacky
1767c11baa02STom Lendacky if (sha->opad_len != block_size) {
1768c11baa02STom Lendacky ret = -EINVAL;
1769c11baa02STom Lendacky goto e_data;
1770c11baa02STom Lendacky }
1771c11baa02STom Lendacky
1772c11baa02STom Lendacky hmac_buf = kmalloc(block_size + digest_size, GFP_KERNEL);
1773c11baa02STom Lendacky if (!hmac_buf) {
1774c11baa02STom Lendacky ret = -ENOMEM;
1775c11baa02STom Lendacky goto e_data;
1776c11baa02STom Lendacky }
1777c11baa02STom Lendacky sg_init_one(&sg, hmac_buf, block_size + digest_size);
1778c11baa02STom Lendacky
1779c11baa02STom Lendacky scatterwalk_map_and_copy(hmac_buf, sha->opad, 0, block_size, 0);
17804b394a23SGary R Hook switch (sha->type) {
17814b394a23SGary R Hook case CCP_SHA_TYPE_1:
17824b394a23SGary R Hook case CCP_SHA_TYPE_224:
17834b394a23SGary R Hook case CCP_SHA_TYPE_256:
17844b394a23SGary R Hook memcpy(hmac_buf + block_size,
17854b394a23SGary R Hook ctx.address + ooffset,
17864b394a23SGary R Hook digest_size);
17874b394a23SGary R Hook break;
1788ccebcf3fSGary R Hook case CCP_SHA_TYPE_384:
1789ccebcf3fSGary R Hook case CCP_SHA_TYPE_512:
1790ccebcf3fSGary R Hook memcpy(hmac_buf + block_size,
1791ccebcf3fSGary R Hook ctx.address + LSB_ITEM_SIZE + ooffset,
1792ccebcf3fSGary R Hook LSB_ITEM_SIZE);
1793ccebcf3fSGary R Hook memcpy(hmac_buf + block_size +
1794ccebcf3fSGary R Hook (LSB_ITEM_SIZE - ooffset),
1795ccebcf3fSGary R Hook ctx.address,
1796ccebcf3fSGary R Hook LSB_ITEM_SIZE);
1797ccebcf3fSGary R Hook break;
17984b394a23SGary R Hook default:
1799128c6642SNavid Emamdoost kfree(hmac_buf);
18004b394a23SGary R Hook ret = -EINVAL;
1801128c6642SNavid Emamdoost goto e_data;
18024b394a23SGary R Hook }
1803c11baa02STom Lendacky
1804c11baa02STom Lendacky memset(&hmac_cmd, 0, sizeof(hmac_cmd));
1805c11baa02STom Lendacky hmac_cmd.engine = CCP_ENGINE_SHA;
1806c11baa02STom Lendacky hmac_cmd.u.sha.type = sha->type;
1807c11baa02STom Lendacky hmac_cmd.u.sha.ctx = sha->ctx;
1808c11baa02STom Lendacky hmac_cmd.u.sha.ctx_len = sha->ctx_len;
1809c11baa02STom Lendacky hmac_cmd.u.sha.src = &sg;
1810c11baa02STom Lendacky hmac_cmd.u.sha.src_len = block_size + digest_size;
1811c11baa02STom Lendacky hmac_cmd.u.sha.opad = NULL;
1812c11baa02STom Lendacky hmac_cmd.u.sha.opad_len = 0;
1813c11baa02STom Lendacky hmac_cmd.u.sha.first = 1;
1814c11baa02STom Lendacky hmac_cmd.u.sha.final = 1;
1815c11baa02STom Lendacky hmac_cmd.u.sha.msg_bits = (block_size + digest_size) << 3;
1816c11baa02STom Lendacky
1817c11baa02STom Lendacky ret = ccp_run_sha_cmd(cmd_q, &hmac_cmd);
1818c11baa02STom Lendacky if (ret)
1819c11baa02STom Lendacky cmd->engine_error = hmac_cmd.engine_error;
1820c11baa02STom Lendacky
1821c11baa02STom Lendacky kfree(hmac_buf);
1822c11baa02STom Lendacky }
1823c11baa02STom Lendacky
182463b94509STom Lendacky e_data:
18254b394a23SGary R Hook if (sha->src)
182663b94509STom Lendacky ccp_free_data(&src, cmd_q);
182763b94509STom Lendacky
182863b94509STom Lendacky e_ctx:
182963b94509STom Lendacky ccp_dm_free(&ctx);
183063b94509STom Lendacky
183163b94509STom Lendacky return ret;
183263b94509STom Lendacky }
183363b94509STom Lendacky
183472c8117aSArnd Bergmann static noinline_for_stack int
ccp_run_rsa_cmd(struct ccp_cmd_queue * cmd_q,struct ccp_cmd * cmd)183572c8117aSArnd Bergmann ccp_run_rsa_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
183663b94509STom Lendacky {
183763b94509STom Lendacky struct ccp_rsa_engine *rsa = &cmd->u.rsa;
18386ba46c7dSGary R Hook struct ccp_dm_workarea exp, src, dst;
183963b94509STom Lendacky struct ccp_op op;
1840956ee21aSGary R Hook unsigned int sb_count, i_len, o_len;
184163b94509STom Lendacky int ret;
184263b94509STom Lendacky
1843e28c190dSGary R Hook /* Check against the maximum allowable size, in bits */
1844e28c190dSGary R Hook if (rsa->key_size > cmd_q->ccp->vdata->rsamax)
184563b94509STom Lendacky return -EINVAL;
184663b94509STom Lendacky
184763b94509STom Lendacky if (!rsa->exp || !rsa->mod || !rsa->src || !rsa->dst)
184863b94509STom Lendacky return -EINVAL;
184963b94509STom Lendacky
18506ba46c7dSGary R Hook memset(&op, 0, sizeof(op));
18516ba46c7dSGary R Hook op.cmd_q = cmd_q;
18526ba46c7dSGary R Hook op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
18536ba46c7dSGary R Hook
185463b94509STom Lendacky /* The RSA modulus must precede the message being acted upon, so
185563b94509STom Lendacky * it must be copied to a DMA area where the message and the
185663b94509STom Lendacky * modulus can be concatenated. Therefore the input buffer
185763b94509STom Lendacky * length required is twice the output buffer length (which
18586ba46c7dSGary R Hook * must be a multiple of 256-bits). Compute o_len, i_len in bytes.
18596ba46c7dSGary R Hook * Buffer sizes must be a multiple of 32 bytes; rounding up may be
18606ba46c7dSGary R Hook * required.
186163b94509STom Lendacky */
18626ba46c7dSGary R Hook o_len = 32 * ((rsa->key_size + 255) / 256);
186363b94509STom Lendacky i_len = o_len * 2;
186463b94509STom Lendacky
1865d634baeaSArnd Bergmann sb_count = 0;
18666ba46c7dSGary R Hook if (cmd_q->ccp->vdata->version < CCP_VERSION(5, 0)) {
18676ba46c7dSGary R Hook /* sb_count is the number of storage block slots required
18686ba46c7dSGary R Hook * for the modulus.
18696ba46c7dSGary R Hook */
1870956ee21aSGary R Hook sb_count = o_len / CCP_SB_BYTES;
18716ba46c7dSGary R Hook op.sb_key = cmd_q->ccp->vdata->perform->sballoc(cmd_q,
18726ba46c7dSGary R Hook sb_count);
1873956ee21aSGary R Hook if (!op.sb_key)
187463b94509STom Lendacky return -EIO;
18756ba46c7dSGary R Hook } else {
18766ba46c7dSGary R Hook /* A version 5 device allows a modulus size that will not fit
18776ba46c7dSGary R Hook * in the LSB, so the command will transfer it from memory.
18786ba46c7dSGary R Hook * Set the sb key to the default, even though it's not used.
18796ba46c7dSGary R Hook */
18806ba46c7dSGary R Hook op.sb_key = cmd_q->sb_key;
18816ba46c7dSGary R Hook }
188263b94509STom Lendacky
18836ba46c7dSGary R Hook /* The RSA exponent must be in little endian format. Reverse its
18846ba46c7dSGary R Hook * byte order.
188563b94509STom Lendacky */
188663b94509STom Lendacky ret = ccp_init_dm_workarea(&exp, cmd_q, o_len, DMA_TO_DEVICE);
188763b94509STom Lendacky if (ret)
1888956ee21aSGary R Hook goto e_sb;
188963b94509STom Lendacky
189083d650abSGary R Hook ret = ccp_reverse_set_dm_area(&exp, 0, rsa->exp, 0, rsa->exp_len);
1891355eba5dSTom Lendacky if (ret)
1892355eba5dSTom Lendacky goto e_exp;
18936ba46c7dSGary R Hook
18946ba46c7dSGary R Hook if (cmd_q->ccp->vdata->version < CCP_VERSION(5, 0)) {
18956ba46c7dSGary R Hook /* Copy the exponent to the local storage block, using
18966ba46c7dSGary R Hook * as many 32-byte blocks as were allocated above. It's
18976ba46c7dSGary R Hook * already little endian, so no further change is required.
18986ba46c7dSGary R Hook */
1899956ee21aSGary R Hook ret = ccp_copy_to_sb(cmd_q, &exp, op.jobid, op.sb_key,
190063b94509STom Lendacky CCP_PASSTHRU_BYTESWAP_NOOP);
190163b94509STom Lendacky if (ret) {
190263b94509STom Lendacky cmd->engine_error = cmd_q->cmd_error;
190363b94509STom Lendacky goto e_exp;
190463b94509STom Lendacky }
19056ba46c7dSGary R Hook } else {
19066ba46c7dSGary R Hook /* The exponent can be retrieved from memory via DMA. */
19076ba46c7dSGary R Hook op.exp.u.dma.address = exp.dma.address;
19086ba46c7dSGary R Hook op.exp.u.dma.offset = 0;
19096ba46c7dSGary R Hook }
191063b94509STom Lendacky
191163b94509STom Lendacky /* Concatenate the modulus and the message. Both the modulus and
191263b94509STom Lendacky * the operands must be in little endian format. Since the input
191363b94509STom Lendacky * is in big endian format it must be converted.
191463b94509STom Lendacky */
191563b94509STom Lendacky ret = ccp_init_dm_workarea(&src, cmd_q, i_len, DMA_TO_DEVICE);
191663b94509STom Lendacky if (ret)
191763b94509STom Lendacky goto e_exp;
191863b94509STom Lendacky
191983d650abSGary R Hook ret = ccp_reverse_set_dm_area(&src, 0, rsa->mod, 0, rsa->mod_len);
1920355eba5dSTom Lendacky if (ret)
1921355eba5dSTom Lendacky goto e_src;
192283d650abSGary R Hook ret = ccp_reverse_set_dm_area(&src, o_len, rsa->src, 0, rsa->src_len);
1923355eba5dSTom Lendacky if (ret)
1924355eba5dSTom Lendacky goto e_src;
192563b94509STom Lendacky
192663b94509STom Lendacky /* Prepare the output area for the operation */
19276ba46c7dSGary R Hook ret = ccp_init_dm_workarea(&dst, cmd_q, o_len, DMA_FROM_DEVICE);
192863b94509STom Lendacky if (ret)
192963b94509STom Lendacky goto e_src;
193063b94509STom Lendacky
193163b94509STom Lendacky op.soc = 1;
193263b94509STom Lendacky op.src.u.dma.address = src.dma.address;
193363b94509STom Lendacky op.src.u.dma.offset = 0;
193463b94509STom Lendacky op.src.u.dma.length = i_len;
19356ba46c7dSGary R Hook op.dst.u.dma.address = dst.dma.address;
193663b94509STom Lendacky op.dst.u.dma.offset = 0;
193763b94509STom Lendacky op.dst.u.dma.length = o_len;
193863b94509STom Lendacky
193963b94509STom Lendacky op.u.rsa.mod_size = rsa->key_size;
194063b94509STom Lendacky op.u.rsa.input_len = i_len;
194163b94509STom Lendacky
1942a43eb985SGary R Hook ret = cmd_q->ccp->vdata->perform->rsa(&op);
194363b94509STom Lendacky if (ret) {
194463b94509STom Lendacky cmd->engine_error = cmd_q->cmd_error;
194563b94509STom Lendacky goto e_dst;
194663b94509STom Lendacky }
194763b94509STom Lendacky
19486ba46c7dSGary R Hook ccp_reverse_get_dm_area(&dst, 0, rsa->dst, 0, rsa->mod_len);
194963b94509STom Lendacky
195063b94509STom Lendacky e_dst:
19516ba46c7dSGary R Hook ccp_dm_free(&dst);
195263b94509STom Lendacky
195363b94509STom Lendacky e_src:
195463b94509STom Lendacky ccp_dm_free(&src);
195563b94509STom Lendacky
195663b94509STom Lendacky e_exp:
195763b94509STom Lendacky ccp_dm_free(&exp);
195863b94509STom Lendacky
1959956ee21aSGary R Hook e_sb:
1960d634baeaSArnd Bergmann if (sb_count)
196158a690b7SGary R Hook cmd_q->ccp->vdata->perform->sbfree(cmd_q, op.sb_key, sb_count);
196263b94509STom Lendacky
196363b94509STom Lendacky return ret;
196463b94509STom Lendacky }
196563b94509STom Lendacky
196672c8117aSArnd Bergmann static noinline_for_stack int
ccp_run_passthru_cmd(struct ccp_cmd_queue * cmd_q,struct ccp_cmd * cmd)196772c8117aSArnd Bergmann ccp_run_passthru_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
196863b94509STom Lendacky {
196963b94509STom Lendacky struct ccp_passthru_engine *pt = &cmd->u.passthru;
197063b94509STom Lendacky struct ccp_dm_workarea mask;
197163b94509STom Lendacky struct ccp_data src, dst;
197263b94509STom Lendacky struct ccp_op op;
197363b94509STom Lendacky bool in_place = false;
197463b94509STom Lendacky unsigned int i;
19754b394a23SGary R Hook int ret = 0;
197663b94509STom Lendacky
197763b94509STom Lendacky if (!pt->final && (pt->src_len & (CCP_PASSTHRU_BLOCKSIZE - 1)))
197863b94509STom Lendacky return -EINVAL;
197963b94509STom Lendacky
198063b94509STom Lendacky if (!pt->src || !pt->dst)
198163b94509STom Lendacky return -EINVAL;
198263b94509STom Lendacky
198363b94509STom Lendacky if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
198463b94509STom Lendacky if (pt->mask_len != CCP_PASSTHRU_MASKSIZE)
198563b94509STom Lendacky return -EINVAL;
198663b94509STom Lendacky if (!pt->mask)
198763b94509STom Lendacky return -EINVAL;
198863b94509STom Lendacky }
198963b94509STom Lendacky
1990956ee21aSGary R Hook BUILD_BUG_ON(CCP_PASSTHRU_SB_COUNT != 1);
199163b94509STom Lendacky
199263b94509STom Lendacky memset(&op, 0, sizeof(op));
199363b94509STom Lendacky op.cmd_q = cmd_q;
19944b394a23SGary R Hook op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
199563b94509STom Lendacky
199663b94509STom Lendacky if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
199763b94509STom Lendacky /* Load the mask */
1998956ee21aSGary R Hook op.sb_key = cmd_q->sb_key;
199963b94509STom Lendacky
200063b94509STom Lendacky ret = ccp_init_dm_workarea(&mask, cmd_q,
2001956ee21aSGary R Hook CCP_PASSTHRU_SB_COUNT *
2002956ee21aSGary R Hook CCP_SB_BYTES,
200363b94509STom Lendacky DMA_TO_DEVICE);
200463b94509STom Lendacky if (ret)
200563b94509STom Lendacky return ret;
200663b94509STom Lendacky
2007b698a9f4SGary R Hook ret = ccp_set_dm_area(&mask, 0, pt->mask, 0, pt->mask_len);
2008b698a9f4SGary R Hook if (ret)
2009b698a9f4SGary R Hook goto e_mask;
2010956ee21aSGary R Hook ret = ccp_copy_to_sb(cmd_q, &mask, op.jobid, op.sb_key,
201163b94509STom Lendacky CCP_PASSTHRU_BYTESWAP_NOOP);
201263b94509STom Lendacky if (ret) {
201363b94509STom Lendacky cmd->engine_error = cmd_q->cmd_error;
201463b94509STom Lendacky goto e_mask;
201563b94509STom Lendacky }
201663b94509STom Lendacky }
201763b94509STom Lendacky
201863b94509STom Lendacky /* Prepare the input and output data workareas. For in-place
201963b94509STom Lendacky * operations we need to set the dma direction to BIDIRECTIONAL
202063b94509STom Lendacky * and copy the src workarea to the dst workarea.
202163b94509STom Lendacky */
202263b94509STom Lendacky if (sg_virt(pt->src) == sg_virt(pt->dst))
202363b94509STom Lendacky in_place = true;
202463b94509STom Lendacky
202563b94509STom Lendacky ret = ccp_init_data(&src, cmd_q, pt->src, pt->src_len,
202663b94509STom Lendacky CCP_PASSTHRU_MASKSIZE,
202763b94509STom Lendacky in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
202863b94509STom Lendacky if (ret)
202963b94509STom Lendacky goto e_mask;
203063b94509STom Lendacky
20318db88467STom Lendacky if (in_place) {
203263b94509STom Lendacky dst = src;
20338db88467STom Lendacky } else {
203463b94509STom Lendacky ret = ccp_init_data(&dst, cmd_q, pt->dst, pt->src_len,
203563b94509STom Lendacky CCP_PASSTHRU_MASKSIZE, DMA_FROM_DEVICE);
203663b94509STom Lendacky if (ret)
203763b94509STom Lendacky goto e_src;
203863b94509STom Lendacky }
203963b94509STom Lendacky
204063b94509STom Lendacky /* Send data to the CCP Passthru engine
204163b94509STom Lendacky * Because the CCP engine works on a single source and destination
204263b94509STom Lendacky * dma address at a time, each entry in the source scatterlist
204363b94509STom Lendacky * (after the dma_map_sg call) must be less than or equal to the
204463b94509STom Lendacky * (remaining) length in the destination scatterlist entry and the
204563b94509STom Lendacky * length must be a multiple of CCP_PASSTHRU_BLOCKSIZE
204663b94509STom Lendacky */
204763b94509STom Lendacky dst.sg_wa.sg_used = 0;
204863b94509STom Lendacky for (i = 1; i <= src.sg_wa.dma_count; i++) {
204963b94509STom Lendacky if (!dst.sg_wa.sg ||
20508a302808SJohn Allen (sg_dma_len(dst.sg_wa.sg) < sg_dma_len(src.sg_wa.sg))) {
205163b94509STom Lendacky ret = -EINVAL;
205263b94509STom Lendacky goto e_dst;
205363b94509STom Lendacky }
205463b94509STom Lendacky
205563b94509STom Lendacky if (i == src.sg_wa.dma_count) {
205663b94509STom Lendacky op.eom = 1;
205763b94509STom Lendacky op.soc = 1;
205863b94509STom Lendacky }
205963b94509STom Lendacky
206063b94509STom Lendacky op.src.type = CCP_MEMTYPE_SYSTEM;
206163b94509STom Lendacky op.src.u.dma.address = sg_dma_address(src.sg_wa.sg);
206263b94509STom Lendacky op.src.u.dma.offset = 0;
206363b94509STom Lendacky op.src.u.dma.length = sg_dma_len(src.sg_wa.sg);
206463b94509STom Lendacky
206563b94509STom Lendacky op.dst.type = CCP_MEMTYPE_SYSTEM;
206663b94509STom Lendacky op.dst.u.dma.address = sg_dma_address(dst.sg_wa.sg);
206780e84c16SDave Jones op.dst.u.dma.offset = dst.sg_wa.sg_used;
206880e84c16SDave Jones op.dst.u.dma.length = op.src.u.dma.length;
206963b94509STom Lendacky
2070a43eb985SGary R Hook ret = cmd_q->ccp->vdata->perform->passthru(&op);
207163b94509STom Lendacky if (ret) {
207263b94509STom Lendacky cmd->engine_error = cmd_q->cmd_error;
207363b94509STom Lendacky goto e_dst;
207463b94509STom Lendacky }
207563b94509STom Lendacky
20768a302808SJohn Allen dst.sg_wa.sg_used += sg_dma_len(src.sg_wa.sg);
20778a302808SJohn Allen if (dst.sg_wa.sg_used == sg_dma_len(dst.sg_wa.sg)) {
207863b94509STom Lendacky dst.sg_wa.sg = sg_next(dst.sg_wa.sg);
207963b94509STom Lendacky dst.sg_wa.sg_used = 0;
208063b94509STom Lendacky }
208163b94509STom Lendacky src.sg_wa.sg = sg_next(src.sg_wa.sg);
208263b94509STom Lendacky }
208363b94509STom Lendacky
208463b94509STom Lendacky e_dst:
208563b94509STom Lendacky if (!in_place)
208663b94509STom Lendacky ccp_free_data(&dst, cmd_q);
208763b94509STom Lendacky
208863b94509STom Lendacky e_src:
208963b94509STom Lendacky ccp_free_data(&src, cmd_q);
209063b94509STom Lendacky
209163b94509STom Lendacky e_mask:
209263b94509STom Lendacky if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP)
209363b94509STom Lendacky ccp_dm_free(&mask);
209463b94509STom Lendacky
209563b94509STom Lendacky return ret;
209663b94509STom Lendacky }
209763b94509STom Lendacky
209872c8117aSArnd Bergmann static noinline_for_stack int
ccp_run_passthru_nomap_cmd(struct ccp_cmd_queue * cmd_q,struct ccp_cmd * cmd)209972c8117aSArnd Bergmann ccp_run_passthru_nomap_cmd(struct ccp_cmd_queue *cmd_q,
210058ea8abfSGary R Hook struct ccp_cmd *cmd)
210158ea8abfSGary R Hook {
210258ea8abfSGary R Hook struct ccp_passthru_nomap_engine *pt = &cmd->u.passthru_nomap;
210358ea8abfSGary R Hook struct ccp_dm_workarea mask;
210458ea8abfSGary R Hook struct ccp_op op;
210558ea8abfSGary R Hook int ret;
210658ea8abfSGary R Hook
210758ea8abfSGary R Hook if (!pt->final && (pt->src_len & (CCP_PASSTHRU_BLOCKSIZE - 1)))
210858ea8abfSGary R Hook return -EINVAL;
210958ea8abfSGary R Hook
211058ea8abfSGary R Hook if (!pt->src_dma || !pt->dst_dma)
211158ea8abfSGary R Hook return -EINVAL;
211258ea8abfSGary R Hook
211358ea8abfSGary R Hook if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
211458ea8abfSGary R Hook if (pt->mask_len != CCP_PASSTHRU_MASKSIZE)
211558ea8abfSGary R Hook return -EINVAL;
211658ea8abfSGary R Hook if (!pt->mask)
211758ea8abfSGary R Hook return -EINVAL;
211858ea8abfSGary R Hook }
211958ea8abfSGary R Hook
2120956ee21aSGary R Hook BUILD_BUG_ON(CCP_PASSTHRU_SB_COUNT != 1);
212158ea8abfSGary R Hook
212258ea8abfSGary R Hook memset(&op, 0, sizeof(op));
212358ea8abfSGary R Hook op.cmd_q = cmd_q;
2124bce386afSGary R Hook op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
212558ea8abfSGary R Hook
212658ea8abfSGary R Hook if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
212758ea8abfSGary R Hook /* Load the mask */
2128956ee21aSGary R Hook op.sb_key = cmd_q->sb_key;
212958ea8abfSGary R Hook
213058ea8abfSGary R Hook mask.length = pt->mask_len;
213158ea8abfSGary R Hook mask.dma.address = pt->mask;
213258ea8abfSGary R Hook mask.dma.length = pt->mask_len;
213358ea8abfSGary R Hook
2134956ee21aSGary R Hook ret = ccp_copy_to_sb(cmd_q, &mask, op.jobid, op.sb_key,
213558ea8abfSGary R Hook CCP_PASSTHRU_BYTESWAP_NOOP);
213658ea8abfSGary R Hook if (ret) {
213758ea8abfSGary R Hook cmd->engine_error = cmd_q->cmd_error;
213858ea8abfSGary R Hook return ret;
213958ea8abfSGary R Hook }
214058ea8abfSGary R Hook }
214158ea8abfSGary R Hook
214258ea8abfSGary R Hook /* Send data to the CCP Passthru engine */
214358ea8abfSGary R Hook op.eom = 1;
214458ea8abfSGary R Hook op.soc = 1;
214558ea8abfSGary R Hook
214658ea8abfSGary R Hook op.src.type = CCP_MEMTYPE_SYSTEM;
214758ea8abfSGary R Hook op.src.u.dma.address = pt->src_dma;
214858ea8abfSGary R Hook op.src.u.dma.offset = 0;
214958ea8abfSGary R Hook op.src.u.dma.length = pt->src_len;
215058ea8abfSGary R Hook
215158ea8abfSGary R Hook op.dst.type = CCP_MEMTYPE_SYSTEM;
215258ea8abfSGary R Hook op.dst.u.dma.address = pt->dst_dma;
215358ea8abfSGary R Hook op.dst.u.dma.offset = 0;
215458ea8abfSGary R Hook op.dst.u.dma.length = pt->src_len;
215558ea8abfSGary R Hook
2156a43eb985SGary R Hook ret = cmd_q->ccp->vdata->perform->passthru(&op);
215758ea8abfSGary R Hook if (ret)
215858ea8abfSGary R Hook cmd->engine_error = cmd_q->cmd_error;
215958ea8abfSGary R Hook
216058ea8abfSGary R Hook return ret;
216158ea8abfSGary R Hook }
216258ea8abfSGary R Hook
ccp_run_ecc_mm_cmd(struct ccp_cmd_queue * cmd_q,struct ccp_cmd * cmd)216363b94509STom Lendacky static int ccp_run_ecc_mm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
216463b94509STom Lendacky {
216563b94509STom Lendacky struct ccp_ecc_engine *ecc = &cmd->u.ecc;
216663b94509STom Lendacky struct ccp_dm_workarea src, dst;
216763b94509STom Lendacky struct ccp_op op;
216863b94509STom Lendacky int ret;
216963b94509STom Lendacky u8 *save;
217063b94509STom Lendacky
217163b94509STom Lendacky if (!ecc->u.mm.operand_1 ||
217263b94509STom Lendacky (ecc->u.mm.operand_1_len > CCP_ECC_MODULUS_BYTES))
217363b94509STom Lendacky return -EINVAL;
217463b94509STom Lendacky
217563b94509STom Lendacky if (ecc->function != CCP_ECC_FUNCTION_MINV_384BIT)
217663b94509STom Lendacky if (!ecc->u.mm.operand_2 ||
217763b94509STom Lendacky (ecc->u.mm.operand_2_len > CCP_ECC_MODULUS_BYTES))
217863b94509STom Lendacky return -EINVAL;
217963b94509STom Lendacky
218063b94509STom Lendacky if (!ecc->u.mm.result ||
218163b94509STom Lendacky (ecc->u.mm.result_len < CCP_ECC_MODULUS_BYTES))
218263b94509STom Lendacky return -EINVAL;
218363b94509STom Lendacky
218463b94509STom Lendacky memset(&op, 0, sizeof(op));
218563b94509STom Lendacky op.cmd_q = cmd_q;
21864b394a23SGary R Hook op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
218763b94509STom Lendacky
218863b94509STom Lendacky /* Concatenate the modulus and the operands. Both the modulus and
218963b94509STom Lendacky * the operands must be in little endian format. Since the input
219063b94509STom Lendacky * is in big endian format it must be converted and placed in a
219163b94509STom Lendacky * fixed length buffer.
219263b94509STom Lendacky */
219363b94509STom Lendacky ret = ccp_init_dm_workarea(&src, cmd_q, CCP_ECC_SRC_BUF_SIZE,
219463b94509STom Lendacky DMA_TO_DEVICE);
219563b94509STom Lendacky if (ret)
219663b94509STom Lendacky return ret;
219763b94509STom Lendacky
219863b94509STom Lendacky /* Save the workarea address since it is updated in order to perform
219963b94509STom Lendacky * the concatenation
220063b94509STom Lendacky */
220163b94509STom Lendacky save = src.address;
220263b94509STom Lendacky
220363b94509STom Lendacky /* Copy the ECC modulus */
220483d650abSGary R Hook ret = ccp_reverse_set_dm_area(&src, 0, ecc->mod, 0, ecc->mod_len);
2205355eba5dSTom Lendacky if (ret)
2206355eba5dSTom Lendacky goto e_src;
220763b94509STom Lendacky src.address += CCP_ECC_OPERAND_SIZE;
220863b94509STom Lendacky
220963b94509STom Lendacky /* Copy the first operand */
221083d650abSGary R Hook ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.mm.operand_1, 0,
221183d650abSGary R Hook ecc->u.mm.operand_1_len);
2212355eba5dSTom Lendacky if (ret)
2213355eba5dSTom Lendacky goto e_src;
221463b94509STom Lendacky src.address += CCP_ECC_OPERAND_SIZE;
221563b94509STom Lendacky
221663b94509STom Lendacky if (ecc->function != CCP_ECC_FUNCTION_MINV_384BIT) {
221763b94509STom Lendacky /* Copy the second operand */
221883d650abSGary R Hook ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.mm.operand_2, 0,
221983d650abSGary R Hook ecc->u.mm.operand_2_len);
2220355eba5dSTom Lendacky if (ret)
2221355eba5dSTom Lendacky goto e_src;
222263b94509STom Lendacky src.address += CCP_ECC_OPERAND_SIZE;
222363b94509STom Lendacky }
222463b94509STom Lendacky
222563b94509STom Lendacky /* Restore the workarea address */
222663b94509STom Lendacky src.address = save;
222763b94509STom Lendacky
222863b94509STom Lendacky /* Prepare the output area for the operation */
222963b94509STom Lendacky ret = ccp_init_dm_workarea(&dst, cmd_q, CCP_ECC_DST_BUF_SIZE,
223063b94509STom Lendacky DMA_FROM_DEVICE);
223163b94509STom Lendacky if (ret)
223263b94509STom Lendacky goto e_src;
223363b94509STom Lendacky
223463b94509STom Lendacky op.soc = 1;
223563b94509STom Lendacky op.src.u.dma.address = src.dma.address;
223663b94509STom Lendacky op.src.u.dma.offset = 0;
223763b94509STom Lendacky op.src.u.dma.length = src.length;
223863b94509STom Lendacky op.dst.u.dma.address = dst.dma.address;
223963b94509STom Lendacky op.dst.u.dma.offset = 0;
224063b94509STom Lendacky op.dst.u.dma.length = dst.length;
224163b94509STom Lendacky
224263b94509STom Lendacky op.u.ecc.function = cmd->u.ecc.function;
224363b94509STom Lendacky
2244a43eb985SGary R Hook ret = cmd_q->ccp->vdata->perform->ecc(&op);
224563b94509STom Lendacky if (ret) {
224663b94509STom Lendacky cmd->engine_error = cmd_q->cmd_error;
224763b94509STom Lendacky goto e_dst;
224863b94509STom Lendacky }
224963b94509STom Lendacky
225063b94509STom Lendacky ecc->ecc_result = le16_to_cpup(
225163b94509STom Lendacky (const __le16 *)(dst.address + CCP_ECC_RESULT_OFFSET));
225263b94509STom Lendacky if (!(ecc->ecc_result & CCP_ECC_RESULT_SUCCESS)) {
225363b94509STom Lendacky ret = -EIO;
225463b94509STom Lendacky goto e_dst;
225563b94509STom Lendacky }
225663b94509STom Lendacky
225763b94509STom Lendacky /* Save the ECC result */
225883d650abSGary R Hook ccp_reverse_get_dm_area(&dst, 0, ecc->u.mm.result, 0,
225983d650abSGary R Hook CCP_ECC_MODULUS_BYTES);
226063b94509STom Lendacky
226163b94509STom Lendacky e_dst:
226263b94509STom Lendacky ccp_dm_free(&dst);
226363b94509STom Lendacky
226463b94509STom Lendacky e_src:
226563b94509STom Lendacky ccp_dm_free(&src);
226663b94509STom Lendacky
226763b94509STom Lendacky return ret;
226863b94509STom Lendacky }
226963b94509STom Lendacky
ccp_run_ecc_pm_cmd(struct ccp_cmd_queue * cmd_q,struct ccp_cmd * cmd)227063b94509STom Lendacky static int ccp_run_ecc_pm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
227163b94509STom Lendacky {
227263b94509STom Lendacky struct ccp_ecc_engine *ecc = &cmd->u.ecc;
227363b94509STom Lendacky struct ccp_dm_workarea src, dst;
227463b94509STom Lendacky struct ccp_op op;
227563b94509STom Lendacky int ret;
227663b94509STom Lendacky u8 *save;
227763b94509STom Lendacky
227863b94509STom Lendacky if (!ecc->u.pm.point_1.x ||
227963b94509STom Lendacky (ecc->u.pm.point_1.x_len > CCP_ECC_MODULUS_BYTES) ||
228063b94509STom Lendacky !ecc->u.pm.point_1.y ||
228163b94509STom Lendacky (ecc->u.pm.point_1.y_len > CCP_ECC_MODULUS_BYTES))
228263b94509STom Lendacky return -EINVAL;
228363b94509STom Lendacky
228463b94509STom Lendacky if (ecc->function == CCP_ECC_FUNCTION_PADD_384BIT) {
228563b94509STom Lendacky if (!ecc->u.pm.point_2.x ||
228663b94509STom Lendacky (ecc->u.pm.point_2.x_len > CCP_ECC_MODULUS_BYTES) ||
228763b94509STom Lendacky !ecc->u.pm.point_2.y ||
228863b94509STom Lendacky (ecc->u.pm.point_2.y_len > CCP_ECC_MODULUS_BYTES))
228963b94509STom Lendacky return -EINVAL;
229063b94509STom Lendacky } else {
229163b94509STom Lendacky if (!ecc->u.pm.domain_a ||
229263b94509STom Lendacky (ecc->u.pm.domain_a_len > CCP_ECC_MODULUS_BYTES))
229363b94509STom Lendacky return -EINVAL;
229463b94509STom Lendacky
229563b94509STom Lendacky if (ecc->function == CCP_ECC_FUNCTION_PMUL_384BIT)
229663b94509STom Lendacky if (!ecc->u.pm.scalar ||
229763b94509STom Lendacky (ecc->u.pm.scalar_len > CCP_ECC_MODULUS_BYTES))
229863b94509STom Lendacky return -EINVAL;
229963b94509STom Lendacky }
230063b94509STom Lendacky
230163b94509STom Lendacky if (!ecc->u.pm.result.x ||
230263b94509STom Lendacky (ecc->u.pm.result.x_len < CCP_ECC_MODULUS_BYTES) ||
230363b94509STom Lendacky !ecc->u.pm.result.y ||
230463b94509STom Lendacky (ecc->u.pm.result.y_len < CCP_ECC_MODULUS_BYTES))
230563b94509STom Lendacky return -EINVAL;
230663b94509STom Lendacky
230763b94509STom Lendacky memset(&op, 0, sizeof(op));
230863b94509STom Lendacky op.cmd_q = cmd_q;
23094b394a23SGary R Hook op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
231063b94509STom Lendacky
231163b94509STom Lendacky /* Concatenate the modulus and the operands. Both the modulus and
231263b94509STom Lendacky * the operands must be in little endian format. Since the input
231363b94509STom Lendacky * is in big endian format it must be converted and placed in a
231463b94509STom Lendacky * fixed length buffer.
231563b94509STom Lendacky */
231663b94509STom Lendacky ret = ccp_init_dm_workarea(&src, cmd_q, CCP_ECC_SRC_BUF_SIZE,
231763b94509STom Lendacky DMA_TO_DEVICE);
231863b94509STom Lendacky if (ret)
231963b94509STom Lendacky return ret;
232063b94509STom Lendacky
232163b94509STom Lendacky /* Save the workarea address since it is updated in order to perform
232263b94509STom Lendacky * the concatenation
232363b94509STom Lendacky */
232463b94509STom Lendacky save = src.address;
232563b94509STom Lendacky
232663b94509STom Lendacky /* Copy the ECC modulus */
232783d650abSGary R Hook ret = ccp_reverse_set_dm_area(&src, 0, ecc->mod, 0, ecc->mod_len);
2328355eba5dSTom Lendacky if (ret)
2329355eba5dSTom Lendacky goto e_src;
233063b94509STom Lendacky src.address += CCP_ECC_OPERAND_SIZE;
233163b94509STom Lendacky
233263b94509STom Lendacky /* Copy the first point X and Y coordinate */
233383d650abSGary R Hook ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_1.x, 0,
233483d650abSGary R Hook ecc->u.pm.point_1.x_len);
2335355eba5dSTom Lendacky if (ret)
2336355eba5dSTom Lendacky goto e_src;
233763b94509STom Lendacky src.address += CCP_ECC_OPERAND_SIZE;
233883d650abSGary R Hook ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_1.y, 0,
233983d650abSGary R Hook ecc->u.pm.point_1.y_len);
2340355eba5dSTom Lendacky if (ret)
2341355eba5dSTom Lendacky goto e_src;
234263b94509STom Lendacky src.address += CCP_ECC_OPERAND_SIZE;
234363b94509STom Lendacky
23444b394a23SGary R Hook /* Set the first point Z coordinate to 1 */
23458db88467STom Lendacky *src.address = 0x01;
234663b94509STom Lendacky src.address += CCP_ECC_OPERAND_SIZE;
234763b94509STom Lendacky
234863b94509STom Lendacky if (ecc->function == CCP_ECC_FUNCTION_PADD_384BIT) {
234963b94509STom Lendacky /* Copy the second point X and Y coordinate */
235083d650abSGary R Hook ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_2.x, 0,
235183d650abSGary R Hook ecc->u.pm.point_2.x_len);
2352355eba5dSTom Lendacky if (ret)
2353355eba5dSTom Lendacky goto e_src;
235463b94509STom Lendacky src.address += CCP_ECC_OPERAND_SIZE;
235583d650abSGary R Hook ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_2.y, 0,
235683d650abSGary R Hook ecc->u.pm.point_2.y_len);
2357355eba5dSTom Lendacky if (ret)
2358355eba5dSTom Lendacky goto e_src;
235963b94509STom Lendacky src.address += CCP_ECC_OPERAND_SIZE;
236063b94509STom Lendacky
23614b394a23SGary R Hook /* Set the second point Z coordinate to 1 */
23628db88467STom Lendacky *src.address = 0x01;
236363b94509STom Lendacky src.address += CCP_ECC_OPERAND_SIZE;
236463b94509STom Lendacky } else {
236563b94509STom Lendacky /* Copy the Domain "a" parameter */
236683d650abSGary R Hook ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.domain_a, 0,
236783d650abSGary R Hook ecc->u.pm.domain_a_len);
2368355eba5dSTom Lendacky if (ret)
2369355eba5dSTom Lendacky goto e_src;
237063b94509STom Lendacky src.address += CCP_ECC_OPERAND_SIZE;
237163b94509STom Lendacky
237263b94509STom Lendacky if (ecc->function == CCP_ECC_FUNCTION_PMUL_384BIT) {
237363b94509STom Lendacky /* Copy the scalar value */
237483d650abSGary R Hook ret = ccp_reverse_set_dm_area(&src, 0,
237583d650abSGary R Hook ecc->u.pm.scalar, 0,
237683d650abSGary R Hook ecc->u.pm.scalar_len);
2377355eba5dSTom Lendacky if (ret)
2378355eba5dSTom Lendacky goto e_src;
237963b94509STom Lendacky src.address += CCP_ECC_OPERAND_SIZE;
238063b94509STom Lendacky }
238163b94509STom Lendacky }
238263b94509STom Lendacky
238363b94509STom Lendacky /* Restore the workarea address */
238463b94509STom Lendacky src.address = save;
238563b94509STom Lendacky
238663b94509STom Lendacky /* Prepare the output area for the operation */
238763b94509STom Lendacky ret = ccp_init_dm_workarea(&dst, cmd_q, CCP_ECC_DST_BUF_SIZE,
238863b94509STom Lendacky DMA_FROM_DEVICE);
238963b94509STom Lendacky if (ret)
239063b94509STom Lendacky goto e_src;
239163b94509STom Lendacky
239263b94509STom Lendacky op.soc = 1;
239363b94509STom Lendacky op.src.u.dma.address = src.dma.address;
239463b94509STom Lendacky op.src.u.dma.offset = 0;
239563b94509STom Lendacky op.src.u.dma.length = src.length;
239663b94509STom Lendacky op.dst.u.dma.address = dst.dma.address;
239763b94509STom Lendacky op.dst.u.dma.offset = 0;
239863b94509STom Lendacky op.dst.u.dma.length = dst.length;
239963b94509STom Lendacky
240063b94509STom Lendacky op.u.ecc.function = cmd->u.ecc.function;
240163b94509STom Lendacky
2402a43eb985SGary R Hook ret = cmd_q->ccp->vdata->perform->ecc(&op);
240363b94509STom Lendacky if (ret) {
240463b94509STom Lendacky cmd->engine_error = cmd_q->cmd_error;
240563b94509STom Lendacky goto e_dst;
240663b94509STom Lendacky }
240763b94509STom Lendacky
240863b94509STom Lendacky ecc->ecc_result = le16_to_cpup(
240963b94509STom Lendacky (const __le16 *)(dst.address + CCP_ECC_RESULT_OFFSET));
241063b94509STom Lendacky if (!(ecc->ecc_result & CCP_ECC_RESULT_SUCCESS)) {
241163b94509STom Lendacky ret = -EIO;
241263b94509STom Lendacky goto e_dst;
241363b94509STom Lendacky }
241463b94509STom Lendacky
241563b94509STom Lendacky /* Save the workarea address since it is updated as we walk through
241663b94509STom Lendacky * to copy the point math result
241763b94509STom Lendacky */
241863b94509STom Lendacky save = dst.address;
241963b94509STom Lendacky
242063b94509STom Lendacky /* Save the ECC result X and Y coordinates */
242183d650abSGary R Hook ccp_reverse_get_dm_area(&dst, 0, ecc->u.pm.result.x, 0,
242263b94509STom Lendacky CCP_ECC_MODULUS_BYTES);
242363b94509STom Lendacky dst.address += CCP_ECC_OUTPUT_SIZE;
242483d650abSGary R Hook ccp_reverse_get_dm_area(&dst, 0, ecc->u.pm.result.y, 0,
242563b94509STom Lendacky CCP_ECC_MODULUS_BYTES);
242663b94509STom Lendacky
242763b94509STom Lendacky /* Restore the workarea address */
242863b94509STom Lendacky dst.address = save;
242963b94509STom Lendacky
243063b94509STom Lendacky e_dst:
243163b94509STom Lendacky ccp_dm_free(&dst);
243263b94509STom Lendacky
243363b94509STom Lendacky e_src:
243463b94509STom Lendacky ccp_dm_free(&src);
243563b94509STom Lendacky
243663b94509STom Lendacky return ret;
243763b94509STom Lendacky }
243863b94509STom Lendacky
243972c8117aSArnd Bergmann static noinline_for_stack int
ccp_run_ecc_cmd(struct ccp_cmd_queue * cmd_q,struct ccp_cmd * cmd)244072c8117aSArnd Bergmann ccp_run_ecc_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
244163b94509STom Lendacky {
244263b94509STom Lendacky struct ccp_ecc_engine *ecc = &cmd->u.ecc;
244363b94509STom Lendacky
244463b94509STom Lendacky ecc->ecc_result = 0;
244563b94509STom Lendacky
244663b94509STom Lendacky if (!ecc->mod ||
244763b94509STom Lendacky (ecc->mod_len > CCP_ECC_MODULUS_BYTES))
244863b94509STom Lendacky return -EINVAL;
244963b94509STom Lendacky
245063b94509STom Lendacky switch (ecc->function) {
245163b94509STom Lendacky case CCP_ECC_FUNCTION_MMUL_384BIT:
245263b94509STom Lendacky case CCP_ECC_FUNCTION_MADD_384BIT:
245363b94509STom Lendacky case CCP_ECC_FUNCTION_MINV_384BIT:
245463b94509STom Lendacky return ccp_run_ecc_mm_cmd(cmd_q, cmd);
245563b94509STom Lendacky
245663b94509STom Lendacky case CCP_ECC_FUNCTION_PADD_384BIT:
245763b94509STom Lendacky case CCP_ECC_FUNCTION_PMUL_384BIT:
245863b94509STom Lendacky case CCP_ECC_FUNCTION_PDBL_384BIT:
245963b94509STom Lendacky return ccp_run_ecc_pm_cmd(cmd_q, cmd);
246063b94509STom Lendacky
246163b94509STom Lendacky default:
246263b94509STom Lendacky return -EINVAL;
246363b94509STom Lendacky }
246463b94509STom Lendacky }
246563b94509STom Lendacky
ccp_run_cmd(struct ccp_cmd_queue * cmd_q,struct ccp_cmd * cmd)246663b94509STom Lendacky int ccp_run_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
246763b94509STom Lendacky {
246863b94509STom Lendacky int ret;
246963b94509STom Lendacky
247063b94509STom Lendacky cmd->engine_error = 0;
247163b94509STom Lendacky cmd_q->cmd_error = 0;
247263b94509STom Lendacky cmd_q->int_rcvd = 0;
2473bb4e89b3SGary R Hook cmd_q->free_slots = cmd_q->ccp->vdata->perform->get_free_slots(cmd_q);
247463b94509STom Lendacky
247563b94509STom Lendacky switch (cmd->engine) {
247663b94509STom Lendacky case CCP_ENGINE_AES:
247772c8117aSArnd Bergmann switch (cmd->u.aes.mode) {
247872c8117aSArnd Bergmann case CCP_AES_MODE_CMAC:
247972c8117aSArnd Bergmann ret = ccp_run_aes_cmac_cmd(cmd_q, cmd);
248072c8117aSArnd Bergmann break;
248172c8117aSArnd Bergmann case CCP_AES_MODE_GCM:
248272c8117aSArnd Bergmann ret = ccp_run_aes_gcm_cmd(cmd_q, cmd);
248372c8117aSArnd Bergmann break;
248472c8117aSArnd Bergmann default:
248563b94509STom Lendacky ret = ccp_run_aes_cmd(cmd_q, cmd);
248663b94509STom Lendacky break;
248772c8117aSArnd Bergmann }
248872c8117aSArnd Bergmann break;
248963b94509STom Lendacky case CCP_ENGINE_XTS_AES_128:
249063b94509STom Lendacky ret = ccp_run_xts_aes_cmd(cmd_q, cmd);
249163b94509STom Lendacky break;
2492990672d4SGary R Hook case CCP_ENGINE_DES3:
2493990672d4SGary R Hook ret = ccp_run_des3_cmd(cmd_q, cmd);
2494990672d4SGary R Hook break;
249563b94509STom Lendacky case CCP_ENGINE_SHA:
249663b94509STom Lendacky ret = ccp_run_sha_cmd(cmd_q, cmd);
249763b94509STom Lendacky break;
249863b94509STom Lendacky case CCP_ENGINE_RSA:
249963b94509STom Lendacky ret = ccp_run_rsa_cmd(cmd_q, cmd);
250063b94509STom Lendacky break;
250163b94509STom Lendacky case CCP_ENGINE_PASSTHRU:
250258ea8abfSGary R Hook if (cmd->flags & CCP_CMD_PASSTHRU_NO_DMA_MAP)
250358ea8abfSGary R Hook ret = ccp_run_passthru_nomap_cmd(cmd_q, cmd);
250458ea8abfSGary R Hook else
250563b94509STom Lendacky ret = ccp_run_passthru_cmd(cmd_q, cmd);
250663b94509STom Lendacky break;
250763b94509STom Lendacky case CCP_ENGINE_ECC:
250863b94509STom Lendacky ret = ccp_run_ecc_cmd(cmd_q, cmd);
250963b94509STom Lendacky break;
251063b94509STom Lendacky default:
251163b94509STom Lendacky ret = -EINVAL;
251263b94509STom Lendacky }
251363b94509STom Lendacky
251463b94509STom Lendacky return ret;
251563b94509STom Lendacky }
2516