1d2912cb1SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only
2537559a5SDmitry Kasatkin /*
3537559a5SDmitry Kasatkin * Cryptographic API.
4537559a5SDmitry Kasatkin *
5537559a5SDmitry Kasatkin * Support for OMAP AES HW acceleration.
6537559a5SDmitry Kasatkin *
7537559a5SDmitry Kasatkin * Copyright (c) 2010 Nokia Corporation
8537559a5SDmitry Kasatkin * Author: Dmitry Kasatkin <dmitry.kasatkin@nokia.com>
90d35583aSMark A. Greer * Copyright (c) 2011 Texas Instruments Incorporated
10537559a5SDmitry Kasatkin */
11537559a5SDmitry Kasatkin
12016af9b5SJoel Fernandes #define pr_fmt(fmt) "%20s: " fmt, __func__
13016af9b5SJoel Fernandes #define prn(num) pr_debug(#num "=%d\n", num)
14016af9b5SJoel Fernandes #define prx(num) pr_debug(#num "=%x\n", num)
15537559a5SDmitry Kasatkin
16b7b23ccbSHerbert Xu #include <crypto/aes.h>
17b7b23ccbSHerbert Xu #include <crypto/gcm.h>
18b7b23ccbSHerbert Xu #include <crypto/internal/aead.h>
19b7b23ccbSHerbert Xu #include <crypto/internal/engine.h>
20b7b23ccbSHerbert Xu #include <crypto/internal/skcipher.h>
21b7b23ccbSHerbert Xu #include <crypto/scatterwalk.h>
22537559a5SDmitry Kasatkin #include <linux/dma-mapping.h>
23ebedbf79SMark A. Greer #include <linux/dmaengine.h>
24b7b23ccbSHerbert Xu #include <linux/err.h>
25b7b23ccbSHerbert Xu #include <linux/init.h>
26b7b23ccbSHerbert Xu #include <linux/interrupt.h>
27b7b23ccbSHerbert Xu #include <linux/io.h>
28b7b23ccbSHerbert Xu #include <linux/kernel.h>
29b7b23ccbSHerbert Xu #include <linux/module.h>
30bc69d124SMark A. Greer #include <linux/of.h>
31bc69d124SMark A. Greer #include <linux/of_address.h>
32b7b23ccbSHerbert Xu #include <linux/platform_device.h>
33b7b23ccbSHerbert Xu #include <linux/pm_runtime.h>
34b7b23ccbSHerbert Xu #include <linux/scatterlist.h>
35*03906fbaSHerbert Xu #include <linux/string.h>
36537559a5SDmitry Kasatkin
37afc2dc13STero Kristo #include "omap-crypto.h"
385b3d4d2eSTero Kristo #include "omap-aes.h"
39537559a5SDmitry Kasatkin
40537559a5SDmitry Kasatkin /* keep registered devices data here */
41537559a5SDmitry Kasatkin static LIST_HEAD(dev_list);
42537559a5SDmitry Kasatkin static DEFINE_SPINLOCK(list_lock);
43537559a5SDmitry Kasatkin
44537c62caSTero Kristo static int aes_fallback_sz = 200;
45537c62caSTero Kristo
46016af9b5SJoel Fernandes #ifdef DEBUG
47016af9b5SJoel Fernandes #define omap_aes_read(dd, offset) \
48016af9b5SJoel Fernandes ({ \
49016af9b5SJoel Fernandes int _read_ret; \
50016af9b5SJoel Fernandes _read_ret = __raw_readl(dd->io_base + offset); \
51016af9b5SJoel Fernandes pr_debug("omap_aes_read(" #offset "=%#x)= %#x\n", \
52016af9b5SJoel Fernandes offset, _read_ret); \
53016af9b5SJoel Fernandes _read_ret; \
54016af9b5SJoel Fernandes })
55016af9b5SJoel Fernandes #else
omap_aes_read(struct omap_aes_dev * dd,u32 offset)56d695bfd6STero Kristo inline u32 omap_aes_read(struct omap_aes_dev *dd, u32 offset)
57537559a5SDmitry Kasatkin {
58537559a5SDmitry Kasatkin return __raw_readl(dd->io_base + offset);
59537559a5SDmitry Kasatkin }
60016af9b5SJoel Fernandes #endif
61537559a5SDmitry Kasatkin
62016af9b5SJoel Fernandes #ifdef DEBUG
63016af9b5SJoel Fernandes #define omap_aes_write(dd, offset, value) \
64016af9b5SJoel Fernandes do { \
65016af9b5SJoel Fernandes pr_debug("omap_aes_write(" #offset "=%#x) value=%#x\n", \
66016af9b5SJoel Fernandes offset, value); \
67016af9b5SJoel Fernandes __raw_writel(value, dd->io_base + offset); \
68016af9b5SJoel Fernandes } while (0)
69016af9b5SJoel Fernandes #else
omap_aes_write(struct omap_aes_dev * dd,u32 offset,u32 value)70d695bfd6STero Kristo inline void omap_aes_write(struct omap_aes_dev *dd, u32 offset,
71537559a5SDmitry Kasatkin u32 value)
72537559a5SDmitry Kasatkin {
73537559a5SDmitry Kasatkin __raw_writel(value, dd->io_base + offset);
74537559a5SDmitry Kasatkin }
75016af9b5SJoel Fernandes #endif
76537559a5SDmitry Kasatkin
omap_aes_write_mask(struct omap_aes_dev * dd,u32 offset,u32 value,u32 mask)77537559a5SDmitry Kasatkin static inline void omap_aes_write_mask(struct omap_aes_dev *dd, u32 offset,
78537559a5SDmitry Kasatkin u32 value, u32 mask)
79537559a5SDmitry Kasatkin {
80537559a5SDmitry Kasatkin u32 val;
81537559a5SDmitry Kasatkin
82537559a5SDmitry Kasatkin val = omap_aes_read(dd, offset);
83537559a5SDmitry Kasatkin val &= ~mask;
84537559a5SDmitry Kasatkin val |= value;
85537559a5SDmitry Kasatkin omap_aes_write(dd, offset, val);
86537559a5SDmitry Kasatkin }
87537559a5SDmitry Kasatkin
omap_aes_write_n(struct omap_aes_dev * dd,u32 offset,u32 * value,int count)88537559a5SDmitry Kasatkin static void omap_aes_write_n(struct omap_aes_dev *dd, u32 offset,
89537559a5SDmitry Kasatkin u32 *value, int count)
90537559a5SDmitry Kasatkin {
91537559a5SDmitry Kasatkin for (; count--; value++, offset += 4)
92537559a5SDmitry Kasatkin omap_aes_write(dd, offset, *value);
93537559a5SDmitry Kasatkin }
94537559a5SDmitry Kasatkin
omap_aes_hw_init(struct omap_aes_dev * dd)95537559a5SDmitry Kasatkin static int omap_aes_hw_init(struct omap_aes_dev *dd)
96537559a5SDmitry Kasatkin {
97f303b455STero Kristo int err;
98f303b455STero Kristo
99537559a5SDmitry Kasatkin if (!(dd->flags & FLAGS_INIT)) {
100537559a5SDmitry Kasatkin dd->flags |= FLAGS_INIT;
10121fe9767SDmitry Kasatkin dd->err = 0;
102537559a5SDmitry Kasatkin }
103537559a5SDmitry Kasatkin
1041f34cc4aSShixin Liu err = pm_runtime_resume_and_get(dd->dev);
105f303b455STero Kristo if (err < 0) {
106f303b455STero Kristo dev_err(dd->dev, "failed to get sync: %d\n", err);
107f303b455STero Kristo return err;
108f303b455STero Kristo }
109f303b455STero Kristo
110eeb2b202SDmitry Kasatkin return 0;
111537559a5SDmitry Kasatkin }
112537559a5SDmitry Kasatkin
omap_aes_clear_copy_flags(struct omap_aes_dev * dd)113ad18cc9dSTero Kristo void omap_aes_clear_copy_flags(struct omap_aes_dev *dd)
114ad18cc9dSTero Kristo {
115ad18cc9dSTero Kristo dd->flags &= ~(OMAP_CRYPTO_COPY_MASK << FLAGS_IN_DATA_ST_SHIFT);
116ad18cc9dSTero Kristo dd->flags &= ~(OMAP_CRYPTO_COPY_MASK << FLAGS_OUT_DATA_ST_SHIFT);
117ad18cc9dSTero Kristo dd->flags &= ~(OMAP_CRYPTO_COPY_MASK << FLAGS_ASSOC_DATA_ST_SHIFT);
118ad18cc9dSTero Kristo }
119ad18cc9dSTero Kristo
omap_aes_write_ctrl(struct omap_aes_dev * dd)120d695bfd6STero Kristo int omap_aes_write_ctrl(struct omap_aes_dev *dd)
121537559a5SDmitry Kasatkin {
122ad18cc9dSTero Kristo struct omap_aes_reqctx *rctx;
123537559a5SDmitry Kasatkin unsigned int key32;
12467a730ceSDmitry Kasatkin int i, err;
1255396c6c0SVutla, Lokesh u32 val;
126537559a5SDmitry Kasatkin
12721fe9767SDmitry Kasatkin err = omap_aes_hw_init(dd);
12821fe9767SDmitry Kasatkin if (err)
12921fe9767SDmitry Kasatkin return err;
13021fe9767SDmitry Kasatkin
131537559a5SDmitry Kasatkin key32 = dd->ctx->keylen / sizeof(u32);
13267a730ceSDmitry Kasatkin
133ad18cc9dSTero Kristo /* RESET the key as previous HASH keys should not get affected*/
134ad18cc9dSTero Kristo if (dd->flags & FLAGS_GCM)
135ad18cc9dSTero Kristo for (i = 0; i < 0x40; i = i + 4)
136ad18cc9dSTero Kristo omap_aes_write(dd, i, 0x0);
137ad18cc9dSTero Kristo
138537559a5SDmitry Kasatkin for (i = 0; i < key32; i++) {
1390d35583aSMark A. Greer omap_aes_write(dd, AES_REG_KEY(dd, i),
140ac855b3cSHerbert Xu (__force u32)cpu_to_le32(dd->ctx->key[i]));
141537559a5SDmitry Kasatkin }
142537559a5SDmitry Kasatkin
143b3e3f0feSArd Biesheuvel if ((dd->flags & (FLAGS_CBC | FLAGS_CTR)) && dd->req->iv)
144b3e3f0feSArd Biesheuvel omap_aes_write_n(dd, AES_REG_IV(dd, 0), (void *)dd->req->iv, 4);
14567a730ceSDmitry Kasatkin
146ad18cc9dSTero Kristo if ((dd->flags & (FLAGS_GCM)) && dd->aead_req->iv) {
147ad18cc9dSTero Kristo rctx = aead_request_ctx(dd->aead_req);
148ad18cc9dSTero Kristo omap_aes_write_n(dd, AES_REG_IV(dd, 0), (u32 *)rctx->iv, 4);
149ad18cc9dSTero Kristo }
150ad18cc9dSTero Kristo
15167a730ceSDmitry Kasatkin val = FLD_VAL(((dd->ctx->keylen >> 3) - 1), 4, 3);
15267a730ceSDmitry Kasatkin if (dd->flags & FLAGS_CBC)
15367a730ceSDmitry Kasatkin val |= AES_REG_CTRL_CBC;
154ad18cc9dSTero Kristo
155ad18cc9dSTero Kristo if (dd->flags & (FLAGS_CTR | FLAGS_GCM))
1568ed49c76SJoel Fernandes val |= AES_REG_CTRL_CTR | AES_REG_CTRL_CTR_WIDTH_128;
1575396c6c0SVutla, Lokesh
158ad18cc9dSTero Kristo if (dd->flags & FLAGS_GCM)
159ad18cc9dSTero Kristo val |= AES_REG_CTRL_GCM;
160ad18cc9dSTero Kristo
16167a730ceSDmitry Kasatkin if (dd->flags & FLAGS_ENCRYPT)
16267a730ceSDmitry Kasatkin val |= AES_REG_CTRL_DIRECTION;
163537559a5SDmitry Kasatkin
1645396c6c0SVutla, Lokesh omap_aes_write_mask(dd, AES_REG_CTRL(dd), val, AES_REG_CTRL_MASK);
165537559a5SDmitry Kasatkin
16621fe9767SDmitry Kasatkin return 0;
167537559a5SDmitry Kasatkin }
168537559a5SDmitry Kasatkin
omap_aes_dma_trigger_omap2(struct omap_aes_dev * dd,int length)1690d35583aSMark A. Greer static void omap_aes_dma_trigger_omap2(struct omap_aes_dev *dd, int length)
1700d35583aSMark A. Greer {
1710d35583aSMark A. Greer u32 mask, val;
1720d35583aSMark A. Greer
1730d35583aSMark A. Greer val = dd->pdata->dma_start;
1740d35583aSMark A. Greer
1750d35583aSMark A. Greer if (dd->dma_lch_out != NULL)
1760d35583aSMark A. Greer val |= dd->pdata->dma_enable_out;
1770d35583aSMark A. Greer if (dd->dma_lch_in != NULL)
1780d35583aSMark A. Greer val |= dd->pdata->dma_enable_in;
1790d35583aSMark A. Greer
1800d35583aSMark A. Greer mask = dd->pdata->dma_enable_out | dd->pdata->dma_enable_in |
1810d35583aSMark A. Greer dd->pdata->dma_start;
1820d35583aSMark A. Greer
1830d35583aSMark A. Greer omap_aes_write_mask(dd, AES_REG_MASK(dd), val, mask);
1840d35583aSMark A. Greer
1850d35583aSMark A. Greer }
1860d35583aSMark A. Greer
omap_aes_dma_trigger_omap4(struct omap_aes_dev * dd,int length)1870d35583aSMark A. Greer static void omap_aes_dma_trigger_omap4(struct omap_aes_dev *dd, int length)
1880d35583aSMark A. Greer {
1890d35583aSMark A. Greer omap_aes_write(dd, AES_REG_LENGTH_N(0), length);
1900d35583aSMark A. Greer omap_aes_write(dd, AES_REG_LENGTH_N(1), 0);
191ad18cc9dSTero Kristo if (dd->flags & FLAGS_GCM)
192ad18cc9dSTero Kristo omap_aes_write(dd, AES_REG_A_LEN, dd->assoc_len);
1930d35583aSMark A. Greer
1940d35583aSMark A. Greer omap_aes_dma_trigger_omap2(dd, length);
1950d35583aSMark A. Greer }
1960d35583aSMark A. Greer
omap_aes_dma_stop(struct omap_aes_dev * dd)1970d35583aSMark A. Greer static void omap_aes_dma_stop(struct omap_aes_dev *dd)
1980d35583aSMark A. Greer {
1990d35583aSMark A. Greer u32 mask;
2000d35583aSMark A. Greer
2010d35583aSMark A. Greer mask = dd->pdata->dma_enable_out | dd->pdata->dma_enable_in |
2020d35583aSMark A. Greer dd->pdata->dma_start;
2030d35583aSMark A. Greer
2040d35583aSMark A. Greer omap_aes_write_mask(dd, AES_REG_MASK(dd), 0, mask);
2050d35583aSMark A. Greer }
2060d35583aSMark A. Greer
omap_aes_find_dev(struct omap_aes_reqctx * rctx)207d695bfd6STero Kristo struct omap_aes_dev *omap_aes_find_dev(struct omap_aes_reqctx *rctx)
208537559a5SDmitry Kasatkin {
209164f3ef3SLokesh Vutla struct omap_aes_dev *dd;
210537559a5SDmitry Kasatkin
211537559a5SDmitry Kasatkin spin_lock_bh(&list_lock);
212164f3ef3SLokesh Vutla dd = list_first_entry(&dev_list, struct omap_aes_dev, list);
213164f3ef3SLokesh Vutla list_move_tail(&dd->list, &dev_list);
214619ce700STero Kristo rctx->dd = dd;
215537559a5SDmitry Kasatkin spin_unlock_bh(&list_lock);
216537559a5SDmitry Kasatkin
217537559a5SDmitry Kasatkin return dd;
218537559a5SDmitry Kasatkin }
219537559a5SDmitry Kasatkin
omap_aes_dma_out_callback(void * data)220ebedbf79SMark A. Greer static void omap_aes_dma_out_callback(void *data)
221ebedbf79SMark A. Greer {
222ebedbf79SMark A. Greer struct omap_aes_dev *dd = data;
223ebedbf79SMark A. Greer
224ebedbf79SMark A. Greer /* dma_lch_out - completed */
225ebedbf79SMark A. Greer tasklet_schedule(&dd->done_task);
226ebedbf79SMark A. Greer }
227537559a5SDmitry Kasatkin
omap_aes_dma_init(struct omap_aes_dev * dd)228537559a5SDmitry Kasatkin static int omap_aes_dma_init(struct omap_aes_dev *dd)
229537559a5SDmitry Kasatkin {
230da8b29a6SPeter Ujfalusi int err;
231537559a5SDmitry Kasatkin
232ebedbf79SMark A. Greer dd->dma_lch_out = NULL;
233ebedbf79SMark A. Greer dd->dma_lch_in = NULL;
234537559a5SDmitry Kasatkin
235da8b29a6SPeter Ujfalusi dd->dma_lch_in = dma_request_chan(dd->dev, "rx");
236da8b29a6SPeter Ujfalusi if (IS_ERR(dd->dma_lch_in)) {
237ebedbf79SMark A. Greer dev_err(dd->dev, "Unable to request in DMA channel\n");
238da8b29a6SPeter Ujfalusi return PTR_ERR(dd->dma_lch_in);
239ebedbf79SMark A. Greer }
240ebedbf79SMark A. Greer
241da8b29a6SPeter Ujfalusi dd->dma_lch_out = dma_request_chan(dd->dev, "tx");
242da8b29a6SPeter Ujfalusi if (IS_ERR(dd->dma_lch_out)) {
243ebedbf79SMark A. Greer dev_err(dd->dev, "Unable to request out DMA channel\n");
244da8b29a6SPeter Ujfalusi err = PTR_ERR(dd->dma_lch_out);
245ebedbf79SMark A. Greer goto err_dma_out;
246ebedbf79SMark A. Greer }
247537559a5SDmitry Kasatkin
248537559a5SDmitry Kasatkin return 0;
249537559a5SDmitry Kasatkin
250537559a5SDmitry Kasatkin err_dma_out:
251ebedbf79SMark A. Greer dma_release_channel(dd->dma_lch_in);
252da8b29a6SPeter Ujfalusi
253537559a5SDmitry Kasatkin return err;
254537559a5SDmitry Kasatkin }
255537559a5SDmitry Kasatkin
omap_aes_dma_cleanup(struct omap_aes_dev * dd)256537559a5SDmitry Kasatkin static void omap_aes_dma_cleanup(struct omap_aes_dev *dd)
257537559a5SDmitry Kasatkin {
258da8b29a6SPeter Ujfalusi if (dd->pio_only)
259da8b29a6SPeter Ujfalusi return;
260da8b29a6SPeter Ujfalusi
261ebedbf79SMark A. Greer dma_release_channel(dd->dma_lch_out);
262ebedbf79SMark A. Greer dma_release_channel(dd->dma_lch_in);
263537559a5SDmitry Kasatkin }
264537559a5SDmitry Kasatkin
omap_aes_crypt_dma(struct omap_aes_dev * dd,struct scatterlist * in_sg,struct scatterlist * out_sg,int in_sg_len,int out_sg_len)265619ce700STero Kristo static int omap_aes_crypt_dma(struct omap_aes_dev *dd,
266619ce700STero Kristo struct scatterlist *in_sg,
267619ce700STero Kristo struct scatterlist *out_sg,
2684b645c94SJoel Fernandes int in_sg_len, int out_sg_len)
269537559a5SDmitry Kasatkin {
2705d5f3eedSTero Kristo struct dma_async_tx_descriptor *tx_in, *tx_out = NULL, *cb_desc;
271ebedbf79SMark A. Greer struct dma_slave_config cfg;
2724b645c94SJoel Fernandes int ret;
273537559a5SDmitry Kasatkin
27498837abcSJoel Fernandes if (dd->pio_only) {
27598837abcSJoel Fernandes scatterwalk_start(&dd->in_walk, dd->in_sg);
2765d5f3eedSTero Kristo if (out_sg_len)
27798837abcSJoel Fernandes scatterwalk_start(&dd->out_walk, dd->out_sg);
27898837abcSJoel Fernandes
27998837abcSJoel Fernandes /* Enable DATAIN interrupt and let it take
28098837abcSJoel Fernandes care of the rest */
28198837abcSJoel Fernandes omap_aes_write(dd, AES_REG_IRQ_ENABLE(dd), 0x2);
28298837abcSJoel Fernandes return 0;
28398837abcSJoel Fernandes }
28498837abcSJoel Fernandes
2850a641712SJoel Fernandes dma_sync_sg_for_device(dd->dev, dd->in_sg, in_sg_len, DMA_TO_DEVICE);
2860a641712SJoel Fernandes
287ebedbf79SMark A. Greer memset(&cfg, 0, sizeof(cfg));
288ebedbf79SMark A. Greer
2890d35583aSMark A. Greer cfg.src_addr = dd->phys_base + AES_REG_DATA_N(dd, 0);
2900d35583aSMark A. Greer cfg.dst_addr = dd->phys_base + AES_REG_DATA_N(dd, 0);
291ebedbf79SMark A. Greer cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
292ebedbf79SMark A. Greer cfg.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
293ebedbf79SMark A. Greer cfg.src_maxburst = DST_MAXBURST;
294ebedbf79SMark A. Greer cfg.dst_maxburst = DST_MAXBURST;
295ebedbf79SMark A. Greer
296ebedbf79SMark A. Greer /* IN */
297ebedbf79SMark A. Greer ret = dmaengine_slave_config(dd->dma_lch_in, &cfg);
298ebedbf79SMark A. Greer if (ret) {
299ebedbf79SMark A. Greer dev_err(dd->dev, "can't configure IN dmaengine slave: %d\n",
300ebedbf79SMark A. Greer ret);
301ebedbf79SMark A. Greer return ret;
302ebedbf79SMark A. Greer }
303ebedbf79SMark A. Greer
3044b645c94SJoel Fernandes tx_in = dmaengine_prep_slave_sg(dd->dma_lch_in, in_sg, in_sg_len,
305ebedbf79SMark A. Greer DMA_MEM_TO_DEV,
306ebedbf79SMark A. Greer DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
307ebedbf79SMark A. Greer if (!tx_in) {
308ebedbf79SMark A. Greer dev_err(dd->dev, "IN prep_slave_sg() failed\n");
309ebedbf79SMark A. Greer return -EINVAL;
310ebedbf79SMark A. Greer }
311ebedbf79SMark A. Greer
312ebedbf79SMark A. Greer /* No callback necessary */
313ebedbf79SMark A. Greer tx_in->callback_param = dd;
3145d5f3eedSTero Kristo tx_in->callback = NULL;
315ebedbf79SMark A. Greer
316ebedbf79SMark A. Greer /* OUT */
3175d5f3eedSTero Kristo if (out_sg_len) {
318ebedbf79SMark A. Greer ret = dmaengine_slave_config(dd->dma_lch_out, &cfg);
319ebedbf79SMark A. Greer if (ret) {
320ebedbf79SMark A. Greer dev_err(dd->dev, "can't configure OUT dmaengine slave: %d\n",
321ebedbf79SMark A. Greer ret);
322ebedbf79SMark A. Greer return ret;
323ebedbf79SMark A. Greer }
324ebedbf79SMark A. Greer
3255d5f3eedSTero Kristo tx_out = dmaengine_prep_slave_sg(dd->dma_lch_out, out_sg,
3265d5f3eedSTero Kristo out_sg_len,
327ebedbf79SMark A. Greer DMA_DEV_TO_MEM,
328ebedbf79SMark A. Greer DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
329ebedbf79SMark A. Greer if (!tx_out) {
330ebedbf79SMark A. Greer dev_err(dd->dev, "OUT prep_slave_sg() failed\n");
331ebedbf79SMark A. Greer return -EINVAL;
332ebedbf79SMark A. Greer }
333ebedbf79SMark A. Greer
3345d5f3eedSTero Kristo cb_desc = tx_out;
3355d5f3eedSTero Kristo } else {
3365d5f3eedSTero Kristo cb_desc = tx_in;
3375d5f3eedSTero Kristo }
3385d5f3eedSTero Kristo
339ad18cc9dSTero Kristo if (dd->flags & FLAGS_GCM)
3405d5f3eedSTero Kristo cb_desc->callback = omap_aes_gcm_dma_out_callback;
341ad18cc9dSTero Kristo else
3425d5f3eedSTero Kristo cb_desc->callback = omap_aes_dma_out_callback;
3435d5f3eedSTero Kristo cb_desc->callback_param = dd;
3445d5f3eedSTero Kristo
345ebedbf79SMark A. Greer
346ebedbf79SMark A. Greer dmaengine_submit(tx_in);
3475d5f3eedSTero Kristo if (tx_out)
348ebedbf79SMark A. Greer dmaengine_submit(tx_out);
349ebedbf79SMark A. Greer
350ebedbf79SMark A. Greer dma_async_issue_pending(dd->dma_lch_in);
3515d5f3eedSTero Kristo if (out_sg_len)
352ebedbf79SMark A. Greer dma_async_issue_pending(dd->dma_lch_out);
353537559a5SDmitry Kasatkin
3540d35583aSMark A. Greer /* start DMA */
3554b645c94SJoel Fernandes dd->pdata->trigger(dd, dd->total);
35683ea7e0fSDmitry Kasatkin
357537559a5SDmitry Kasatkin return 0;
358537559a5SDmitry Kasatkin }
359537559a5SDmitry Kasatkin
omap_aes_crypt_dma_start(struct omap_aes_dev * dd)360d695bfd6STero Kristo int omap_aes_crypt_dma_start(struct omap_aes_dev *dd)
361537559a5SDmitry Kasatkin {
3624b645c94SJoel Fernandes int err;
363537559a5SDmitry Kasatkin
364ac855b3cSHerbert Xu pr_debug("total: %zu\n", dd->total);
365537559a5SDmitry Kasatkin
36698837abcSJoel Fernandes if (!dd->pio_only) {
36798837abcSJoel Fernandes err = dma_map_sg(dd->dev, dd->in_sg, dd->in_sg_len,
36898837abcSJoel Fernandes DMA_TO_DEVICE);
369537559a5SDmitry Kasatkin if (!err) {
370537559a5SDmitry Kasatkin dev_err(dd->dev, "dma_map_sg() error\n");
371537559a5SDmitry Kasatkin return -EINVAL;
372537559a5SDmitry Kasatkin }
373537559a5SDmitry Kasatkin
3745d5f3eedSTero Kristo if (dd->out_sg_len) {
37598837abcSJoel Fernandes err = dma_map_sg(dd->dev, dd->out_sg, dd->out_sg_len,
37698837abcSJoel Fernandes DMA_FROM_DEVICE);
377537559a5SDmitry Kasatkin if (!err) {
378537559a5SDmitry Kasatkin dev_err(dd->dev, "dma_map_sg() error\n");
379537559a5SDmitry Kasatkin return -EINVAL;
380537559a5SDmitry Kasatkin }
38198837abcSJoel Fernandes }
3825d5f3eedSTero Kristo }
383537559a5SDmitry Kasatkin
384619ce700STero Kristo err = omap_aes_crypt_dma(dd, dd->in_sg, dd->out_sg, dd->in_sg_len,
3854b645c94SJoel Fernandes dd->out_sg_len);
38698837abcSJoel Fernandes if (err && !dd->pio_only) {
3874b645c94SJoel Fernandes dma_unmap_sg(dd->dev, dd->in_sg, dd->in_sg_len, DMA_TO_DEVICE);
3885d5f3eedSTero Kristo if (dd->out_sg_len)
3894b645c94SJoel Fernandes dma_unmap_sg(dd->dev, dd->out_sg, dd->out_sg_len,
3904b645c94SJoel Fernandes DMA_FROM_DEVICE);
39121fe9767SDmitry Kasatkin }
392537559a5SDmitry Kasatkin
393537559a5SDmitry Kasatkin return err;
394537559a5SDmitry Kasatkin }
395537559a5SDmitry Kasatkin
omap_aes_finish_req(struct omap_aes_dev * dd,int err)396537559a5SDmitry Kasatkin static void omap_aes_finish_req(struct omap_aes_dev *dd, int err)
397537559a5SDmitry Kasatkin {
398b3e3f0feSArd Biesheuvel struct skcipher_request *req = dd->req;
399537559a5SDmitry Kasatkin
400537559a5SDmitry Kasatkin pr_debug("err: %d\n", err);
401537559a5SDmitry Kasatkin
402b3e3f0feSArd Biesheuvel crypto_finalize_skcipher_request(dd->engine, req, err);
403f303b455STero Kristo
404f303b455STero Kristo pm_runtime_mark_last_busy(dd->dev);
405f303b455STero Kristo pm_runtime_put_autosuspend(dd->dev);
406537559a5SDmitry Kasatkin }
407537559a5SDmitry Kasatkin
omap_aes_crypt_dma_stop(struct omap_aes_dev * dd)408d695bfd6STero Kristo int omap_aes_crypt_dma_stop(struct omap_aes_dev *dd)
409537559a5SDmitry Kasatkin {
410ac855b3cSHerbert Xu pr_debug("total: %zu\n", dd->total);
411537559a5SDmitry Kasatkin
4120d35583aSMark A. Greer omap_aes_dma_stop(dd);
413537559a5SDmitry Kasatkin
414537559a5SDmitry Kasatkin
41516f080aaSRahul Pathak return 0;
416537559a5SDmitry Kasatkin }
417537559a5SDmitry Kasatkin
omap_aes_handle_queue(struct omap_aes_dev * dd,struct skcipher_request * req)41821fe9767SDmitry Kasatkin static int omap_aes_handle_queue(struct omap_aes_dev *dd,
419b3e3f0feSArd Biesheuvel struct skcipher_request *req)
420537559a5SDmitry Kasatkin {
421eeb2b202SDmitry Kasatkin if (req)
422b3e3f0feSArd Biesheuvel return crypto_transfer_skcipher_request_to_engine(dd->engine, req);
4230529900aSBaolin Wang
4240529900aSBaolin Wang return 0;
425eeb2b202SDmitry Kasatkin }
426537559a5SDmitry Kasatkin
omap_aes_prepare_req(struct skcipher_request * req,struct omap_aes_dev * dd)427c752c013SHerbert Xu static int omap_aes_prepare_req(struct skcipher_request *req,
428c752c013SHerbert Xu struct omap_aes_dev *dd)
4290529900aSBaolin Wang {
430b3e3f0feSArd Biesheuvel struct omap_aes_ctx *ctx = crypto_skcipher_ctx(
431b3e3f0feSArd Biesheuvel crypto_skcipher_reqtfm(req));
432b3e3f0feSArd Biesheuvel struct omap_aes_reqctx *rctx = skcipher_request_ctx(req);
433afc2dc13STero Kristo int ret;
434afc2dc13STero Kristo u16 flags;
435537559a5SDmitry Kasatkin
436537559a5SDmitry Kasatkin /* assign new request to device */
437537559a5SDmitry Kasatkin dd->req = req;
438b3e3f0feSArd Biesheuvel dd->total = req->cryptlen;
439b3e3f0feSArd Biesheuvel dd->total_save = req->cryptlen;
440537559a5SDmitry Kasatkin dd->in_sg = req->src;
441537559a5SDmitry Kasatkin dd->out_sg = req->dst;
442afc2dc13STero Kristo dd->orig_out = req->dst;
443afc2dc13STero Kristo
444afc2dc13STero Kristo flags = OMAP_CRYPTO_COPY_DATA;
445afc2dc13STero Kristo if (req->src == req->dst)
446afc2dc13STero Kristo flags |= OMAP_CRYPTO_FORCE_COPY;
447afc2dc13STero Kristo
448afc2dc13STero Kristo ret = omap_crypto_align_sg(&dd->in_sg, dd->total, AES_BLOCK_SIZE,
449ad18cc9dSTero Kristo dd->in_sgl, flags,
450afc2dc13STero Kristo FLAGS_IN_DATA_ST_SHIFT, &dd->flags);
451afc2dc13STero Kristo if (ret)
452afc2dc13STero Kristo return ret;
453afc2dc13STero Kristo
454afc2dc13STero Kristo ret = omap_crypto_align_sg(&dd->out_sg, dd->total, AES_BLOCK_SIZE,
455afc2dc13STero Kristo &dd->out_sgl, 0,
456afc2dc13STero Kristo FLAGS_OUT_DATA_ST_SHIFT, &dd->flags);
457afc2dc13STero Kristo if (ret)
458afc2dc13STero Kristo return ret;
459537559a5SDmitry Kasatkin
4607c001a86SHerbert Xu dd->in_sg_len = sg_nents_for_len(dd->in_sg, dd->total);
4617c001a86SHerbert Xu if (dd->in_sg_len < 0)
4627c001a86SHerbert Xu return dd->in_sg_len;
4637c001a86SHerbert Xu
4647c001a86SHerbert Xu dd->out_sg_len = sg_nents_for_len(dd->out_sg, dd->total);
4657c001a86SHerbert Xu if (dd->out_sg_len < 0)
4667c001a86SHerbert Xu return dd->out_sg_len;
4677c001a86SHerbert Xu
468537559a5SDmitry Kasatkin rctx->mode &= FLAGS_MODE_MASK;
469537559a5SDmitry Kasatkin dd->flags = (dd->flags & ~FLAGS_MODE_MASK) | rctx->mode;
470537559a5SDmitry Kasatkin
471537559a5SDmitry Kasatkin dd->ctx = ctx;
472619ce700STero Kristo rctx->dd = dd;
473537559a5SDmitry Kasatkin
4740529900aSBaolin Wang return omap_aes_write_ctrl(dd);
475537559a5SDmitry Kasatkin }
476537559a5SDmitry Kasatkin
omap_aes_crypt_req(struct crypto_engine * engine,void * areq)4770529900aSBaolin Wang static int omap_aes_crypt_req(struct crypto_engine *engine,
478c21c8b89SCorentin LABBE void *areq)
4790529900aSBaolin Wang {
480b3e3f0feSArd Biesheuvel struct skcipher_request *req = container_of(areq, struct skcipher_request, base);
481b3e3f0feSArd Biesheuvel struct omap_aes_reqctx *rctx = skcipher_request_ctx(req);
482619ce700STero Kristo struct omap_aes_dev *dd = rctx->dd;
4830529900aSBaolin Wang
4840529900aSBaolin Wang if (!dd)
4850529900aSBaolin Wang return -ENODEV;
4860529900aSBaolin Wang
487c752c013SHerbert Xu return omap_aes_prepare_req(req, dd) ?:
488c752c013SHerbert Xu omap_aes_crypt_dma_start(dd);
48921fe9767SDmitry Kasatkin }
49021fe9767SDmitry Kasatkin
omap_aes_copy_ivout(struct omap_aes_dev * dd,u8 * ivbuf)491891dcbbbSTero Kristo static void omap_aes_copy_ivout(struct omap_aes_dev *dd, u8 *ivbuf)
492891dcbbbSTero Kristo {
493891dcbbbSTero Kristo int i;
494891dcbbbSTero Kristo
495891dcbbbSTero Kristo for (i = 0; i < 4; i++)
496891dcbbbSTero Kristo ((u32 *)ivbuf)[i] = omap_aes_read(dd, AES_REG_IV(dd, i));
497891dcbbbSTero Kristo }
498891dcbbbSTero Kristo
omap_aes_done_task(unsigned long data)49921fe9767SDmitry Kasatkin static void omap_aes_done_task(unsigned long data)
500537559a5SDmitry Kasatkin {
501537559a5SDmitry Kasatkin struct omap_aes_dev *dd = (struct omap_aes_dev *)data;
502537559a5SDmitry Kasatkin
5034b645c94SJoel Fernandes pr_debug("enter done_task\n");
504537559a5SDmitry Kasatkin
50598837abcSJoel Fernandes if (!dd->pio_only) {
50698837abcSJoel Fernandes dma_sync_sg_for_device(dd->dev, dd->out_sg, dd->out_sg_len,
50798837abcSJoel Fernandes DMA_FROM_DEVICE);
5086242332fSJoel Fernandes dma_unmap_sg(dd->dev, dd->in_sg, dd->in_sg_len, DMA_TO_DEVICE);
5096242332fSJoel Fernandes dma_unmap_sg(dd->dev, dd->out_sg, dd->out_sg_len,
5106242332fSJoel Fernandes DMA_FROM_DEVICE);
5114b645c94SJoel Fernandes omap_aes_crypt_dma_stop(dd);
51298837abcSJoel Fernandes }
5136242332fSJoel Fernandes
5146585cd36STero Kristo omap_crypto_cleanup(dd->in_sg, NULL, 0, dd->total_save,
515afc2dc13STero Kristo FLAGS_IN_DATA_ST_SHIFT, dd->flags);
5166242332fSJoel Fernandes
5176585cd36STero Kristo omap_crypto_cleanup(dd->out_sg, dd->orig_out, 0, dd->total_save,
518afc2dc13STero Kristo FLAGS_OUT_DATA_ST_SHIFT, dd->flags);
5196242332fSJoel Fernandes
520891dcbbbSTero Kristo /* Update IV output */
521891dcbbbSTero Kristo if (dd->flags & (FLAGS_CBC | FLAGS_CTR))
522891dcbbbSTero Kristo omap_aes_copy_ivout(dd, dd->req->iv);
523891dcbbbSTero Kristo
5244b645c94SJoel Fernandes omap_aes_finish_req(dd, 0);
525537559a5SDmitry Kasatkin
526537559a5SDmitry Kasatkin pr_debug("exit\n");
527537559a5SDmitry Kasatkin }
528537559a5SDmitry Kasatkin
omap_aes_crypt(struct skcipher_request * req,unsigned long mode)529b3e3f0feSArd Biesheuvel static int omap_aes_crypt(struct skcipher_request *req, unsigned long mode)
530537559a5SDmitry Kasatkin {
531b3e3f0feSArd Biesheuvel struct omap_aes_ctx *ctx = crypto_skcipher_ctx(
532b3e3f0feSArd Biesheuvel crypto_skcipher_reqtfm(req));
533b3e3f0feSArd Biesheuvel struct omap_aes_reqctx *rctx = skcipher_request_ctx(req);
534537559a5SDmitry Kasatkin struct omap_aes_dev *dd;
5359fcb191aSLokesh Vutla int ret;
536537559a5SDmitry Kasatkin
537dbb326fdSArd Biesheuvel if ((req->cryptlen % AES_BLOCK_SIZE) && !(mode & FLAGS_CTR))
538dbb326fdSArd Biesheuvel return -EINVAL;
539dbb326fdSArd Biesheuvel
540b3e3f0feSArd Biesheuvel pr_debug("nbytes: %d, enc: %d, cbc: %d\n", req->cryptlen,
541537559a5SDmitry Kasatkin !!(mode & FLAGS_ENCRYPT),
542537559a5SDmitry Kasatkin !!(mode & FLAGS_CBC));
543537559a5SDmitry Kasatkin
544b3e3f0feSArd Biesheuvel if (req->cryptlen < aes_fallback_sz) {
5456a99d7a2SArd Biesheuvel skcipher_request_set_tfm(&rctx->fallback_req, ctx->fallback);
5466a99d7a2SArd Biesheuvel skcipher_request_set_callback(&rctx->fallback_req,
5476a99d7a2SArd Biesheuvel req->base.flags,
5486a99d7a2SArd Biesheuvel req->base.complete,
5496a99d7a2SArd Biesheuvel req->base.data);
5506a99d7a2SArd Biesheuvel skcipher_request_set_crypt(&rctx->fallback_req, req->src,
5516a99d7a2SArd Biesheuvel req->dst, req->cryptlen, req->iv);
5529fcb191aSLokesh Vutla
5539fcb191aSLokesh Vutla if (mode & FLAGS_ENCRYPT)
5546a99d7a2SArd Biesheuvel ret = crypto_skcipher_encrypt(&rctx->fallback_req);
5559fcb191aSLokesh Vutla else
5566a99d7a2SArd Biesheuvel ret = crypto_skcipher_decrypt(&rctx->fallback_req);
5579fcb191aSLokesh Vutla return ret;
5589fcb191aSLokesh Vutla }
559619ce700STero Kristo dd = omap_aes_find_dev(rctx);
560537559a5SDmitry Kasatkin if (!dd)
561537559a5SDmitry Kasatkin return -ENODEV;
562537559a5SDmitry Kasatkin
563537559a5SDmitry Kasatkin rctx->mode = mode;
564537559a5SDmitry Kasatkin
56521fe9767SDmitry Kasatkin return omap_aes_handle_queue(dd, req);
566537559a5SDmitry Kasatkin }
567537559a5SDmitry Kasatkin
568537559a5SDmitry Kasatkin /* ********************** ALG API ************************************ */
569537559a5SDmitry Kasatkin
omap_aes_setkey(struct crypto_skcipher * tfm,const u8 * key,unsigned int keylen)570b3e3f0feSArd Biesheuvel static int omap_aes_setkey(struct crypto_skcipher *tfm, const u8 *key,
571537559a5SDmitry Kasatkin unsigned int keylen)
572537559a5SDmitry Kasatkin {
573b3e3f0feSArd Biesheuvel struct omap_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
5749fcb191aSLokesh Vutla int ret;
575537559a5SDmitry Kasatkin
576537559a5SDmitry Kasatkin if (keylen != AES_KEYSIZE_128 && keylen != AES_KEYSIZE_192 &&
577537559a5SDmitry Kasatkin keylen != AES_KEYSIZE_256)
578537559a5SDmitry Kasatkin return -EINVAL;
579537559a5SDmitry Kasatkin
580537559a5SDmitry Kasatkin pr_debug("enter, keylen: %d\n", keylen);
581537559a5SDmitry Kasatkin
582537559a5SDmitry Kasatkin memcpy(ctx->key, key, keylen);
583537559a5SDmitry Kasatkin ctx->keylen = keylen;
584537559a5SDmitry Kasatkin
5856a99d7a2SArd Biesheuvel crypto_skcipher_clear_flags(ctx->fallback, CRYPTO_TFM_REQ_MASK);
5866a99d7a2SArd Biesheuvel crypto_skcipher_set_flags(ctx->fallback, tfm->base.crt_flags &
5879fcb191aSLokesh Vutla CRYPTO_TFM_REQ_MASK);
5889fcb191aSLokesh Vutla
5896a99d7a2SArd Biesheuvel ret = crypto_skcipher_setkey(ctx->fallback, key, keylen);
5909fcb191aSLokesh Vutla if (!ret)
5919fcb191aSLokesh Vutla return 0;
5929fcb191aSLokesh Vutla
593537559a5SDmitry Kasatkin return 0;
594537559a5SDmitry Kasatkin }
595537559a5SDmitry Kasatkin
omap_aes_ecb_encrypt(struct skcipher_request * req)596b3e3f0feSArd Biesheuvel static int omap_aes_ecb_encrypt(struct skcipher_request *req)
597537559a5SDmitry Kasatkin {
598537559a5SDmitry Kasatkin return omap_aes_crypt(req, FLAGS_ENCRYPT);
599537559a5SDmitry Kasatkin }
600537559a5SDmitry Kasatkin
omap_aes_ecb_decrypt(struct skcipher_request * req)601b3e3f0feSArd Biesheuvel static int omap_aes_ecb_decrypt(struct skcipher_request *req)
602537559a5SDmitry Kasatkin {
603537559a5SDmitry Kasatkin return omap_aes_crypt(req, 0);
604537559a5SDmitry Kasatkin }
605537559a5SDmitry Kasatkin
omap_aes_cbc_encrypt(struct skcipher_request * req)606b3e3f0feSArd Biesheuvel static int omap_aes_cbc_encrypt(struct skcipher_request *req)
607537559a5SDmitry Kasatkin {
608537559a5SDmitry Kasatkin return omap_aes_crypt(req, FLAGS_ENCRYPT | FLAGS_CBC);
609537559a5SDmitry Kasatkin }
610537559a5SDmitry Kasatkin
omap_aes_cbc_decrypt(struct skcipher_request * req)611b3e3f0feSArd Biesheuvel static int omap_aes_cbc_decrypt(struct skcipher_request *req)
612537559a5SDmitry Kasatkin {
613537559a5SDmitry Kasatkin return omap_aes_crypt(req, FLAGS_CBC);
614537559a5SDmitry Kasatkin }
615537559a5SDmitry Kasatkin
omap_aes_ctr_encrypt(struct skcipher_request * req)616b3e3f0feSArd Biesheuvel static int omap_aes_ctr_encrypt(struct skcipher_request *req)
617f9fb69e7SMark A. Greer {
618f9fb69e7SMark A. Greer return omap_aes_crypt(req, FLAGS_ENCRYPT | FLAGS_CTR);
619f9fb69e7SMark A. Greer }
620f9fb69e7SMark A. Greer
omap_aes_ctr_decrypt(struct skcipher_request * req)621b3e3f0feSArd Biesheuvel static int omap_aes_ctr_decrypt(struct skcipher_request *req)
622f9fb69e7SMark A. Greer {
623f9fb69e7SMark A. Greer return omap_aes_crypt(req, FLAGS_CTR);
624f9fb69e7SMark A. Greer }
625f9fb69e7SMark A. Greer
omap_aes_init_tfm(struct crypto_skcipher * tfm)626b3e3f0feSArd Biesheuvel static int omap_aes_init_tfm(struct crypto_skcipher *tfm)
627537559a5SDmitry Kasatkin {
628b3e3f0feSArd Biesheuvel const char *name = crypto_tfm_alg_name(&tfm->base);
629b3e3f0feSArd Biesheuvel struct omap_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
6306a99d7a2SArd Biesheuvel struct crypto_skcipher *blk;
6319fcb191aSLokesh Vutla
6326a99d7a2SArd Biesheuvel blk = crypto_alloc_skcipher(name, 0, CRYPTO_ALG_NEED_FALLBACK);
6339fcb191aSLokesh Vutla if (IS_ERR(blk))
6349fcb191aSLokesh Vutla return PTR_ERR(blk);
6359fcb191aSLokesh Vutla
6369fcb191aSLokesh Vutla ctx->fallback = blk;
6379fcb191aSLokesh Vutla
6386a99d7a2SArd Biesheuvel crypto_skcipher_set_reqsize(tfm, sizeof(struct omap_aes_reqctx) +
6396a99d7a2SArd Biesheuvel crypto_skcipher_reqsize(blk));
640537559a5SDmitry Kasatkin
641537559a5SDmitry Kasatkin return 0;
642537559a5SDmitry Kasatkin }
643537559a5SDmitry Kasatkin
omap_aes_exit_tfm(struct crypto_skcipher * tfm)644b3e3f0feSArd Biesheuvel static void omap_aes_exit_tfm(struct crypto_skcipher *tfm)
645537559a5SDmitry Kasatkin {
646b3e3f0feSArd Biesheuvel struct omap_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
6479fcb191aSLokesh Vutla
6489fcb191aSLokesh Vutla if (ctx->fallback)
6496a99d7a2SArd Biesheuvel crypto_free_skcipher(ctx->fallback);
6509fcb191aSLokesh Vutla
6519fcb191aSLokesh Vutla ctx->fallback = NULL;
652537559a5SDmitry Kasatkin }
653537559a5SDmitry Kasatkin
654537559a5SDmitry Kasatkin /* ********************** ALGS ************************************ */
655537559a5SDmitry Kasatkin
656*03906fbaSHerbert Xu static struct skcipher_engine_alg algs_ecb_cbc[] = {
657537559a5SDmitry Kasatkin {
658*03906fbaSHerbert Xu .base = {
659b3e3f0feSArd Biesheuvel .base.cra_name = "ecb(aes)",
660b3e3f0feSArd Biesheuvel .base.cra_driver_name = "ecb-aes-omap",
661b3e3f0feSArd Biesheuvel .base.cra_priority = 300,
662b3e3f0feSArd Biesheuvel .base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY |
663b3e3f0feSArd Biesheuvel CRYPTO_ALG_ASYNC |
664b3e3f0feSArd Biesheuvel CRYPTO_ALG_NEED_FALLBACK,
665b3e3f0feSArd Biesheuvel .base.cra_blocksize = AES_BLOCK_SIZE,
666b3e3f0feSArd Biesheuvel .base.cra_ctxsize = sizeof(struct omap_aes_ctx),
667b3e3f0feSArd Biesheuvel .base.cra_module = THIS_MODULE,
668b3e3f0feSArd Biesheuvel
669537559a5SDmitry Kasatkin .min_keysize = AES_MIN_KEY_SIZE,
670537559a5SDmitry Kasatkin .max_keysize = AES_MAX_KEY_SIZE,
671537559a5SDmitry Kasatkin .setkey = omap_aes_setkey,
672537559a5SDmitry Kasatkin .encrypt = omap_aes_ecb_encrypt,
673537559a5SDmitry Kasatkin .decrypt = omap_aes_ecb_decrypt,
674b3e3f0feSArd Biesheuvel .init = omap_aes_init_tfm,
675b3e3f0feSArd Biesheuvel .exit = omap_aes_exit_tfm,
676537559a5SDmitry Kasatkin },
677*03906fbaSHerbert Xu .op.do_one_request = omap_aes_crypt_req,
678*03906fbaSHerbert Xu },
679537559a5SDmitry Kasatkin {
680*03906fbaSHerbert Xu .base = {
681b3e3f0feSArd Biesheuvel .base.cra_name = "cbc(aes)",
682b3e3f0feSArd Biesheuvel .base.cra_driver_name = "cbc-aes-omap",
683b3e3f0feSArd Biesheuvel .base.cra_priority = 300,
684b3e3f0feSArd Biesheuvel .base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY |
685b3e3f0feSArd Biesheuvel CRYPTO_ALG_ASYNC |
686b3e3f0feSArd Biesheuvel CRYPTO_ALG_NEED_FALLBACK,
687b3e3f0feSArd Biesheuvel .base.cra_blocksize = AES_BLOCK_SIZE,
688b3e3f0feSArd Biesheuvel .base.cra_ctxsize = sizeof(struct omap_aes_ctx),
689b3e3f0feSArd Biesheuvel .base.cra_module = THIS_MODULE,
690b3e3f0feSArd Biesheuvel
691537559a5SDmitry Kasatkin .min_keysize = AES_MIN_KEY_SIZE,
692537559a5SDmitry Kasatkin .max_keysize = AES_MAX_KEY_SIZE,
693537559a5SDmitry Kasatkin .ivsize = AES_BLOCK_SIZE,
694537559a5SDmitry Kasatkin .setkey = omap_aes_setkey,
695537559a5SDmitry Kasatkin .encrypt = omap_aes_cbc_encrypt,
696537559a5SDmitry Kasatkin .decrypt = omap_aes_cbc_decrypt,
697b3e3f0feSArd Biesheuvel .init = omap_aes_init_tfm,
698b3e3f0feSArd Biesheuvel .exit = omap_aes_exit_tfm,
699*03906fbaSHerbert Xu },
700*03906fbaSHerbert Xu .op.do_one_request = omap_aes_crypt_req,
701537559a5SDmitry Kasatkin }
702537559a5SDmitry Kasatkin };
703537559a5SDmitry Kasatkin
704*03906fbaSHerbert Xu static struct skcipher_engine_alg algs_ctr[] = {
705f9fb69e7SMark A. Greer {
706*03906fbaSHerbert Xu .base = {
707b3e3f0feSArd Biesheuvel .base.cra_name = "ctr(aes)",
708b3e3f0feSArd Biesheuvel .base.cra_driver_name = "ctr-aes-omap",
709b3e3f0feSArd Biesheuvel .base.cra_priority = 300,
710b3e3f0feSArd Biesheuvel .base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY |
711b3e3f0feSArd Biesheuvel CRYPTO_ALG_ASYNC |
712b3e3f0feSArd Biesheuvel CRYPTO_ALG_NEED_FALLBACK,
713a9459bdcSArd Biesheuvel .base.cra_blocksize = 1,
714b3e3f0feSArd Biesheuvel .base.cra_ctxsize = sizeof(struct omap_aes_ctx),
715b3e3f0feSArd Biesheuvel .base.cra_module = THIS_MODULE,
716b3e3f0feSArd Biesheuvel
717f9fb69e7SMark A. Greer .min_keysize = AES_MIN_KEY_SIZE,
718f9fb69e7SMark A. Greer .max_keysize = AES_MAX_KEY_SIZE,
719f9fb69e7SMark A. Greer .ivsize = AES_BLOCK_SIZE,
720f9fb69e7SMark A. Greer .setkey = omap_aes_setkey,
721f9fb69e7SMark A. Greer .encrypt = omap_aes_ctr_encrypt,
722f9fb69e7SMark A. Greer .decrypt = omap_aes_ctr_decrypt,
723b3e3f0feSArd Biesheuvel .init = omap_aes_init_tfm,
724b3e3f0feSArd Biesheuvel .exit = omap_aes_exit_tfm,
725*03906fbaSHerbert Xu },
726*03906fbaSHerbert Xu .op.do_one_request = omap_aes_crypt_req,
727f9fb69e7SMark A. Greer }
728f9fb69e7SMark A. Greer };
729f9fb69e7SMark A. Greer
730f9fb69e7SMark A. Greer static struct omap_aes_algs_info omap_aes_algs_info_ecb_cbc[] = {
731f9fb69e7SMark A. Greer {
732f9fb69e7SMark A. Greer .algs_list = algs_ecb_cbc,
733f9fb69e7SMark A. Greer .size = ARRAY_SIZE(algs_ecb_cbc),
734f9fb69e7SMark A. Greer },
735f9fb69e7SMark A. Greer };
736f9fb69e7SMark A. Greer
737*03906fbaSHerbert Xu static struct aead_engine_alg algs_aead_gcm[] = {
738ad18cc9dSTero Kristo {
739ad18cc9dSTero Kristo .base = {
740*03906fbaSHerbert Xu .base = {
741ad18cc9dSTero Kristo .cra_name = "gcm(aes)",
742ad18cc9dSTero Kristo .cra_driver_name = "gcm-aes-omap",
743ad18cc9dSTero Kristo .cra_priority = 300,
744ad18cc9dSTero Kristo .cra_flags = CRYPTO_ALG_ASYNC |
745ad18cc9dSTero Kristo CRYPTO_ALG_KERN_DRIVER_ONLY,
746ad18cc9dSTero Kristo .cra_blocksize = 1,
747f0956d42SArd Biesheuvel .cra_ctxsize = sizeof(struct omap_aes_gcm_ctx),
748ad18cc9dSTero Kristo .cra_alignmask = 0xf,
749ad18cc9dSTero Kristo .cra_module = THIS_MODULE,
750ad18cc9dSTero Kristo },
751ad18cc9dSTero Kristo .init = omap_aes_gcm_cra_init,
752cb3f3817SCorentin LABBE .ivsize = GCM_AES_IV_SIZE,
753ad18cc9dSTero Kristo .maxauthsize = AES_BLOCK_SIZE,
754ad18cc9dSTero Kristo .setkey = omap_aes_gcm_setkey,
75512adf9d6SArd Biesheuvel .setauthsize = omap_aes_gcm_setauthsize,
756ad18cc9dSTero Kristo .encrypt = omap_aes_gcm_encrypt,
757ad18cc9dSTero Kristo .decrypt = omap_aes_gcm_decrypt,
758ad18cc9dSTero Kristo },
759*03906fbaSHerbert Xu .op.do_one_request = omap_aes_gcm_crypt_req,
760*03906fbaSHerbert Xu },
761ad18cc9dSTero Kristo {
762ad18cc9dSTero Kristo .base = {
763*03906fbaSHerbert Xu .base = {
764ad18cc9dSTero Kristo .cra_name = "rfc4106(gcm(aes))",
765ad18cc9dSTero Kristo .cra_driver_name = "rfc4106-gcm-aes-omap",
766ad18cc9dSTero Kristo .cra_priority = 300,
767ad18cc9dSTero Kristo .cra_flags = CRYPTO_ALG_ASYNC |
768ad18cc9dSTero Kristo CRYPTO_ALG_KERN_DRIVER_ONLY,
769ad18cc9dSTero Kristo .cra_blocksize = 1,
770f0956d42SArd Biesheuvel .cra_ctxsize = sizeof(struct omap_aes_gcm_ctx),
771ad18cc9dSTero Kristo .cra_alignmask = 0xf,
772ad18cc9dSTero Kristo .cra_module = THIS_MODULE,
773ad18cc9dSTero Kristo },
774ad18cc9dSTero Kristo .init = omap_aes_gcm_cra_init,
775ad18cc9dSTero Kristo .maxauthsize = AES_BLOCK_SIZE,
776cb3f3817SCorentin LABBE .ivsize = GCM_RFC4106_IV_SIZE,
777ad18cc9dSTero Kristo .setkey = omap_aes_4106gcm_setkey,
77812adf9d6SArd Biesheuvel .setauthsize = omap_aes_4106gcm_setauthsize,
779ad18cc9dSTero Kristo .encrypt = omap_aes_4106gcm_encrypt,
780ad18cc9dSTero Kristo .decrypt = omap_aes_4106gcm_decrypt,
781ad18cc9dSTero Kristo },
782*03906fbaSHerbert Xu .op.do_one_request = omap_aes_gcm_crypt_req,
783*03906fbaSHerbert Xu },
784ad18cc9dSTero Kristo };
785ad18cc9dSTero Kristo
786ad18cc9dSTero Kristo static struct omap_aes_aead_algs omap_aes_aead_info = {
787ad18cc9dSTero Kristo .algs_list = algs_aead_gcm,
788ad18cc9dSTero Kristo .size = ARRAY_SIZE(algs_aead_gcm),
789ad18cc9dSTero Kristo };
790ad18cc9dSTero Kristo
7910d35583aSMark A. Greer static const struct omap_aes_pdata omap_aes_pdata_omap2 = {
792f9fb69e7SMark A. Greer .algs_info = omap_aes_algs_info_ecb_cbc,
793f9fb69e7SMark A. Greer .algs_info_size = ARRAY_SIZE(omap_aes_algs_info_ecb_cbc),
7940d35583aSMark A. Greer .trigger = omap_aes_dma_trigger_omap2,
7950d35583aSMark A. Greer .key_ofs = 0x1c,
7960d35583aSMark A. Greer .iv_ofs = 0x20,
7970d35583aSMark A. Greer .ctrl_ofs = 0x30,
7980d35583aSMark A. Greer .data_ofs = 0x34,
7990d35583aSMark A. Greer .rev_ofs = 0x44,
8000d35583aSMark A. Greer .mask_ofs = 0x48,
8010d35583aSMark A. Greer .dma_enable_in = BIT(2),
8020d35583aSMark A. Greer .dma_enable_out = BIT(3),
8030d35583aSMark A. Greer .dma_start = BIT(5),
8040d35583aSMark A. Greer .major_mask = 0xf0,
8050d35583aSMark A. Greer .major_shift = 4,
8060d35583aSMark A. Greer .minor_mask = 0x0f,
8070d35583aSMark A. Greer .minor_shift = 0,
8080d35583aSMark A. Greer };
8090d35583aSMark A. Greer
810bc69d124SMark A. Greer #ifdef CONFIG_OF
811f9fb69e7SMark A. Greer static struct omap_aes_algs_info omap_aes_algs_info_ecb_cbc_ctr[] = {
812f9fb69e7SMark A. Greer {
813f9fb69e7SMark A. Greer .algs_list = algs_ecb_cbc,
814f9fb69e7SMark A. Greer .size = ARRAY_SIZE(algs_ecb_cbc),
815f9fb69e7SMark A. Greer },
816f9fb69e7SMark A. Greer {
817f9fb69e7SMark A. Greer .algs_list = algs_ctr,
818f9fb69e7SMark A. Greer .size = ARRAY_SIZE(algs_ctr),
819f9fb69e7SMark A. Greer },
820f9fb69e7SMark A. Greer };
821f9fb69e7SMark A. Greer
822f9fb69e7SMark A. Greer static const struct omap_aes_pdata omap_aes_pdata_omap3 = {
823f9fb69e7SMark A. Greer .algs_info = omap_aes_algs_info_ecb_cbc_ctr,
824f9fb69e7SMark A. Greer .algs_info_size = ARRAY_SIZE(omap_aes_algs_info_ecb_cbc_ctr),
825f9fb69e7SMark A. Greer .trigger = omap_aes_dma_trigger_omap2,
826f9fb69e7SMark A. Greer .key_ofs = 0x1c,
827f9fb69e7SMark A. Greer .iv_ofs = 0x20,
828f9fb69e7SMark A. Greer .ctrl_ofs = 0x30,
829f9fb69e7SMark A. Greer .data_ofs = 0x34,
830f9fb69e7SMark A. Greer .rev_ofs = 0x44,
831f9fb69e7SMark A. Greer .mask_ofs = 0x48,
832f9fb69e7SMark A. Greer .dma_enable_in = BIT(2),
833f9fb69e7SMark A. Greer .dma_enable_out = BIT(3),
834f9fb69e7SMark A. Greer .dma_start = BIT(5),
835f9fb69e7SMark A. Greer .major_mask = 0xf0,
836f9fb69e7SMark A. Greer .major_shift = 4,
837f9fb69e7SMark A. Greer .minor_mask = 0x0f,
838f9fb69e7SMark A. Greer .minor_shift = 0,
839f9fb69e7SMark A. Greer };
840f9fb69e7SMark A. Greer
8410d35583aSMark A. Greer static const struct omap_aes_pdata omap_aes_pdata_omap4 = {
842f9fb69e7SMark A. Greer .algs_info = omap_aes_algs_info_ecb_cbc_ctr,
843f9fb69e7SMark A. Greer .algs_info_size = ARRAY_SIZE(omap_aes_algs_info_ecb_cbc_ctr),
844ad18cc9dSTero Kristo .aead_algs_info = &omap_aes_aead_info,
8450d35583aSMark A. Greer .trigger = omap_aes_dma_trigger_omap4,
8460d35583aSMark A. Greer .key_ofs = 0x3c,
8470d35583aSMark A. Greer .iv_ofs = 0x40,
8480d35583aSMark A. Greer .ctrl_ofs = 0x50,
8490d35583aSMark A. Greer .data_ofs = 0x60,
8500d35583aSMark A. Greer .rev_ofs = 0x80,
8510d35583aSMark A. Greer .mask_ofs = 0x84,
85267216756SJoel Fernandes .irq_status_ofs = 0x8c,
85367216756SJoel Fernandes .irq_enable_ofs = 0x90,
8540d35583aSMark A. Greer .dma_enable_in = BIT(5),
8550d35583aSMark A. Greer .dma_enable_out = BIT(6),
8560d35583aSMark A. Greer .major_mask = 0x0700,
8570d35583aSMark A. Greer .major_shift = 8,
8580d35583aSMark A. Greer .minor_mask = 0x003f,
8590d35583aSMark A. Greer .minor_shift = 0,
8600d35583aSMark A. Greer };
8610d35583aSMark A. Greer
omap_aes_irq(int irq,void * dev_id)8621bf95ccaSJoel Fernandes static irqreturn_t omap_aes_irq(int irq, void *dev_id)
8631bf95ccaSJoel Fernandes {
8641bf95ccaSJoel Fernandes struct omap_aes_dev *dd = dev_id;
8651bf95ccaSJoel Fernandes u32 status, i;
8661bf95ccaSJoel Fernandes u32 *src, *dst;
8671bf95ccaSJoel Fernandes
8681bf95ccaSJoel Fernandes status = omap_aes_read(dd, AES_REG_IRQ_STATUS(dd));
8691bf95ccaSJoel Fernandes if (status & AES_REG_IRQ_DATA_IN) {
8701bf95ccaSJoel Fernandes omap_aes_write(dd, AES_REG_IRQ_ENABLE(dd), 0x0);
8711bf95ccaSJoel Fernandes
8721bf95ccaSJoel Fernandes BUG_ON(!dd->in_sg);
8731bf95ccaSJoel Fernandes
8741bf95ccaSJoel Fernandes BUG_ON(_calc_walked(in) > dd->in_sg->length);
8751bf95ccaSJoel Fernandes
8761bf95ccaSJoel Fernandes src = sg_virt(dd->in_sg) + _calc_walked(in);
8771bf95ccaSJoel Fernandes
8781bf95ccaSJoel Fernandes for (i = 0; i < AES_BLOCK_WORDS; i++) {
8791bf95ccaSJoel Fernandes omap_aes_write(dd, AES_REG_DATA_N(dd, i), *src);
8801bf95ccaSJoel Fernandes
8811bf95ccaSJoel Fernandes scatterwalk_advance(&dd->in_walk, 4);
8821bf95ccaSJoel Fernandes if (dd->in_sg->length == _calc_walked(in)) {
8835be4d4c9SCristian Stoica dd->in_sg = sg_next(dd->in_sg);
8841bf95ccaSJoel Fernandes if (dd->in_sg) {
8851bf95ccaSJoel Fernandes scatterwalk_start(&dd->in_walk,
8861bf95ccaSJoel Fernandes dd->in_sg);
8871bf95ccaSJoel Fernandes src = sg_virt(dd->in_sg) +
8881bf95ccaSJoel Fernandes _calc_walked(in);
8891bf95ccaSJoel Fernandes }
8901bf95ccaSJoel Fernandes } else {
8911bf95ccaSJoel Fernandes src++;
8921bf95ccaSJoel Fernandes }
8931bf95ccaSJoel Fernandes }
8941bf95ccaSJoel Fernandes
8951bf95ccaSJoel Fernandes /* Clear IRQ status */
8961bf95ccaSJoel Fernandes status &= ~AES_REG_IRQ_DATA_IN;
8971bf95ccaSJoel Fernandes omap_aes_write(dd, AES_REG_IRQ_STATUS(dd), status);
8981bf95ccaSJoel Fernandes
8991bf95ccaSJoel Fernandes /* Enable DATA_OUT interrupt */
9001bf95ccaSJoel Fernandes omap_aes_write(dd, AES_REG_IRQ_ENABLE(dd), 0x4);
9011bf95ccaSJoel Fernandes
9021bf95ccaSJoel Fernandes } else if (status & AES_REG_IRQ_DATA_OUT) {
9031bf95ccaSJoel Fernandes omap_aes_write(dd, AES_REG_IRQ_ENABLE(dd), 0x0);
9041bf95ccaSJoel Fernandes
9051bf95ccaSJoel Fernandes BUG_ON(!dd->out_sg);
9061bf95ccaSJoel Fernandes
9071bf95ccaSJoel Fernandes BUG_ON(_calc_walked(out) > dd->out_sg->length);
9081bf95ccaSJoel Fernandes
9091bf95ccaSJoel Fernandes dst = sg_virt(dd->out_sg) + _calc_walked(out);
9101bf95ccaSJoel Fernandes
9111bf95ccaSJoel Fernandes for (i = 0; i < AES_BLOCK_WORDS; i++) {
9121bf95ccaSJoel Fernandes *dst = omap_aes_read(dd, AES_REG_DATA_N(dd, i));
9131bf95ccaSJoel Fernandes scatterwalk_advance(&dd->out_walk, 4);
9141bf95ccaSJoel Fernandes if (dd->out_sg->length == _calc_walked(out)) {
9155be4d4c9SCristian Stoica dd->out_sg = sg_next(dd->out_sg);
9161bf95ccaSJoel Fernandes if (dd->out_sg) {
9171bf95ccaSJoel Fernandes scatterwalk_start(&dd->out_walk,
9181bf95ccaSJoel Fernandes dd->out_sg);
9191bf95ccaSJoel Fernandes dst = sg_virt(dd->out_sg) +
9201bf95ccaSJoel Fernandes _calc_walked(out);
9211bf95ccaSJoel Fernandes }
9221bf95ccaSJoel Fernandes } else {
9231bf95ccaSJoel Fernandes dst++;
9241bf95ccaSJoel Fernandes }
9251bf95ccaSJoel Fernandes }
9261bf95ccaSJoel Fernandes
927310b0d55SVutla, Lokesh dd->total -= min_t(size_t, AES_BLOCK_SIZE, dd->total);
9281bf95ccaSJoel Fernandes
9291bf95ccaSJoel Fernandes /* Clear IRQ status */
9301bf95ccaSJoel Fernandes status &= ~AES_REG_IRQ_DATA_OUT;
9311bf95ccaSJoel Fernandes omap_aes_write(dd, AES_REG_IRQ_STATUS(dd), status);
9321bf95ccaSJoel Fernandes
9331bf95ccaSJoel Fernandes if (!dd->total)
9341bf95ccaSJoel Fernandes /* All bytes read! */
9351bf95ccaSJoel Fernandes tasklet_schedule(&dd->done_task);
9361bf95ccaSJoel Fernandes else
9371bf95ccaSJoel Fernandes /* Enable DATA_IN interrupt for next block */
9381bf95ccaSJoel Fernandes omap_aes_write(dd, AES_REG_IRQ_ENABLE(dd), 0x2);
9391bf95ccaSJoel Fernandes }
9401bf95ccaSJoel Fernandes
9411bf95ccaSJoel Fernandes return IRQ_HANDLED;
9421bf95ccaSJoel Fernandes }
9431bf95ccaSJoel Fernandes
944bc69d124SMark A. Greer static const struct of_device_id omap_aes_of_match[] = {
945bc69d124SMark A. Greer {
946bc69d124SMark A. Greer .compatible = "ti,omap2-aes",
9470d35583aSMark A. Greer .data = &omap_aes_pdata_omap2,
9480d35583aSMark A. Greer },
9490d35583aSMark A. Greer {
950f9fb69e7SMark A. Greer .compatible = "ti,omap3-aes",
951f9fb69e7SMark A. Greer .data = &omap_aes_pdata_omap3,
952f9fb69e7SMark A. Greer },
953f9fb69e7SMark A. Greer {
9540d35583aSMark A. Greer .compatible = "ti,omap4-aes",
9550d35583aSMark A. Greer .data = &omap_aes_pdata_omap4,
956bc69d124SMark A. Greer },
957bc69d124SMark A. Greer {},
958bc69d124SMark A. Greer };
959bc69d124SMark A. Greer MODULE_DEVICE_TABLE(of, omap_aes_of_match);
960bc69d124SMark A. Greer
omap_aes_get_res_of(struct omap_aes_dev * dd,struct device * dev,struct resource * res)961bc69d124SMark A. Greer static int omap_aes_get_res_of(struct omap_aes_dev *dd,
962bc69d124SMark A. Greer struct device *dev, struct resource *res)
963bc69d124SMark A. Greer {
964bc69d124SMark A. Greer struct device_node *node = dev->of_node;
965bc69d124SMark A. Greer int err = 0;
966bc69d124SMark A. Greer
9677d556931SCorentin LABBE dd->pdata = of_device_get_match_data(dev);
9687d556931SCorentin LABBE if (!dd->pdata) {
969bc69d124SMark A. Greer dev_err(dev, "no compatible OF match\n");
970bc69d124SMark A. Greer err = -EINVAL;
971bc69d124SMark A. Greer goto err;
972bc69d124SMark A. Greer }
973bc69d124SMark A. Greer
974bc69d124SMark A. Greer err = of_address_to_resource(node, 0, res);
975bc69d124SMark A. Greer if (err < 0) {
976bc69d124SMark A. Greer dev_err(dev, "can't translate OF node address\n");
977bc69d124SMark A. Greer err = -EINVAL;
978bc69d124SMark A. Greer goto err;
979bc69d124SMark A. Greer }
980bc69d124SMark A. Greer
981bc69d124SMark A. Greer err:
982bc69d124SMark A. Greer return err;
983bc69d124SMark A. Greer }
984bc69d124SMark A. Greer #else
985bc69d124SMark A. Greer static const struct of_device_id omap_aes_of_match[] = {
986bc69d124SMark A. Greer {},
987bc69d124SMark A. Greer };
988bc69d124SMark A. Greer
omap_aes_get_res_of(struct omap_aes_dev * dd,struct device * dev,struct resource * res)989bc69d124SMark A. Greer static int omap_aes_get_res_of(struct omap_aes_dev *dd,
990bc69d124SMark A. Greer struct device *dev, struct resource *res)
991bc69d124SMark A. Greer {
992bc69d124SMark A. Greer return -EINVAL;
993bc69d124SMark A. Greer }
994bc69d124SMark A. Greer #endif
995bc69d124SMark A. Greer
omap_aes_get_res_pdev(struct omap_aes_dev * dd,struct platform_device * pdev,struct resource * res)996bc69d124SMark A. Greer static int omap_aes_get_res_pdev(struct omap_aes_dev *dd,
997bc69d124SMark A. Greer struct platform_device *pdev, struct resource *res)
998bc69d124SMark A. Greer {
999bc69d124SMark A. Greer struct device *dev = &pdev->dev;
1000bc69d124SMark A. Greer struct resource *r;
1001bc69d124SMark A. Greer int err = 0;
1002bc69d124SMark A. Greer
1003bc69d124SMark A. Greer /* Get the base address */
1004bc69d124SMark A. Greer r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1005bc69d124SMark A. Greer if (!r) {
1006bc69d124SMark A. Greer dev_err(dev, "no MEM resource info\n");
1007bc69d124SMark A. Greer err = -ENODEV;
1008bc69d124SMark A. Greer goto err;
1009bc69d124SMark A. Greer }
1010bc69d124SMark A. Greer memcpy(res, r, sizeof(*res));
1011bc69d124SMark A. Greer
10120d35583aSMark A. Greer /* Only OMAP2/3 can be non-DT */
10130d35583aSMark A. Greer dd->pdata = &omap_aes_pdata_omap2;
10140d35583aSMark A. Greer
1015bc69d124SMark A. Greer err:
1016bc69d124SMark A. Greer return err;
1017bc69d124SMark A. Greer }
1018bc69d124SMark A. Greer
fallback_show(struct device * dev,struct device_attribute * attr,char * buf)1019537c62caSTero Kristo static ssize_t fallback_show(struct device *dev, struct device_attribute *attr,
1020537c62caSTero Kristo char *buf)
1021537c62caSTero Kristo {
1022537c62caSTero Kristo return sprintf(buf, "%d\n", aes_fallback_sz);
1023537c62caSTero Kristo }
1024537c62caSTero Kristo
fallback_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t size)1025537c62caSTero Kristo static ssize_t fallback_store(struct device *dev, struct device_attribute *attr,
1026537c62caSTero Kristo const char *buf, size_t size)
1027537c62caSTero Kristo {
1028537c62caSTero Kristo ssize_t status;
1029537c62caSTero Kristo long value;
1030537c62caSTero Kristo
1031537c62caSTero Kristo status = kstrtol(buf, 0, &value);
1032537c62caSTero Kristo if (status)
1033537c62caSTero Kristo return status;
1034537c62caSTero Kristo
1035537c62caSTero Kristo /* HW accelerator only works with buffers > 9 */
1036537c62caSTero Kristo if (value < 9) {
1037537c62caSTero Kristo dev_err(dev, "minimum fallback size 9\n");
1038537c62caSTero Kristo return -EINVAL;
1039537c62caSTero Kristo }
1040537c62caSTero Kristo
1041537c62caSTero Kristo aes_fallback_sz = value;
1042537c62caSTero Kristo
1043537c62caSTero Kristo return size;
1044537c62caSTero Kristo }
1045537c62caSTero Kristo
queue_len_show(struct device * dev,struct device_attribute * attr,char * buf)10465007387fSTero Kristo static ssize_t queue_len_show(struct device *dev, struct device_attribute *attr,
10475007387fSTero Kristo char *buf)
10485007387fSTero Kristo {
10495007387fSTero Kristo struct omap_aes_dev *dd = dev_get_drvdata(dev);
10505007387fSTero Kristo
10515007387fSTero Kristo return sprintf(buf, "%d\n", dd->engine->queue.max_qlen);
10525007387fSTero Kristo }
10535007387fSTero Kristo
queue_len_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t size)10545007387fSTero Kristo static ssize_t queue_len_store(struct device *dev,
10555007387fSTero Kristo struct device_attribute *attr, const char *buf,
10565007387fSTero Kristo size_t size)
10575007387fSTero Kristo {
10585007387fSTero Kristo struct omap_aes_dev *dd;
10595007387fSTero Kristo ssize_t status;
10605007387fSTero Kristo long value;
10615007387fSTero Kristo unsigned long flags;
10625007387fSTero Kristo
10635007387fSTero Kristo status = kstrtol(buf, 0, &value);
10645007387fSTero Kristo if (status)
10655007387fSTero Kristo return status;
10665007387fSTero Kristo
10675007387fSTero Kristo if (value < 1)
10685007387fSTero Kristo return -EINVAL;
10695007387fSTero Kristo
10705007387fSTero Kristo /*
10715007387fSTero Kristo * Changing the queue size in fly is safe, if size becomes smaller
10725007387fSTero Kristo * than current size, it will just not accept new entries until
10735007387fSTero Kristo * it has shrank enough.
10745007387fSTero Kristo */
10755007387fSTero Kristo spin_lock_bh(&list_lock);
10765007387fSTero Kristo list_for_each_entry(dd, &dev_list, list) {
10775007387fSTero Kristo spin_lock_irqsave(&dd->lock, flags);
10785007387fSTero Kristo dd->engine->queue.max_qlen = value;
10795007387fSTero Kristo dd->aead_queue.base.max_qlen = value;
10805007387fSTero Kristo spin_unlock_irqrestore(&dd->lock, flags);
10815007387fSTero Kristo }
10825007387fSTero Kristo spin_unlock_bh(&list_lock);
10835007387fSTero Kristo
10845007387fSTero Kristo return size;
10855007387fSTero Kristo }
10865007387fSTero Kristo
10875007387fSTero Kristo static DEVICE_ATTR_RW(queue_len);
1088537c62caSTero Kristo static DEVICE_ATTR_RW(fallback);
1089537c62caSTero Kristo
1090537c62caSTero Kristo static struct attribute *omap_aes_attrs[] = {
10915007387fSTero Kristo &dev_attr_queue_len.attr,
1092537c62caSTero Kristo &dev_attr_fallback.attr,
1093537c62caSTero Kristo NULL,
1094537c62caSTero Kristo };
1095537c62caSTero Kristo
1096882f6c60SRikard Falkeborn static const struct attribute_group omap_aes_attr_group = {
1097537c62caSTero Kristo .attrs = omap_aes_attrs,
1098537c62caSTero Kristo };
1099537c62caSTero Kristo
omap_aes_probe(struct platform_device * pdev)1100537559a5SDmitry Kasatkin static int omap_aes_probe(struct platform_device *pdev)
1101537559a5SDmitry Kasatkin {
1102537559a5SDmitry Kasatkin struct device *dev = &pdev->dev;
1103537559a5SDmitry Kasatkin struct omap_aes_dev *dd;
1104*03906fbaSHerbert Xu struct skcipher_engine_alg *algp;
1105*03906fbaSHerbert Xu struct aead_engine_alg *aalg;
1106bc69d124SMark A. Greer struct resource res;
11071801ad94SJoel Fernandes int err = -ENOMEM, i, j, irq = -1;
1108537559a5SDmitry Kasatkin u32 reg;
1109537559a5SDmitry Kasatkin
111005007c10SJoel Fernandes dd = devm_kzalloc(dev, sizeof(struct omap_aes_dev), GFP_KERNEL);
1111537559a5SDmitry Kasatkin if (dd == NULL) {
1112537559a5SDmitry Kasatkin dev_err(dev, "unable to alloc data struct.\n");
1113537559a5SDmitry Kasatkin goto err_data;
1114537559a5SDmitry Kasatkin }
1115537559a5SDmitry Kasatkin dd->dev = dev;
1116537559a5SDmitry Kasatkin platform_set_drvdata(pdev, dd);
1117537559a5SDmitry Kasatkin
1118ad18cc9dSTero Kristo aead_init_queue(&dd->aead_queue, OMAP_AES_QUEUE_LENGTH);
1119ad18cc9dSTero Kristo
1120bc69d124SMark A. Greer err = (dev->of_node) ? omap_aes_get_res_of(dd, dev, &res) :
1121bc69d124SMark A. Greer omap_aes_get_res_pdev(dd, pdev, &res);
1122bc69d124SMark A. Greer if (err)
1123537559a5SDmitry Kasatkin goto err_res;
1124537559a5SDmitry Kasatkin
112530862281SLaurent Navet dd->io_base = devm_ioremap_resource(dev, &res);
112630862281SLaurent Navet if (IS_ERR(dd->io_base)) {
112730862281SLaurent Navet err = PTR_ERR(dd->io_base);
11285946c4a5SMark A. Greer goto err_res;
1129537559a5SDmitry Kasatkin }
1130bc69d124SMark A. Greer dd->phys_base = res.start;
1131537559a5SDmitry Kasatkin
1132f303b455STero Kristo pm_runtime_use_autosuspend(dev);
1133f303b455STero Kristo pm_runtime_set_autosuspend_delay(dev, DEFAULT_AUTOSUSPEND_DELAY);
1134f303b455STero Kristo
11355946c4a5SMark A. Greer pm_runtime_enable(dev);
11361f34cc4aSShixin Liu err = pm_runtime_resume_and_get(dev);
1137f7b2b5ddSNishanth Menon if (err < 0) {
1138f7b2b5ddSNishanth Menon dev_err(dev, "%s: failed to get_sync(%d)\n",
1139f7b2b5ddSNishanth Menon __func__, err);
1140ff810720SZhang Qilong goto err_pm_disable;
1141f7b2b5ddSNishanth Menon }
11425946c4a5SMark A. Greer
11430d35583aSMark A. Greer omap_aes_dma_stop(dd);
11440d35583aSMark A. Greer
11450d35583aSMark A. Greer reg = omap_aes_read(dd, AES_REG_REV(dd));
11465946c4a5SMark A. Greer
11475946c4a5SMark A. Greer pm_runtime_put_sync(dev);
1148537559a5SDmitry Kasatkin
11490d35583aSMark A. Greer dev_info(dev, "OMAP AES hw accel rev: %u.%u\n",
11500d35583aSMark A. Greer (reg & dd->pdata->major_mask) >> dd->pdata->major_shift,
11510d35583aSMark A. Greer (reg & dd->pdata->minor_mask) >> dd->pdata->minor_shift);
11520d35583aSMark A. Greer
115321fe9767SDmitry Kasatkin tasklet_init(&dd->done_task, omap_aes_done_task, (unsigned long)dd);
1154537559a5SDmitry Kasatkin
1155537559a5SDmitry Kasatkin err = omap_aes_dma_init(dd);
1156da8b29a6SPeter Ujfalusi if (err == -EPROBE_DEFER) {
1157da8b29a6SPeter Ujfalusi goto err_irq;
1158da8b29a6SPeter Ujfalusi } else if (err && AES_REG_IRQ_STATUS(dd) && AES_REG_IRQ_ENABLE(dd)) {
11591801ad94SJoel Fernandes dd->pio_only = 1;
11601801ad94SJoel Fernandes
11611801ad94SJoel Fernandes irq = platform_get_irq(pdev, 0);
11621801ad94SJoel Fernandes if (irq < 0) {
116362c58f8dSGustavo A. R. Silva err = irq;
11641801ad94SJoel Fernandes goto err_irq;
11651801ad94SJoel Fernandes }
11661801ad94SJoel Fernandes
1167bce2a228SJoel Fernandes err = devm_request_irq(dev, irq, omap_aes_irq, 0,
11681801ad94SJoel Fernandes dev_name(dev), dd);
11691801ad94SJoel Fernandes if (err) {
11701801ad94SJoel Fernandes dev_err(dev, "Unable to grab omap-aes IRQ\n");
11711801ad94SJoel Fernandes goto err_irq;
11721801ad94SJoel Fernandes }
11731801ad94SJoel Fernandes }
11741801ad94SJoel Fernandes
1175ad18cc9dSTero Kristo spin_lock_init(&dd->lock);
1176537559a5SDmitry Kasatkin
1177537559a5SDmitry Kasatkin INIT_LIST_HEAD(&dd->list);
1178fe4d5577SBen Hutchings spin_lock_bh(&list_lock);
1179537559a5SDmitry Kasatkin list_add_tail(&dd->list, &dev_list);
1180fe4d5577SBen Hutchings spin_unlock_bh(&list_lock);
1181537559a5SDmitry Kasatkin
11820d0cda93STero Kristo /* Initialize crypto engine */
11830d0cda93STero Kristo dd->engine = crypto_engine_alloc_init(dev, 1);
1184c98ef8dbSWei Yongjun if (!dd->engine) {
1185c98ef8dbSWei Yongjun err = -ENOMEM;
11860d0cda93STero Kristo goto err_engine;
1187c98ef8dbSWei Yongjun }
11880d0cda93STero Kristo
11890d0cda93STero Kristo err = crypto_engine_start(dd->engine);
11900d0cda93STero Kristo if (err)
11910d0cda93STero Kristo goto err_engine;
11920d0cda93STero Kristo
1193f9fb69e7SMark A. Greer for (i = 0; i < dd->pdata->algs_info_size; i++) {
11943741bbb2SLokesh Vutla if (!dd->pdata->algs_info[i].registered) {
1195f9fb69e7SMark A. Greer for (j = 0; j < dd->pdata->algs_info[i].size; j++) {
1196f9fb69e7SMark A. Greer algp = &dd->pdata->algs_info[i].algs_list[j];
1197f9fb69e7SMark A. Greer
1198*03906fbaSHerbert Xu pr_debug("reg alg: %s\n", algp->base.base.cra_name);
1199f9fb69e7SMark A. Greer
1200*03906fbaSHerbert Xu err = crypto_engine_register_skcipher(algp);
1201537559a5SDmitry Kasatkin if (err)
1202537559a5SDmitry Kasatkin goto err_algs;
1203f9fb69e7SMark A. Greer
1204f9fb69e7SMark A. Greer dd->pdata->algs_info[i].registered++;
1205f9fb69e7SMark A. Greer }
1206537559a5SDmitry Kasatkin }
12073741bbb2SLokesh Vutla }
1208537559a5SDmitry Kasatkin
1209ad18cc9dSTero Kristo if (dd->pdata->aead_algs_info &&
1210ad18cc9dSTero Kristo !dd->pdata->aead_algs_info->registered) {
1211ad18cc9dSTero Kristo for (i = 0; i < dd->pdata->aead_algs_info->size; i++) {
1212ad18cc9dSTero Kristo aalg = &dd->pdata->aead_algs_info->algs_list[i];
1213ad18cc9dSTero Kristo
1214*03906fbaSHerbert Xu pr_debug("reg alg: %s\n", aalg->base.base.cra_name);
1215ad18cc9dSTero Kristo
1216*03906fbaSHerbert Xu err = crypto_engine_register_aead(aalg);
1217ad18cc9dSTero Kristo if (err)
1218ad18cc9dSTero Kristo goto err_aead_algs;
1219ad18cc9dSTero Kristo
1220ad18cc9dSTero Kristo dd->pdata->aead_algs_info->registered++;
1221ad18cc9dSTero Kristo }
1222ad18cc9dSTero Kristo }
1223ad18cc9dSTero Kristo
1224537c62caSTero Kristo err = sysfs_create_group(&dev->kobj, &omap_aes_attr_group);
1225537c62caSTero Kristo if (err) {
1226537c62caSTero Kristo dev_err(dev, "could not create sysfs device attrs\n");
1227537c62caSTero Kristo goto err_aead_algs;
1228537c62caSTero Kristo }
1229537c62caSTero Kristo
1230537559a5SDmitry Kasatkin return 0;
1231ad18cc9dSTero Kristo err_aead_algs:
1232ad18cc9dSTero Kristo for (i = dd->pdata->aead_algs_info->registered - 1; i >= 0; i--) {
1233ad18cc9dSTero Kristo aalg = &dd->pdata->aead_algs_info->algs_list[i];
1234*03906fbaSHerbert Xu crypto_engine_unregister_aead(aalg);
1235ad18cc9dSTero Kristo }
1236537559a5SDmitry Kasatkin err_algs:
1237f9fb69e7SMark A. Greer for (i = dd->pdata->algs_info_size - 1; i >= 0; i--)
1238f9fb69e7SMark A. Greer for (j = dd->pdata->algs_info[i].registered - 1; j >= 0; j--)
1239*03906fbaSHerbert Xu crypto_engine_unregister_skcipher(
1240f9fb69e7SMark A. Greer &dd->pdata->algs_info[i].algs_list[j]);
1241da8b29a6SPeter Ujfalusi
12420d0cda93STero Kristo err_engine:
12430d0cda93STero Kristo if (dd->engine)
12440d0cda93STero Kristo crypto_engine_exit(dd->engine);
12450d0cda93STero Kristo
1246537559a5SDmitry Kasatkin omap_aes_dma_cleanup(dd);
12471801ad94SJoel Fernandes err_irq:
124821fe9767SDmitry Kasatkin tasklet_kill(&dd->done_task);
1249ff810720SZhang Qilong err_pm_disable:
12505946c4a5SMark A. Greer pm_runtime_disable(dev);
1251537559a5SDmitry Kasatkin err_res:
1252537559a5SDmitry Kasatkin dd = NULL;
1253537559a5SDmitry Kasatkin err_data:
1254537559a5SDmitry Kasatkin dev_err(dev, "initialization failed.\n");
1255537559a5SDmitry Kasatkin return err;
1256537559a5SDmitry Kasatkin }
1257537559a5SDmitry Kasatkin
omap_aes_remove(struct platform_device * pdev)1258537559a5SDmitry Kasatkin static int omap_aes_remove(struct platform_device *pdev)
1259537559a5SDmitry Kasatkin {
1260537559a5SDmitry Kasatkin struct omap_aes_dev *dd = platform_get_drvdata(pdev);
1261*03906fbaSHerbert Xu struct aead_engine_alg *aalg;
1262f9fb69e7SMark A. Greer int i, j;
1263537559a5SDmitry Kasatkin
1264fe4d5577SBen Hutchings spin_lock_bh(&list_lock);
1265537559a5SDmitry Kasatkin list_del(&dd->list);
1266fe4d5577SBen Hutchings spin_unlock_bh(&list_lock);
1267537559a5SDmitry Kasatkin
1268f9fb69e7SMark A. Greer for (i = dd->pdata->algs_info_size - 1; i >= 0; i--)
12699ef4e6e5STero Kristo for (j = dd->pdata->algs_info[i].registered - 1; j >= 0; j--) {
1270*03906fbaSHerbert Xu crypto_engine_unregister_skcipher(
1271f9fb69e7SMark A. Greer &dd->pdata->algs_info[i].algs_list[j]);
12729ef4e6e5STero Kristo dd->pdata->algs_info[i].registered--;
12739ef4e6e5STero Kristo }
1274537559a5SDmitry Kasatkin
12759ef4e6e5STero Kristo for (i = dd->pdata->aead_algs_info->registered - 1; i >= 0; i--) {
1276ad18cc9dSTero Kristo aalg = &dd->pdata->aead_algs_info->algs_list[i];
1277*03906fbaSHerbert Xu crypto_engine_unregister_aead(aalg);
12789ef4e6e5STero Kristo dd->pdata->aead_algs_info->registered--;
1279ad18cc9dSTero Kristo }
1280ad18cc9dSTero Kristo
12810529900aSBaolin Wang crypto_engine_exit(dd->engine);
1282ad18cc9dSTero Kristo
128321fe9767SDmitry Kasatkin tasklet_kill(&dd->done_task);
1284537559a5SDmitry Kasatkin omap_aes_dma_cleanup(dd);
12855946c4a5SMark A. Greer pm_runtime_disable(dd->dev);
1286e7508ef2STero Kristo
1287e7508ef2STero Kristo sysfs_remove_group(&dd->dev->kobj, &omap_aes_attr_group);
1288537559a5SDmitry Kasatkin
1289537559a5SDmitry Kasatkin return 0;
1290537559a5SDmitry Kasatkin }
1291537559a5SDmitry Kasatkin
12920635fb3aSMark A. Greer #ifdef CONFIG_PM_SLEEP
omap_aes_suspend(struct device * dev)12930635fb3aSMark A. Greer static int omap_aes_suspend(struct device *dev)
12940635fb3aSMark A. Greer {
12950635fb3aSMark A. Greer pm_runtime_put_sync(dev);
12960635fb3aSMark A. Greer return 0;
12970635fb3aSMark A. Greer }
12980635fb3aSMark A. Greer
omap_aes_resume(struct device * dev)12990635fb3aSMark A. Greer static int omap_aes_resume(struct device *dev)
13000635fb3aSMark A. Greer {
1301c2aec59bSHeiner Kallweit pm_runtime_get_sync(dev);
13020635fb3aSMark A. Greer return 0;
13030635fb3aSMark A. Greer }
13040635fb3aSMark A. Greer #endif
13050635fb3aSMark A. Greer
1306ea7b2843SJingoo Han static SIMPLE_DEV_PM_OPS(omap_aes_pm_ops, omap_aes_suspend, omap_aes_resume);
13070635fb3aSMark A. Greer
1308537559a5SDmitry Kasatkin static struct platform_driver omap_aes_driver = {
1309537559a5SDmitry Kasatkin .probe = omap_aes_probe,
1310537559a5SDmitry Kasatkin .remove = omap_aes_remove,
1311537559a5SDmitry Kasatkin .driver = {
1312537559a5SDmitry Kasatkin .name = "omap-aes",
13130635fb3aSMark A. Greer .pm = &omap_aes_pm_ops,
1314bc69d124SMark A. Greer .of_match_table = omap_aes_of_match,
1315537559a5SDmitry Kasatkin },
1316537559a5SDmitry Kasatkin };
1317537559a5SDmitry Kasatkin
131894e51df9SSachin Kamat module_platform_driver(omap_aes_driver);
1319537559a5SDmitry Kasatkin
1320537559a5SDmitry Kasatkin MODULE_DESCRIPTION("OMAP AES hw acceleration support.");
1321537559a5SDmitry Kasatkin MODULE_LICENSE("GPL v2");
1322537559a5SDmitry Kasatkin MODULE_AUTHOR("Dmitry Kasatkin");
1323537559a5SDmitry Kasatkin
1324