1bd3c7b5cSNicolas Royer /* 2bd3c7b5cSNicolas Royer * Cryptographic API. 3bd3c7b5cSNicolas Royer * 4bd3c7b5cSNicolas Royer * Support for ATMEL AES HW acceleration. 5bd3c7b5cSNicolas Royer * 6bd3c7b5cSNicolas Royer * Copyright (c) 2012 Eukréa Electromatique - ATMEL 7bd3c7b5cSNicolas Royer * Author: Nicolas Royer <nicolas@eukrea.com> 8bd3c7b5cSNicolas Royer * 9bd3c7b5cSNicolas Royer * This program is free software; you can redistribute it and/or modify 10bd3c7b5cSNicolas Royer * it under the terms of the GNU General Public License version 2 as published 11bd3c7b5cSNicolas Royer * by the Free Software Foundation. 12bd3c7b5cSNicolas Royer * 13bd3c7b5cSNicolas Royer * Some ideas are from omap-aes.c driver. 14bd3c7b5cSNicolas Royer */ 15bd3c7b5cSNicolas Royer 16bd3c7b5cSNicolas Royer 17bd3c7b5cSNicolas Royer #include <linux/kernel.h> 18bd3c7b5cSNicolas Royer #include <linux/module.h> 19bd3c7b5cSNicolas Royer #include <linux/slab.h> 20bd3c7b5cSNicolas Royer #include <linux/err.h> 21bd3c7b5cSNicolas Royer #include <linux/clk.h> 22bd3c7b5cSNicolas Royer #include <linux/io.h> 23bd3c7b5cSNicolas Royer #include <linux/hw_random.h> 24bd3c7b5cSNicolas Royer #include <linux/platform_device.h> 25bd3c7b5cSNicolas Royer 26bd3c7b5cSNicolas Royer #include <linux/device.h> 27bd3c7b5cSNicolas Royer #include <linux/init.h> 28bd3c7b5cSNicolas Royer #include <linux/errno.h> 29bd3c7b5cSNicolas Royer #include <linux/interrupt.h> 30bd3c7b5cSNicolas Royer #include <linux/irq.h> 31bd3c7b5cSNicolas Royer #include <linux/scatterlist.h> 32bd3c7b5cSNicolas Royer #include <linux/dma-mapping.h> 33be943c7dSNicolas Ferre #include <linux/of_device.h> 34bd3c7b5cSNicolas Royer #include <linux/delay.h> 35bd3c7b5cSNicolas Royer #include <linux/crypto.h> 36bd3c7b5cSNicolas Royer #include <linux/cryptohash.h> 37bd3c7b5cSNicolas Royer #include <crypto/scatterwalk.h> 38bd3c7b5cSNicolas Royer #include <crypto/algapi.h> 39bd3c7b5cSNicolas Royer #include <crypto/aes.h> 40bd3c7b5cSNicolas Royer #include <crypto/hash.h> 41bd3c7b5cSNicolas Royer #include <crypto/internal/hash.h> 42cadc4ab8SNicolas Royer #include <linux/platform_data/crypto-atmel.h> 43be943c7dSNicolas Ferre #include <dt-bindings/dma/at91.h> 44bd3c7b5cSNicolas Royer #include "atmel-aes-regs.h" 45bd3c7b5cSNicolas Royer 46bd3c7b5cSNicolas Royer #define CFB8_BLOCK_SIZE 1 47bd3c7b5cSNicolas Royer #define CFB16_BLOCK_SIZE 2 48bd3c7b5cSNicolas Royer #define CFB32_BLOCK_SIZE 4 49bd3c7b5cSNicolas Royer #define CFB64_BLOCK_SIZE 8 50bd3c7b5cSNicolas Royer 51bd3c7b5cSNicolas Royer /* AES flags */ 52cadc4ab8SNicolas Royer #define AES_FLAGS_MODE_MASK 0x03ff 53bd3c7b5cSNicolas Royer #define AES_FLAGS_ENCRYPT BIT(0) 54bd3c7b5cSNicolas Royer #define AES_FLAGS_CBC BIT(1) 55bd3c7b5cSNicolas Royer #define AES_FLAGS_CFB BIT(2) 56bd3c7b5cSNicolas Royer #define AES_FLAGS_CFB8 BIT(3) 57bd3c7b5cSNicolas Royer #define AES_FLAGS_CFB16 BIT(4) 58bd3c7b5cSNicolas Royer #define AES_FLAGS_CFB32 BIT(5) 59bd3c7b5cSNicolas Royer #define AES_FLAGS_CFB64 BIT(6) 60cadc4ab8SNicolas Royer #define AES_FLAGS_CFB128 BIT(7) 61cadc4ab8SNicolas Royer #define AES_FLAGS_OFB BIT(8) 62cadc4ab8SNicolas Royer #define AES_FLAGS_CTR BIT(9) 63bd3c7b5cSNicolas Royer 64bd3c7b5cSNicolas Royer #define AES_FLAGS_INIT BIT(16) 65bd3c7b5cSNicolas Royer #define AES_FLAGS_DMA BIT(17) 66bd3c7b5cSNicolas Royer #define AES_FLAGS_BUSY BIT(18) 67cadc4ab8SNicolas Royer #define AES_FLAGS_FAST BIT(19) 68bd3c7b5cSNicolas Royer 69cadc4ab8SNicolas Royer #define ATMEL_AES_QUEUE_LENGTH 50 70bd3c7b5cSNicolas Royer 71bd3c7b5cSNicolas Royer #define ATMEL_AES_DMA_THRESHOLD 16 72bd3c7b5cSNicolas Royer 73bd3c7b5cSNicolas Royer 74cadc4ab8SNicolas Royer struct atmel_aes_caps { 75cadc4ab8SNicolas Royer bool has_dualbuff; 76cadc4ab8SNicolas Royer bool has_cfb64; 77cadc4ab8SNicolas Royer u32 max_burst_size; 78cadc4ab8SNicolas Royer }; 79cadc4ab8SNicolas Royer 80bd3c7b5cSNicolas Royer struct atmel_aes_dev; 81bd3c7b5cSNicolas Royer 82bd3c7b5cSNicolas Royer struct atmel_aes_ctx { 83bd3c7b5cSNicolas Royer struct atmel_aes_dev *dd; 84bd3c7b5cSNicolas Royer 85bd3c7b5cSNicolas Royer int keylen; 86bd3c7b5cSNicolas Royer u32 key[AES_KEYSIZE_256 / sizeof(u32)]; 87cadc4ab8SNicolas Royer 88cadc4ab8SNicolas Royer u16 block_size; 89bd3c7b5cSNicolas Royer }; 90bd3c7b5cSNicolas Royer 91bd3c7b5cSNicolas Royer struct atmel_aes_reqctx { 92bd3c7b5cSNicolas Royer unsigned long mode; 93bd3c7b5cSNicolas Royer }; 94bd3c7b5cSNicolas Royer 95bd3c7b5cSNicolas Royer struct atmel_aes_dma { 96bd3c7b5cSNicolas Royer struct dma_chan *chan; 97bd3c7b5cSNicolas Royer struct dma_slave_config dma_conf; 98bd3c7b5cSNicolas Royer }; 99bd3c7b5cSNicolas Royer 100bd3c7b5cSNicolas Royer struct atmel_aes_dev { 101bd3c7b5cSNicolas Royer struct list_head list; 102bd3c7b5cSNicolas Royer unsigned long phys_base; 103bd3c7b5cSNicolas Royer void __iomem *io_base; 104bd3c7b5cSNicolas Royer 105bd3c7b5cSNicolas Royer struct atmel_aes_ctx *ctx; 106bd3c7b5cSNicolas Royer struct device *dev; 107bd3c7b5cSNicolas Royer struct clk *iclk; 108bd3c7b5cSNicolas Royer int irq; 109bd3c7b5cSNicolas Royer 110bd3c7b5cSNicolas Royer unsigned long flags; 111bd3c7b5cSNicolas Royer int err; 112bd3c7b5cSNicolas Royer 113bd3c7b5cSNicolas Royer spinlock_t lock; 114bd3c7b5cSNicolas Royer struct crypto_queue queue; 115bd3c7b5cSNicolas Royer 116bd3c7b5cSNicolas Royer struct tasklet_struct done_task; 117bd3c7b5cSNicolas Royer struct tasklet_struct queue_task; 118bd3c7b5cSNicolas Royer 119bd3c7b5cSNicolas Royer struct ablkcipher_request *req; 120bd3c7b5cSNicolas Royer size_t total; 121bd3c7b5cSNicolas Royer 122bd3c7b5cSNicolas Royer struct scatterlist *in_sg; 123bd3c7b5cSNicolas Royer unsigned int nb_in_sg; 124cadc4ab8SNicolas Royer size_t in_offset; 125bd3c7b5cSNicolas Royer struct scatterlist *out_sg; 126bd3c7b5cSNicolas Royer unsigned int nb_out_sg; 127cadc4ab8SNicolas Royer size_t out_offset; 128bd3c7b5cSNicolas Royer 129bd3c7b5cSNicolas Royer size_t bufcnt; 130cadc4ab8SNicolas Royer size_t buflen; 131cadc4ab8SNicolas Royer size_t dma_size; 132bd3c7b5cSNicolas Royer 133cadc4ab8SNicolas Royer void *buf_in; 134bd3c7b5cSNicolas Royer int dma_in; 135cadc4ab8SNicolas Royer dma_addr_t dma_addr_in; 136bd3c7b5cSNicolas Royer struct atmel_aes_dma dma_lch_in; 137bd3c7b5cSNicolas Royer 138cadc4ab8SNicolas Royer void *buf_out; 139bd3c7b5cSNicolas Royer int dma_out; 140cadc4ab8SNicolas Royer dma_addr_t dma_addr_out; 141bd3c7b5cSNicolas Royer struct atmel_aes_dma dma_lch_out; 142bd3c7b5cSNicolas Royer 143cadc4ab8SNicolas Royer struct atmel_aes_caps caps; 144cadc4ab8SNicolas Royer 145bd3c7b5cSNicolas Royer u32 hw_version; 146bd3c7b5cSNicolas Royer }; 147bd3c7b5cSNicolas Royer 148bd3c7b5cSNicolas Royer struct atmel_aes_drv { 149bd3c7b5cSNicolas Royer struct list_head dev_list; 150bd3c7b5cSNicolas Royer spinlock_t lock; 151bd3c7b5cSNicolas Royer }; 152bd3c7b5cSNicolas Royer 153bd3c7b5cSNicolas Royer static struct atmel_aes_drv atmel_aes = { 154bd3c7b5cSNicolas Royer .dev_list = LIST_HEAD_INIT(atmel_aes.dev_list), 155bd3c7b5cSNicolas Royer .lock = __SPIN_LOCK_UNLOCKED(atmel_aes.lock), 156bd3c7b5cSNicolas Royer }; 157bd3c7b5cSNicolas Royer 158bd3c7b5cSNicolas Royer static int atmel_aes_sg_length(struct ablkcipher_request *req, 159bd3c7b5cSNicolas Royer struct scatterlist *sg) 160bd3c7b5cSNicolas Royer { 161bd3c7b5cSNicolas Royer unsigned int total = req->nbytes; 162bd3c7b5cSNicolas Royer int sg_nb; 163bd3c7b5cSNicolas Royer unsigned int len; 164bd3c7b5cSNicolas Royer struct scatterlist *sg_list; 165bd3c7b5cSNicolas Royer 166bd3c7b5cSNicolas Royer sg_nb = 0; 167bd3c7b5cSNicolas Royer sg_list = sg; 168bd3c7b5cSNicolas Royer total = req->nbytes; 169bd3c7b5cSNicolas Royer 170bd3c7b5cSNicolas Royer while (total) { 171bd3c7b5cSNicolas Royer len = min(sg_list->length, total); 172bd3c7b5cSNicolas Royer 173bd3c7b5cSNicolas Royer sg_nb++; 174bd3c7b5cSNicolas Royer total -= len; 175bd3c7b5cSNicolas Royer 176bd3c7b5cSNicolas Royer sg_list = sg_next(sg_list); 177bd3c7b5cSNicolas Royer if (!sg_list) 178bd3c7b5cSNicolas Royer total = 0; 179bd3c7b5cSNicolas Royer } 180bd3c7b5cSNicolas Royer 181bd3c7b5cSNicolas Royer return sg_nb; 182bd3c7b5cSNicolas Royer } 183bd3c7b5cSNicolas Royer 184cadc4ab8SNicolas Royer static int atmel_aes_sg_copy(struct scatterlist **sg, size_t *offset, 185cadc4ab8SNicolas Royer void *buf, size_t buflen, size_t total, int out) 186cadc4ab8SNicolas Royer { 187cadc4ab8SNicolas Royer unsigned int count, off = 0; 188cadc4ab8SNicolas Royer 189cadc4ab8SNicolas Royer while (buflen && total) { 190cadc4ab8SNicolas Royer count = min((*sg)->length - *offset, total); 191cadc4ab8SNicolas Royer count = min(count, buflen); 192cadc4ab8SNicolas Royer 193cadc4ab8SNicolas Royer if (!count) 194cadc4ab8SNicolas Royer return off; 195cadc4ab8SNicolas Royer 196cadc4ab8SNicolas Royer scatterwalk_map_and_copy(buf + off, *sg, *offset, count, out); 197cadc4ab8SNicolas Royer 198cadc4ab8SNicolas Royer off += count; 199cadc4ab8SNicolas Royer buflen -= count; 200cadc4ab8SNicolas Royer *offset += count; 201cadc4ab8SNicolas Royer total -= count; 202cadc4ab8SNicolas Royer 203cadc4ab8SNicolas Royer if (*offset == (*sg)->length) { 204cadc4ab8SNicolas Royer *sg = sg_next(*sg); 205cadc4ab8SNicolas Royer if (*sg) 206cadc4ab8SNicolas Royer *offset = 0; 207cadc4ab8SNicolas Royer else 208cadc4ab8SNicolas Royer total = 0; 209cadc4ab8SNicolas Royer } 210cadc4ab8SNicolas Royer } 211cadc4ab8SNicolas Royer 212cadc4ab8SNicolas Royer return off; 213cadc4ab8SNicolas Royer } 214cadc4ab8SNicolas Royer 215bd3c7b5cSNicolas Royer static inline u32 atmel_aes_read(struct atmel_aes_dev *dd, u32 offset) 216bd3c7b5cSNicolas Royer { 217bd3c7b5cSNicolas Royer return readl_relaxed(dd->io_base + offset); 218bd3c7b5cSNicolas Royer } 219bd3c7b5cSNicolas Royer 220bd3c7b5cSNicolas Royer static inline void atmel_aes_write(struct atmel_aes_dev *dd, 221bd3c7b5cSNicolas Royer u32 offset, u32 value) 222bd3c7b5cSNicolas Royer { 223bd3c7b5cSNicolas Royer writel_relaxed(value, dd->io_base + offset); 224bd3c7b5cSNicolas Royer } 225bd3c7b5cSNicolas Royer 226bd3c7b5cSNicolas Royer static void atmel_aes_read_n(struct atmel_aes_dev *dd, u32 offset, 227bd3c7b5cSNicolas Royer u32 *value, int count) 228bd3c7b5cSNicolas Royer { 229bd3c7b5cSNicolas Royer for (; count--; value++, offset += 4) 230bd3c7b5cSNicolas Royer *value = atmel_aes_read(dd, offset); 231bd3c7b5cSNicolas Royer } 232bd3c7b5cSNicolas Royer 233bd3c7b5cSNicolas Royer static void atmel_aes_write_n(struct atmel_aes_dev *dd, u32 offset, 234bd3c7b5cSNicolas Royer u32 *value, int count) 235bd3c7b5cSNicolas Royer { 236bd3c7b5cSNicolas Royer for (; count--; value++, offset += 4) 237bd3c7b5cSNicolas Royer atmel_aes_write(dd, offset, *value); 238bd3c7b5cSNicolas Royer } 239bd3c7b5cSNicolas Royer 240bd3c7b5cSNicolas Royer static struct atmel_aes_dev *atmel_aes_find_dev(struct atmel_aes_ctx *ctx) 241bd3c7b5cSNicolas Royer { 242bd3c7b5cSNicolas Royer struct atmel_aes_dev *aes_dd = NULL; 243bd3c7b5cSNicolas Royer struct atmel_aes_dev *tmp; 244bd3c7b5cSNicolas Royer 245bd3c7b5cSNicolas Royer spin_lock_bh(&atmel_aes.lock); 246bd3c7b5cSNicolas Royer if (!ctx->dd) { 247bd3c7b5cSNicolas Royer list_for_each_entry(tmp, &atmel_aes.dev_list, list) { 248bd3c7b5cSNicolas Royer aes_dd = tmp; 249bd3c7b5cSNicolas Royer break; 250bd3c7b5cSNicolas Royer } 251bd3c7b5cSNicolas Royer ctx->dd = aes_dd; 252bd3c7b5cSNicolas Royer } else { 253bd3c7b5cSNicolas Royer aes_dd = ctx->dd; 254bd3c7b5cSNicolas Royer } 255bd3c7b5cSNicolas Royer 256bd3c7b5cSNicolas Royer spin_unlock_bh(&atmel_aes.lock); 257bd3c7b5cSNicolas Royer 258bd3c7b5cSNicolas Royer return aes_dd; 259bd3c7b5cSNicolas Royer } 260bd3c7b5cSNicolas Royer 261bd3c7b5cSNicolas Royer static int atmel_aes_hw_init(struct atmel_aes_dev *dd) 262bd3c7b5cSNicolas Royer { 263bd3c7b5cSNicolas Royer clk_prepare_enable(dd->iclk); 264bd3c7b5cSNicolas Royer 265bd3c7b5cSNicolas Royer if (!(dd->flags & AES_FLAGS_INIT)) { 266bd3c7b5cSNicolas Royer atmel_aes_write(dd, AES_CR, AES_CR_SWRST); 267cadc4ab8SNicolas Royer atmel_aes_write(dd, AES_MR, 0xE << AES_MR_CKEY_OFFSET); 268bd3c7b5cSNicolas Royer dd->flags |= AES_FLAGS_INIT; 269bd3c7b5cSNicolas Royer dd->err = 0; 270bd3c7b5cSNicolas Royer } 271bd3c7b5cSNicolas Royer 272bd3c7b5cSNicolas Royer return 0; 273bd3c7b5cSNicolas Royer } 274bd3c7b5cSNicolas Royer 275cadc4ab8SNicolas Royer static inline unsigned int atmel_aes_get_version(struct atmel_aes_dev *dd) 276cadc4ab8SNicolas Royer { 277cadc4ab8SNicolas Royer return atmel_aes_read(dd, AES_HW_VERSION) & 0x00000fff; 278cadc4ab8SNicolas Royer } 279cadc4ab8SNicolas Royer 280bd3c7b5cSNicolas Royer static void atmel_aes_hw_version_init(struct atmel_aes_dev *dd) 281bd3c7b5cSNicolas Royer { 282bd3c7b5cSNicolas Royer atmel_aes_hw_init(dd); 283bd3c7b5cSNicolas Royer 284cadc4ab8SNicolas Royer dd->hw_version = atmel_aes_get_version(dd); 285cadc4ab8SNicolas Royer 286cadc4ab8SNicolas Royer dev_info(dd->dev, 287cadc4ab8SNicolas Royer "version: 0x%x\n", dd->hw_version); 288bd3c7b5cSNicolas Royer 289bd3c7b5cSNicolas Royer clk_disable_unprepare(dd->iclk); 290bd3c7b5cSNicolas Royer } 291bd3c7b5cSNicolas Royer 292bd3c7b5cSNicolas Royer static void atmel_aes_finish_req(struct atmel_aes_dev *dd, int err) 293bd3c7b5cSNicolas Royer { 294bd3c7b5cSNicolas Royer struct ablkcipher_request *req = dd->req; 295bd3c7b5cSNicolas Royer 296bd3c7b5cSNicolas Royer clk_disable_unprepare(dd->iclk); 297bd3c7b5cSNicolas Royer dd->flags &= ~AES_FLAGS_BUSY; 298bd3c7b5cSNicolas Royer 299bd3c7b5cSNicolas Royer req->base.complete(&req->base, err); 300bd3c7b5cSNicolas Royer } 301bd3c7b5cSNicolas Royer 302bd3c7b5cSNicolas Royer static void atmel_aes_dma_callback(void *data) 303bd3c7b5cSNicolas Royer { 304bd3c7b5cSNicolas Royer struct atmel_aes_dev *dd = data; 305bd3c7b5cSNicolas Royer 306bd3c7b5cSNicolas Royer /* dma_lch_out - completed */ 307bd3c7b5cSNicolas Royer tasklet_schedule(&dd->done_task); 308bd3c7b5cSNicolas Royer } 309bd3c7b5cSNicolas Royer 310cadc4ab8SNicolas Royer static int atmel_aes_crypt_dma(struct atmel_aes_dev *dd, 311cadc4ab8SNicolas Royer dma_addr_t dma_addr_in, dma_addr_t dma_addr_out, int length) 312bd3c7b5cSNicolas Royer { 313cadc4ab8SNicolas Royer struct scatterlist sg[2]; 314bd3c7b5cSNicolas Royer struct dma_async_tx_descriptor *in_desc, *out_desc; 315bd3c7b5cSNicolas Royer 316cadc4ab8SNicolas Royer dd->dma_size = length; 317bd3c7b5cSNicolas Royer 318cadc4ab8SNicolas Royer if (!(dd->flags & AES_FLAGS_FAST)) { 319cadc4ab8SNicolas Royer dma_sync_single_for_device(dd->dev, dma_addr_in, length, 320bd3c7b5cSNicolas Royer DMA_TO_DEVICE); 321cadc4ab8SNicolas Royer } 322bd3c7b5cSNicolas Royer 323cadc4ab8SNicolas Royer if (dd->flags & AES_FLAGS_CFB8) { 324cadc4ab8SNicolas Royer dd->dma_lch_in.dma_conf.dst_addr_width = 325cadc4ab8SNicolas Royer DMA_SLAVE_BUSWIDTH_1_BYTE; 326cadc4ab8SNicolas Royer dd->dma_lch_out.dma_conf.src_addr_width = 327cadc4ab8SNicolas Royer DMA_SLAVE_BUSWIDTH_1_BYTE; 328cadc4ab8SNicolas Royer } else if (dd->flags & AES_FLAGS_CFB16) { 329cadc4ab8SNicolas Royer dd->dma_lch_in.dma_conf.dst_addr_width = 330cadc4ab8SNicolas Royer DMA_SLAVE_BUSWIDTH_2_BYTES; 331cadc4ab8SNicolas Royer dd->dma_lch_out.dma_conf.src_addr_width = 332cadc4ab8SNicolas Royer DMA_SLAVE_BUSWIDTH_2_BYTES; 333cadc4ab8SNicolas Royer } else { 334cadc4ab8SNicolas Royer dd->dma_lch_in.dma_conf.dst_addr_width = 335cadc4ab8SNicolas Royer DMA_SLAVE_BUSWIDTH_4_BYTES; 336cadc4ab8SNicolas Royer dd->dma_lch_out.dma_conf.src_addr_width = 337cadc4ab8SNicolas Royer DMA_SLAVE_BUSWIDTH_4_BYTES; 338cadc4ab8SNicolas Royer } 339cadc4ab8SNicolas Royer 340cadc4ab8SNicolas Royer if (dd->flags & (AES_FLAGS_CFB8 | AES_FLAGS_CFB16 | 341cadc4ab8SNicolas Royer AES_FLAGS_CFB32 | AES_FLAGS_CFB64)) { 342cadc4ab8SNicolas Royer dd->dma_lch_in.dma_conf.src_maxburst = 1; 343cadc4ab8SNicolas Royer dd->dma_lch_in.dma_conf.dst_maxburst = 1; 344cadc4ab8SNicolas Royer dd->dma_lch_out.dma_conf.src_maxburst = 1; 345cadc4ab8SNicolas Royer dd->dma_lch_out.dma_conf.dst_maxburst = 1; 346cadc4ab8SNicolas Royer } else { 347cadc4ab8SNicolas Royer dd->dma_lch_in.dma_conf.src_maxburst = dd->caps.max_burst_size; 348cadc4ab8SNicolas Royer dd->dma_lch_in.dma_conf.dst_maxburst = dd->caps.max_burst_size; 349cadc4ab8SNicolas Royer dd->dma_lch_out.dma_conf.src_maxburst = dd->caps.max_burst_size; 350cadc4ab8SNicolas Royer dd->dma_lch_out.dma_conf.dst_maxburst = dd->caps.max_burst_size; 351cadc4ab8SNicolas Royer } 352cadc4ab8SNicolas Royer 353cadc4ab8SNicolas Royer dmaengine_slave_config(dd->dma_lch_in.chan, &dd->dma_lch_in.dma_conf); 354cadc4ab8SNicolas Royer dmaengine_slave_config(dd->dma_lch_out.chan, &dd->dma_lch_out.dma_conf); 355cadc4ab8SNicolas Royer 356cadc4ab8SNicolas Royer dd->flags |= AES_FLAGS_DMA; 357cadc4ab8SNicolas Royer 358cadc4ab8SNicolas Royer sg_init_table(&sg[0], 1); 359cadc4ab8SNicolas Royer sg_dma_address(&sg[0]) = dma_addr_in; 360cadc4ab8SNicolas Royer sg_dma_len(&sg[0]) = length; 361cadc4ab8SNicolas Royer 362cadc4ab8SNicolas Royer sg_init_table(&sg[1], 1); 363cadc4ab8SNicolas Royer sg_dma_address(&sg[1]) = dma_addr_out; 364cadc4ab8SNicolas Royer sg_dma_len(&sg[1]) = length; 365cadc4ab8SNicolas Royer 366cadc4ab8SNicolas Royer in_desc = dmaengine_prep_slave_sg(dd->dma_lch_in.chan, &sg[0], 367cadc4ab8SNicolas Royer 1, DMA_MEM_TO_DEV, 368bd3c7b5cSNicolas Royer DMA_PREP_INTERRUPT | DMA_CTRL_ACK); 369bd3c7b5cSNicolas Royer if (!in_desc) 370cadc4ab8SNicolas Royer return -EINVAL; 371bd3c7b5cSNicolas Royer 372cadc4ab8SNicolas Royer out_desc = dmaengine_prep_slave_sg(dd->dma_lch_out.chan, &sg[1], 373cadc4ab8SNicolas Royer 1, DMA_DEV_TO_MEM, 374bd3c7b5cSNicolas Royer DMA_PREP_INTERRUPT | DMA_CTRL_ACK); 375bd3c7b5cSNicolas Royer if (!out_desc) 376cadc4ab8SNicolas Royer return -EINVAL; 377bd3c7b5cSNicolas Royer 378bd3c7b5cSNicolas Royer out_desc->callback = atmel_aes_dma_callback; 379bd3c7b5cSNicolas Royer out_desc->callback_param = dd; 380bd3c7b5cSNicolas Royer 381bd3c7b5cSNicolas Royer dmaengine_submit(out_desc); 382bd3c7b5cSNicolas Royer dma_async_issue_pending(dd->dma_lch_out.chan); 383bd3c7b5cSNicolas Royer 384bd3c7b5cSNicolas Royer dmaengine_submit(in_desc); 385bd3c7b5cSNicolas Royer dma_async_issue_pending(dd->dma_lch_in.chan); 386bd3c7b5cSNicolas Royer 387bd3c7b5cSNicolas Royer return 0; 388bd3c7b5cSNicolas Royer } 389bd3c7b5cSNicolas Royer 390bd3c7b5cSNicolas Royer static int atmel_aes_crypt_cpu_start(struct atmel_aes_dev *dd) 391bd3c7b5cSNicolas Royer { 392bd3c7b5cSNicolas Royer dd->flags &= ~AES_FLAGS_DMA; 393bd3c7b5cSNicolas Royer 394bd3c7b5cSNicolas Royer /* use cache buffers */ 395bd3c7b5cSNicolas Royer dd->nb_in_sg = atmel_aes_sg_length(dd->req, dd->in_sg); 396bd3c7b5cSNicolas Royer if (!dd->nb_in_sg) 397bd3c7b5cSNicolas Royer return -EINVAL; 398bd3c7b5cSNicolas Royer 399bd3c7b5cSNicolas Royer dd->nb_out_sg = atmel_aes_sg_length(dd->req, dd->out_sg); 4007b5c253cSJulia Lawall if (!dd->nb_out_sg) 401bd3c7b5cSNicolas Royer return -EINVAL; 402bd3c7b5cSNicolas Royer 403bd3c7b5cSNicolas Royer dd->bufcnt = sg_copy_to_buffer(dd->in_sg, dd->nb_in_sg, 404bd3c7b5cSNicolas Royer dd->buf_in, dd->total); 405bd3c7b5cSNicolas Royer 406bd3c7b5cSNicolas Royer if (!dd->bufcnt) 407bd3c7b5cSNicolas Royer return -EINVAL; 408bd3c7b5cSNicolas Royer 409bd3c7b5cSNicolas Royer dd->total -= dd->bufcnt; 410bd3c7b5cSNicolas Royer 411bd3c7b5cSNicolas Royer atmel_aes_write(dd, AES_IER, AES_INT_DATARDY); 412bd3c7b5cSNicolas Royer atmel_aes_write_n(dd, AES_IDATAR(0), (u32 *) dd->buf_in, 413bd3c7b5cSNicolas Royer dd->bufcnt >> 2); 414bd3c7b5cSNicolas Royer 415bd3c7b5cSNicolas Royer return 0; 416bd3c7b5cSNicolas Royer } 417bd3c7b5cSNicolas Royer 418bd3c7b5cSNicolas Royer static int atmel_aes_crypt_dma_start(struct atmel_aes_dev *dd) 419bd3c7b5cSNicolas Royer { 420cadc4ab8SNicolas Royer int err, fast = 0, in, out; 421cadc4ab8SNicolas Royer size_t count; 422cadc4ab8SNicolas Royer dma_addr_t addr_in, addr_out; 423bd3c7b5cSNicolas Royer 424cadc4ab8SNicolas Royer if ((!dd->in_offset) && (!dd->out_offset)) { 425cadc4ab8SNicolas Royer /* check for alignment */ 426cadc4ab8SNicolas Royer in = IS_ALIGNED((u32)dd->in_sg->offset, sizeof(u32)) && 427cadc4ab8SNicolas Royer IS_ALIGNED(dd->in_sg->length, dd->ctx->block_size); 428cadc4ab8SNicolas Royer out = IS_ALIGNED((u32)dd->out_sg->offset, sizeof(u32)) && 429cadc4ab8SNicolas Royer IS_ALIGNED(dd->out_sg->length, dd->ctx->block_size); 430cadc4ab8SNicolas Royer fast = in && out; 431cadc4ab8SNicolas Royer 432cadc4ab8SNicolas Royer if (sg_dma_len(dd->in_sg) != sg_dma_len(dd->out_sg)) 433cadc4ab8SNicolas Royer fast = 0; 434bd3c7b5cSNicolas Royer } 435bd3c7b5cSNicolas Royer 436bd3c7b5cSNicolas Royer 437cadc4ab8SNicolas Royer if (fast) { 438cadc4ab8SNicolas Royer count = min(dd->total, sg_dma_len(dd->in_sg)); 439cadc4ab8SNicolas Royer count = min(count, sg_dma_len(dd->out_sg)); 440cadc4ab8SNicolas Royer 441cadc4ab8SNicolas Royer err = dma_map_sg(dd->dev, dd->in_sg, 1, DMA_TO_DEVICE); 442cadc4ab8SNicolas Royer if (!err) { 443cadc4ab8SNicolas Royer dev_err(dd->dev, "dma_map_sg() error\n"); 444cadc4ab8SNicolas Royer return -EINVAL; 445cadc4ab8SNicolas Royer } 446cadc4ab8SNicolas Royer 447cadc4ab8SNicolas Royer err = dma_map_sg(dd->dev, dd->out_sg, 1, 448cadc4ab8SNicolas Royer DMA_FROM_DEVICE); 449cadc4ab8SNicolas Royer if (!err) { 450cadc4ab8SNicolas Royer dev_err(dd->dev, "dma_map_sg() error\n"); 451cadc4ab8SNicolas Royer dma_unmap_sg(dd->dev, dd->in_sg, 1, 452cadc4ab8SNicolas Royer DMA_TO_DEVICE); 453cadc4ab8SNicolas Royer return -EINVAL; 454cadc4ab8SNicolas Royer } 455cadc4ab8SNicolas Royer 456cadc4ab8SNicolas Royer addr_in = sg_dma_address(dd->in_sg); 457cadc4ab8SNicolas Royer addr_out = sg_dma_address(dd->out_sg); 458cadc4ab8SNicolas Royer 459cadc4ab8SNicolas Royer dd->flags |= AES_FLAGS_FAST; 460cadc4ab8SNicolas Royer 461cadc4ab8SNicolas Royer } else { 462cadc4ab8SNicolas Royer /* use cache buffers */ 463cadc4ab8SNicolas Royer count = atmel_aes_sg_copy(&dd->in_sg, &dd->in_offset, 464cadc4ab8SNicolas Royer dd->buf_in, dd->buflen, dd->total, 0); 465cadc4ab8SNicolas Royer 466cadc4ab8SNicolas Royer addr_in = dd->dma_addr_in; 467cadc4ab8SNicolas Royer addr_out = dd->dma_addr_out; 468cadc4ab8SNicolas Royer 469cadc4ab8SNicolas Royer dd->flags &= ~AES_FLAGS_FAST; 470cadc4ab8SNicolas Royer } 471cadc4ab8SNicolas Royer 472cadc4ab8SNicolas Royer dd->total -= count; 473cadc4ab8SNicolas Royer 474cadc4ab8SNicolas Royer err = atmel_aes_crypt_dma(dd, addr_in, addr_out, count); 475cadc4ab8SNicolas Royer 476cadc4ab8SNicolas Royer if (err && (dd->flags & AES_FLAGS_FAST)) { 477cadc4ab8SNicolas Royer dma_unmap_sg(dd->dev, dd->in_sg, 1, DMA_TO_DEVICE); 478cadc4ab8SNicolas Royer dma_unmap_sg(dd->dev, dd->out_sg, 1, DMA_TO_DEVICE); 479cadc4ab8SNicolas Royer } 480bd3c7b5cSNicolas Royer 481bd3c7b5cSNicolas Royer return err; 482bd3c7b5cSNicolas Royer } 483bd3c7b5cSNicolas Royer 484bd3c7b5cSNicolas Royer static int atmel_aes_write_ctrl(struct atmel_aes_dev *dd) 485bd3c7b5cSNicolas Royer { 486bd3c7b5cSNicolas Royer int err; 487bd3c7b5cSNicolas Royer u32 valcr = 0, valmr = 0; 488bd3c7b5cSNicolas Royer 489bd3c7b5cSNicolas Royer err = atmel_aes_hw_init(dd); 490bd3c7b5cSNicolas Royer 491bd3c7b5cSNicolas Royer if (err) 492bd3c7b5cSNicolas Royer return err; 493bd3c7b5cSNicolas Royer 494bd3c7b5cSNicolas Royer /* MR register must be set before IV registers */ 495bd3c7b5cSNicolas Royer if (dd->ctx->keylen == AES_KEYSIZE_128) 496bd3c7b5cSNicolas Royer valmr |= AES_MR_KEYSIZE_128; 497bd3c7b5cSNicolas Royer else if (dd->ctx->keylen == AES_KEYSIZE_192) 498bd3c7b5cSNicolas Royer valmr |= AES_MR_KEYSIZE_192; 499bd3c7b5cSNicolas Royer else 500bd3c7b5cSNicolas Royer valmr |= AES_MR_KEYSIZE_256; 501bd3c7b5cSNicolas Royer 502bd3c7b5cSNicolas Royer if (dd->flags & AES_FLAGS_CBC) { 503bd3c7b5cSNicolas Royer valmr |= AES_MR_OPMOD_CBC; 504bd3c7b5cSNicolas Royer } else if (dd->flags & AES_FLAGS_CFB) { 505bd3c7b5cSNicolas Royer valmr |= AES_MR_OPMOD_CFB; 506bd3c7b5cSNicolas Royer if (dd->flags & AES_FLAGS_CFB8) 507bd3c7b5cSNicolas Royer valmr |= AES_MR_CFBS_8b; 508bd3c7b5cSNicolas Royer else if (dd->flags & AES_FLAGS_CFB16) 509bd3c7b5cSNicolas Royer valmr |= AES_MR_CFBS_16b; 510bd3c7b5cSNicolas Royer else if (dd->flags & AES_FLAGS_CFB32) 511bd3c7b5cSNicolas Royer valmr |= AES_MR_CFBS_32b; 512bd3c7b5cSNicolas Royer else if (dd->flags & AES_FLAGS_CFB64) 513bd3c7b5cSNicolas Royer valmr |= AES_MR_CFBS_64b; 514cadc4ab8SNicolas Royer else if (dd->flags & AES_FLAGS_CFB128) 515cadc4ab8SNicolas Royer valmr |= AES_MR_CFBS_128b; 516bd3c7b5cSNicolas Royer } else if (dd->flags & AES_FLAGS_OFB) { 517bd3c7b5cSNicolas Royer valmr |= AES_MR_OPMOD_OFB; 518bd3c7b5cSNicolas Royer } else if (dd->flags & AES_FLAGS_CTR) { 519bd3c7b5cSNicolas Royer valmr |= AES_MR_OPMOD_CTR; 520bd3c7b5cSNicolas Royer } else { 521bd3c7b5cSNicolas Royer valmr |= AES_MR_OPMOD_ECB; 522bd3c7b5cSNicolas Royer } 523bd3c7b5cSNicolas Royer 524bd3c7b5cSNicolas Royer if (dd->flags & AES_FLAGS_ENCRYPT) 525bd3c7b5cSNicolas Royer valmr |= AES_MR_CYPHER_ENC; 526bd3c7b5cSNicolas Royer 527bd3c7b5cSNicolas Royer if (dd->total > ATMEL_AES_DMA_THRESHOLD) { 528bd3c7b5cSNicolas Royer valmr |= AES_MR_SMOD_IDATAR0; 529cadc4ab8SNicolas Royer if (dd->caps.has_dualbuff) 530bd3c7b5cSNicolas Royer valmr |= AES_MR_DUALBUFF; 531bd3c7b5cSNicolas Royer } else { 532bd3c7b5cSNicolas Royer valmr |= AES_MR_SMOD_AUTO; 533bd3c7b5cSNicolas Royer } 534bd3c7b5cSNicolas Royer 535bd3c7b5cSNicolas Royer atmel_aes_write(dd, AES_CR, valcr); 536bd3c7b5cSNicolas Royer atmel_aes_write(dd, AES_MR, valmr); 537bd3c7b5cSNicolas Royer 538bd3c7b5cSNicolas Royer atmel_aes_write_n(dd, AES_KEYWR(0), dd->ctx->key, 539bd3c7b5cSNicolas Royer dd->ctx->keylen >> 2); 540bd3c7b5cSNicolas Royer 541bd3c7b5cSNicolas Royer if (((dd->flags & AES_FLAGS_CBC) || (dd->flags & AES_FLAGS_CFB) || 542bd3c7b5cSNicolas Royer (dd->flags & AES_FLAGS_OFB) || (dd->flags & AES_FLAGS_CTR)) && 543bd3c7b5cSNicolas Royer dd->req->info) { 544bd3c7b5cSNicolas Royer atmel_aes_write_n(dd, AES_IVR(0), dd->req->info, 4); 545bd3c7b5cSNicolas Royer } 546bd3c7b5cSNicolas Royer 547bd3c7b5cSNicolas Royer return 0; 548bd3c7b5cSNicolas Royer } 549bd3c7b5cSNicolas Royer 550bd3c7b5cSNicolas Royer static int atmel_aes_handle_queue(struct atmel_aes_dev *dd, 551bd3c7b5cSNicolas Royer struct ablkcipher_request *req) 552bd3c7b5cSNicolas Royer { 553bd3c7b5cSNicolas Royer struct crypto_async_request *async_req, *backlog; 554bd3c7b5cSNicolas Royer struct atmel_aes_ctx *ctx; 555bd3c7b5cSNicolas Royer struct atmel_aes_reqctx *rctx; 556bd3c7b5cSNicolas Royer unsigned long flags; 557bd3c7b5cSNicolas Royer int err, ret = 0; 558bd3c7b5cSNicolas Royer 559bd3c7b5cSNicolas Royer spin_lock_irqsave(&dd->lock, flags); 560bd3c7b5cSNicolas Royer if (req) 561bd3c7b5cSNicolas Royer ret = ablkcipher_enqueue_request(&dd->queue, req); 562bd3c7b5cSNicolas Royer if (dd->flags & AES_FLAGS_BUSY) { 563bd3c7b5cSNicolas Royer spin_unlock_irqrestore(&dd->lock, flags); 564bd3c7b5cSNicolas Royer return ret; 565bd3c7b5cSNicolas Royer } 566bd3c7b5cSNicolas Royer backlog = crypto_get_backlog(&dd->queue); 567bd3c7b5cSNicolas Royer async_req = crypto_dequeue_request(&dd->queue); 568bd3c7b5cSNicolas Royer if (async_req) 569bd3c7b5cSNicolas Royer dd->flags |= AES_FLAGS_BUSY; 570bd3c7b5cSNicolas Royer spin_unlock_irqrestore(&dd->lock, flags); 571bd3c7b5cSNicolas Royer 572bd3c7b5cSNicolas Royer if (!async_req) 573bd3c7b5cSNicolas Royer return ret; 574bd3c7b5cSNicolas Royer 575bd3c7b5cSNicolas Royer if (backlog) 576bd3c7b5cSNicolas Royer backlog->complete(backlog, -EINPROGRESS); 577bd3c7b5cSNicolas Royer 578bd3c7b5cSNicolas Royer req = ablkcipher_request_cast(async_req); 579bd3c7b5cSNicolas Royer 580bd3c7b5cSNicolas Royer /* assign new request to device */ 581bd3c7b5cSNicolas Royer dd->req = req; 582bd3c7b5cSNicolas Royer dd->total = req->nbytes; 583cadc4ab8SNicolas Royer dd->in_offset = 0; 584bd3c7b5cSNicolas Royer dd->in_sg = req->src; 585cadc4ab8SNicolas Royer dd->out_offset = 0; 586bd3c7b5cSNicolas Royer dd->out_sg = req->dst; 587bd3c7b5cSNicolas Royer 588bd3c7b5cSNicolas Royer rctx = ablkcipher_request_ctx(req); 589bd3c7b5cSNicolas Royer ctx = crypto_ablkcipher_ctx(crypto_ablkcipher_reqtfm(req)); 590bd3c7b5cSNicolas Royer rctx->mode &= AES_FLAGS_MODE_MASK; 591bd3c7b5cSNicolas Royer dd->flags = (dd->flags & ~AES_FLAGS_MODE_MASK) | rctx->mode; 592bd3c7b5cSNicolas Royer dd->ctx = ctx; 593bd3c7b5cSNicolas Royer ctx->dd = dd; 594bd3c7b5cSNicolas Royer 595bd3c7b5cSNicolas Royer err = atmel_aes_write_ctrl(dd); 596bd3c7b5cSNicolas Royer if (!err) { 597bd3c7b5cSNicolas Royer if (dd->total > ATMEL_AES_DMA_THRESHOLD) 598bd3c7b5cSNicolas Royer err = atmel_aes_crypt_dma_start(dd); 599bd3c7b5cSNicolas Royer else 600bd3c7b5cSNicolas Royer err = atmel_aes_crypt_cpu_start(dd); 601bd3c7b5cSNicolas Royer } 602bd3c7b5cSNicolas Royer if (err) { 603bd3c7b5cSNicolas Royer /* aes_task will not finish it, so do it here */ 604bd3c7b5cSNicolas Royer atmel_aes_finish_req(dd, err); 605bd3c7b5cSNicolas Royer tasklet_schedule(&dd->queue_task); 606bd3c7b5cSNicolas Royer } 607bd3c7b5cSNicolas Royer 608bd3c7b5cSNicolas Royer return ret; 609bd3c7b5cSNicolas Royer } 610bd3c7b5cSNicolas Royer 611bd3c7b5cSNicolas Royer static int atmel_aes_crypt_dma_stop(struct atmel_aes_dev *dd) 612bd3c7b5cSNicolas Royer { 613bd3c7b5cSNicolas Royer int err = -EINVAL; 614cadc4ab8SNicolas Royer size_t count; 615bd3c7b5cSNicolas Royer 616bd3c7b5cSNicolas Royer if (dd->flags & AES_FLAGS_DMA) { 617bd3c7b5cSNicolas Royer err = 0; 618cadc4ab8SNicolas Royer if (dd->flags & AES_FLAGS_FAST) { 619cadc4ab8SNicolas Royer dma_unmap_sg(dd->dev, dd->out_sg, 1, DMA_FROM_DEVICE); 620cadc4ab8SNicolas Royer dma_unmap_sg(dd->dev, dd->in_sg, 1, DMA_TO_DEVICE); 621cadc4ab8SNicolas Royer } else { 622cadc4ab8SNicolas Royer dma_sync_single_for_device(dd->dev, dd->dma_addr_out, 623cadc4ab8SNicolas Royer dd->dma_size, DMA_FROM_DEVICE); 624cadc4ab8SNicolas Royer 625cadc4ab8SNicolas Royer /* copy data */ 626cadc4ab8SNicolas Royer count = atmel_aes_sg_copy(&dd->out_sg, &dd->out_offset, 627cadc4ab8SNicolas Royer dd->buf_out, dd->buflen, dd->dma_size, 1); 628cadc4ab8SNicolas Royer if (count != dd->dma_size) { 629cadc4ab8SNicolas Royer err = -EINVAL; 630cadc4ab8SNicolas Royer pr_err("not all data converted: %u\n", count); 631cadc4ab8SNicolas Royer } 632cadc4ab8SNicolas Royer } 633bd3c7b5cSNicolas Royer } 634bd3c7b5cSNicolas Royer 635bd3c7b5cSNicolas Royer return err; 636bd3c7b5cSNicolas Royer } 637bd3c7b5cSNicolas Royer 638cadc4ab8SNicolas Royer 639cadc4ab8SNicolas Royer static int atmel_aes_buff_init(struct atmel_aes_dev *dd) 640cadc4ab8SNicolas Royer { 641cadc4ab8SNicolas Royer int err = -ENOMEM; 642cadc4ab8SNicolas Royer 643cadc4ab8SNicolas Royer dd->buf_in = (void *)__get_free_pages(GFP_KERNEL, 0); 644cadc4ab8SNicolas Royer dd->buf_out = (void *)__get_free_pages(GFP_KERNEL, 0); 645cadc4ab8SNicolas Royer dd->buflen = PAGE_SIZE; 646cadc4ab8SNicolas Royer dd->buflen &= ~(AES_BLOCK_SIZE - 1); 647cadc4ab8SNicolas Royer 648cadc4ab8SNicolas Royer if (!dd->buf_in || !dd->buf_out) { 649cadc4ab8SNicolas Royer dev_err(dd->dev, "unable to alloc pages.\n"); 650cadc4ab8SNicolas Royer goto err_alloc; 651cadc4ab8SNicolas Royer } 652cadc4ab8SNicolas Royer 653cadc4ab8SNicolas Royer /* MAP here */ 654cadc4ab8SNicolas Royer dd->dma_addr_in = dma_map_single(dd->dev, dd->buf_in, 655cadc4ab8SNicolas Royer dd->buflen, DMA_TO_DEVICE); 656cadc4ab8SNicolas Royer if (dma_mapping_error(dd->dev, dd->dma_addr_in)) { 657cadc4ab8SNicolas Royer dev_err(dd->dev, "dma %d bytes error\n", dd->buflen); 658cadc4ab8SNicolas Royer err = -EINVAL; 659cadc4ab8SNicolas Royer goto err_map_in; 660cadc4ab8SNicolas Royer } 661cadc4ab8SNicolas Royer 662cadc4ab8SNicolas Royer dd->dma_addr_out = dma_map_single(dd->dev, dd->buf_out, 663cadc4ab8SNicolas Royer dd->buflen, DMA_FROM_DEVICE); 664cadc4ab8SNicolas Royer if (dma_mapping_error(dd->dev, dd->dma_addr_out)) { 665cadc4ab8SNicolas Royer dev_err(dd->dev, "dma %d bytes error\n", dd->buflen); 666cadc4ab8SNicolas Royer err = -EINVAL; 667cadc4ab8SNicolas Royer goto err_map_out; 668cadc4ab8SNicolas Royer } 669cadc4ab8SNicolas Royer 670cadc4ab8SNicolas Royer return 0; 671cadc4ab8SNicolas Royer 672cadc4ab8SNicolas Royer err_map_out: 673cadc4ab8SNicolas Royer dma_unmap_single(dd->dev, dd->dma_addr_in, dd->buflen, 674cadc4ab8SNicolas Royer DMA_TO_DEVICE); 675cadc4ab8SNicolas Royer err_map_in: 676088f628cSChristophe Jaillet err_alloc: 677cadc4ab8SNicolas Royer free_page((unsigned long)dd->buf_out); 678cadc4ab8SNicolas Royer free_page((unsigned long)dd->buf_in); 679cadc4ab8SNicolas Royer if (err) 680cadc4ab8SNicolas Royer pr_err("error: %d\n", err); 681cadc4ab8SNicolas Royer return err; 682cadc4ab8SNicolas Royer } 683cadc4ab8SNicolas Royer 684cadc4ab8SNicolas Royer static void atmel_aes_buff_cleanup(struct atmel_aes_dev *dd) 685cadc4ab8SNicolas Royer { 686cadc4ab8SNicolas Royer dma_unmap_single(dd->dev, dd->dma_addr_out, dd->buflen, 687cadc4ab8SNicolas Royer DMA_FROM_DEVICE); 688cadc4ab8SNicolas Royer dma_unmap_single(dd->dev, dd->dma_addr_in, dd->buflen, 689cadc4ab8SNicolas Royer DMA_TO_DEVICE); 690cadc4ab8SNicolas Royer free_page((unsigned long)dd->buf_out); 691cadc4ab8SNicolas Royer free_page((unsigned long)dd->buf_in); 692cadc4ab8SNicolas Royer } 693cadc4ab8SNicolas Royer 694bd3c7b5cSNicolas Royer static int atmel_aes_crypt(struct ablkcipher_request *req, unsigned long mode) 695bd3c7b5cSNicolas Royer { 696bd3c7b5cSNicolas Royer struct atmel_aes_ctx *ctx = crypto_ablkcipher_ctx( 697bd3c7b5cSNicolas Royer crypto_ablkcipher_reqtfm(req)); 698bd3c7b5cSNicolas Royer struct atmel_aes_reqctx *rctx = ablkcipher_request_ctx(req); 699bd3c7b5cSNicolas Royer struct atmel_aes_dev *dd; 700bd3c7b5cSNicolas Royer 701cadc4ab8SNicolas Royer if (mode & AES_FLAGS_CFB8) { 702cadc4ab8SNicolas Royer if (!IS_ALIGNED(req->nbytes, CFB8_BLOCK_SIZE)) { 703cadc4ab8SNicolas Royer pr_err("request size is not exact amount of CFB8 blocks\n"); 704cadc4ab8SNicolas Royer return -EINVAL; 705cadc4ab8SNicolas Royer } 706cadc4ab8SNicolas Royer ctx->block_size = CFB8_BLOCK_SIZE; 707cadc4ab8SNicolas Royer } else if (mode & AES_FLAGS_CFB16) { 708cadc4ab8SNicolas Royer if (!IS_ALIGNED(req->nbytes, CFB16_BLOCK_SIZE)) { 709cadc4ab8SNicolas Royer pr_err("request size is not exact amount of CFB16 blocks\n"); 710cadc4ab8SNicolas Royer return -EINVAL; 711cadc4ab8SNicolas Royer } 712cadc4ab8SNicolas Royer ctx->block_size = CFB16_BLOCK_SIZE; 713cadc4ab8SNicolas Royer } else if (mode & AES_FLAGS_CFB32) { 714cadc4ab8SNicolas Royer if (!IS_ALIGNED(req->nbytes, CFB32_BLOCK_SIZE)) { 715cadc4ab8SNicolas Royer pr_err("request size is not exact amount of CFB32 blocks\n"); 716cadc4ab8SNicolas Royer return -EINVAL; 717cadc4ab8SNicolas Royer } 718cadc4ab8SNicolas Royer ctx->block_size = CFB32_BLOCK_SIZE; 7199f84951fSLeilei Zhao } else if (mode & AES_FLAGS_CFB64) { 7209f84951fSLeilei Zhao if (!IS_ALIGNED(req->nbytes, CFB64_BLOCK_SIZE)) { 7219f84951fSLeilei Zhao pr_err("request size is not exact amount of CFB64 blocks\n"); 7229f84951fSLeilei Zhao return -EINVAL; 7239f84951fSLeilei Zhao } 7249f84951fSLeilei Zhao ctx->block_size = CFB64_BLOCK_SIZE; 725cadc4ab8SNicolas Royer } else { 726bd3c7b5cSNicolas Royer if (!IS_ALIGNED(req->nbytes, AES_BLOCK_SIZE)) { 727bd3c7b5cSNicolas Royer pr_err("request size is not exact amount of AES blocks\n"); 728bd3c7b5cSNicolas Royer return -EINVAL; 729bd3c7b5cSNicolas Royer } 730cadc4ab8SNicolas Royer ctx->block_size = AES_BLOCK_SIZE; 731cadc4ab8SNicolas Royer } 732bd3c7b5cSNicolas Royer 733bd3c7b5cSNicolas Royer dd = atmel_aes_find_dev(ctx); 734bd3c7b5cSNicolas Royer if (!dd) 735bd3c7b5cSNicolas Royer return -ENODEV; 736bd3c7b5cSNicolas Royer 737bd3c7b5cSNicolas Royer rctx->mode = mode; 738bd3c7b5cSNicolas Royer 739bd3c7b5cSNicolas Royer return atmel_aes_handle_queue(dd, req); 740bd3c7b5cSNicolas Royer } 741bd3c7b5cSNicolas Royer 742bd3c7b5cSNicolas Royer static bool atmel_aes_filter(struct dma_chan *chan, void *slave) 743bd3c7b5cSNicolas Royer { 744bd3c7b5cSNicolas Royer struct at_dma_slave *sl = slave; 745bd3c7b5cSNicolas Royer 746bd3c7b5cSNicolas Royer if (sl && sl->dma_dev == chan->device->dev) { 747bd3c7b5cSNicolas Royer chan->private = sl; 748bd3c7b5cSNicolas Royer return true; 749bd3c7b5cSNicolas Royer } else { 750bd3c7b5cSNicolas Royer return false; 751bd3c7b5cSNicolas Royer } 752bd3c7b5cSNicolas Royer } 753bd3c7b5cSNicolas Royer 754cadc4ab8SNicolas Royer static int atmel_aes_dma_init(struct atmel_aes_dev *dd, 755cadc4ab8SNicolas Royer struct crypto_platform_data *pdata) 756bd3c7b5cSNicolas Royer { 757bd3c7b5cSNicolas Royer int err = -ENOMEM; 758be943c7dSNicolas Ferre dma_cap_mask_t mask; 759bd3c7b5cSNicolas Royer 760be943c7dSNicolas Ferre dma_cap_zero(mask); 761be943c7dSNicolas Ferre dma_cap_set(DMA_SLAVE, mask); 762bd3c7b5cSNicolas Royer 763bd3c7b5cSNicolas Royer /* Try to grab 2 DMA channels */ 764be943c7dSNicolas Ferre dd->dma_lch_in.chan = dma_request_slave_channel_compat(mask, 765be943c7dSNicolas Ferre atmel_aes_filter, &pdata->dma_slave->rxdata, dd->dev, "tx"); 766bd3c7b5cSNicolas Royer if (!dd->dma_lch_in.chan) 767bd3c7b5cSNicolas Royer goto err_dma_in; 768bd3c7b5cSNicolas Royer 769bd3c7b5cSNicolas Royer dd->dma_lch_in.dma_conf.direction = DMA_MEM_TO_DEV; 770bd3c7b5cSNicolas Royer dd->dma_lch_in.dma_conf.dst_addr = dd->phys_base + 771bd3c7b5cSNicolas Royer AES_IDATAR(0); 772cadc4ab8SNicolas Royer dd->dma_lch_in.dma_conf.src_maxburst = dd->caps.max_burst_size; 773cadc4ab8SNicolas Royer dd->dma_lch_in.dma_conf.src_addr_width = 774cadc4ab8SNicolas Royer DMA_SLAVE_BUSWIDTH_4_BYTES; 775cadc4ab8SNicolas Royer dd->dma_lch_in.dma_conf.dst_maxburst = dd->caps.max_burst_size; 776cadc4ab8SNicolas Royer dd->dma_lch_in.dma_conf.dst_addr_width = 777cadc4ab8SNicolas Royer DMA_SLAVE_BUSWIDTH_4_BYTES; 778bd3c7b5cSNicolas Royer dd->dma_lch_in.dma_conf.device_fc = false; 779bd3c7b5cSNicolas Royer 780be943c7dSNicolas Ferre dd->dma_lch_out.chan = dma_request_slave_channel_compat(mask, 781be943c7dSNicolas Ferre atmel_aes_filter, &pdata->dma_slave->txdata, dd->dev, "rx"); 782bd3c7b5cSNicolas Royer if (!dd->dma_lch_out.chan) 783bd3c7b5cSNicolas Royer goto err_dma_out; 784bd3c7b5cSNicolas Royer 785bd3c7b5cSNicolas Royer dd->dma_lch_out.dma_conf.direction = DMA_DEV_TO_MEM; 786bd3c7b5cSNicolas Royer dd->dma_lch_out.dma_conf.src_addr = dd->phys_base + 787bd3c7b5cSNicolas Royer AES_ODATAR(0); 788cadc4ab8SNicolas Royer dd->dma_lch_out.dma_conf.src_maxburst = dd->caps.max_burst_size; 789cadc4ab8SNicolas Royer dd->dma_lch_out.dma_conf.src_addr_width = 790cadc4ab8SNicolas Royer DMA_SLAVE_BUSWIDTH_4_BYTES; 791cadc4ab8SNicolas Royer dd->dma_lch_out.dma_conf.dst_maxburst = dd->caps.max_burst_size; 792cadc4ab8SNicolas Royer dd->dma_lch_out.dma_conf.dst_addr_width = 793cadc4ab8SNicolas Royer DMA_SLAVE_BUSWIDTH_4_BYTES; 794bd3c7b5cSNicolas Royer dd->dma_lch_out.dma_conf.device_fc = false; 795bd3c7b5cSNicolas Royer 796bd3c7b5cSNicolas Royer return 0; 797bd3c7b5cSNicolas Royer 798bd3c7b5cSNicolas Royer err_dma_out: 799bd3c7b5cSNicolas Royer dma_release_channel(dd->dma_lch_in.chan); 800bd3c7b5cSNicolas Royer err_dma_in: 801be943c7dSNicolas Ferre dev_warn(dd->dev, "no DMA channel available\n"); 802bd3c7b5cSNicolas Royer return err; 803bd3c7b5cSNicolas Royer } 804bd3c7b5cSNicolas Royer 805bd3c7b5cSNicolas Royer static void atmel_aes_dma_cleanup(struct atmel_aes_dev *dd) 806bd3c7b5cSNicolas Royer { 807bd3c7b5cSNicolas Royer dma_release_channel(dd->dma_lch_in.chan); 808bd3c7b5cSNicolas Royer dma_release_channel(dd->dma_lch_out.chan); 809bd3c7b5cSNicolas Royer } 810bd3c7b5cSNicolas Royer 811bd3c7b5cSNicolas Royer static int atmel_aes_setkey(struct crypto_ablkcipher *tfm, const u8 *key, 812bd3c7b5cSNicolas Royer unsigned int keylen) 813bd3c7b5cSNicolas Royer { 814bd3c7b5cSNicolas Royer struct atmel_aes_ctx *ctx = crypto_ablkcipher_ctx(tfm); 815bd3c7b5cSNicolas Royer 816bd3c7b5cSNicolas Royer if (keylen != AES_KEYSIZE_128 && keylen != AES_KEYSIZE_192 && 817bd3c7b5cSNicolas Royer keylen != AES_KEYSIZE_256) { 818bd3c7b5cSNicolas Royer crypto_ablkcipher_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); 819bd3c7b5cSNicolas Royer return -EINVAL; 820bd3c7b5cSNicolas Royer } 821bd3c7b5cSNicolas Royer 822bd3c7b5cSNicolas Royer memcpy(ctx->key, key, keylen); 823bd3c7b5cSNicolas Royer ctx->keylen = keylen; 824bd3c7b5cSNicolas Royer 825bd3c7b5cSNicolas Royer return 0; 826bd3c7b5cSNicolas Royer } 827bd3c7b5cSNicolas Royer 828bd3c7b5cSNicolas Royer static int atmel_aes_ecb_encrypt(struct ablkcipher_request *req) 829bd3c7b5cSNicolas Royer { 830bd3c7b5cSNicolas Royer return atmel_aes_crypt(req, 831bd3c7b5cSNicolas Royer AES_FLAGS_ENCRYPT); 832bd3c7b5cSNicolas Royer } 833bd3c7b5cSNicolas Royer 834bd3c7b5cSNicolas Royer static int atmel_aes_ecb_decrypt(struct ablkcipher_request *req) 835bd3c7b5cSNicolas Royer { 836bd3c7b5cSNicolas Royer return atmel_aes_crypt(req, 837bd3c7b5cSNicolas Royer 0); 838bd3c7b5cSNicolas Royer } 839bd3c7b5cSNicolas Royer 840bd3c7b5cSNicolas Royer static int atmel_aes_cbc_encrypt(struct ablkcipher_request *req) 841bd3c7b5cSNicolas Royer { 842bd3c7b5cSNicolas Royer return atmel_aes_crypt(req, 843bd3c7b5cSNicolas Royer AES_FLAGS_ENCRYPT | AES_FLAGS_CBC); 844bd3c7b5cSNicolas Royer } 845bd3c7b5cSNicolas Royer 846bd3c7b5cSNicolas Royer static int atmel_aes_cbc_decrypt(struct ablkcipher_request *req) 847bd3c7b5cSNicolas Royer { 848bd3c7b5cSNicolas Royer return atmel_aes_crypt(req, 849bd3c7b5cSNicolas Royer AES_FLAGS_CBC); 850bd3c7b5cSNicolas Royer } 851bd3c7b5cSNicolas Royer 852bd3c7b5cSNicolas Royer static int atmel_aes_ofb_encrypt(struct ablkcipher_request *req) 853bd3c7b5cSNicolas Royer { 854bd3c7b5cSNicolas Royer return atmel_aes_crypt(req, 855bd3c7b5cSNicolas Royer AES_FLAGS_ENCRYPT | AES_FLAGS_OFB); 856bd3c7b5cSNicolas Royer } 857bd3c7b5cSNicolas Royer 858bd3c7b5cSNicolas Royer static int atmel_aes_ofb_decrypt(struct ablkcipher_request *req) 859bd3c7b5cSNicolas Royer { 860bd3c7b5cSNicolas Royer return atmel_aes_crypt(req, 861bd3c7b5cSNicolas Royer AES_FLAGS_OFB); 862bd3c7b5cSNicolas Royer } 863bd3c7b5cSNicolas Royer 864bd3c7b5cSNicolas Royer static int atmel_aes_cfb_encrypt(struct ablkcipher_request *req) 865bd3c7b5cSNicolas Royer { 866bd3c7b5cSNicolas Royer return atmel_aes_crypt(req, 867cadc4ab8SNicolas Royer AES_FLAGS_ENCRYPT | AES_FLAGS_CFB | AES_FLAGS_CFB128); 868bd3c7b5cSNicolas Royer } 869bd3c7b5cSNicolas Royer 870bd3c7b5cSNicolas Royer static int atmel_aes_cfb_decrypt(struct ablkcipher_request *req) 871bd3c7b5cSNicolas Royer { 872bd3c7b5cSNicolas Royer return atmel_aes_crypt(req, 873cadc4ab8SNicolas Royer AES_FLAGS_CFB | AES_FLAGS_CFB128); 874bd3c7b5cSNicolas Royer } 875bd3c7b5cSNicolas Royer 876bd3c7b5cSNicolas Royer static int atmel_aes_cfb64_encrypt(struct ablkcipher_request *req) 877bd3c7b5cSNicolas Royer { 878bd3c7b5cSNicolas Royer return atmel_aes_crypt(req, 879bd3c7b5cSNicolas Royer AES_FLAGS_ENCRYPT | AES_FLAGS_CFB | AES_FLAGS_CFB64); 880bd3c7b5cSNicolas Royer } 881bd3c7b5cSNicolas Royer 882bd3c7b5cSNicolas Royer static int atmel_aes_cfb64_decrypt(struct ablkcipher_request *req) 883bd3c7b5cSNicolas Royer { 884bd3c7b5cSNicolas Royer return atmel_aes_crypt(req, 885bd3c7b5cSNicolas Royer AES_FLAGS_CFB | AES_FLAGS_CFB64); 886bd3c7b5cSNicolas Royer } 887bd3c7b5cSNicolas Royer 888bd3c7b5cSNicolas Royer static int atmel_aes_cfb32_encrypt(struct ablkcipher_request *req) 889bd3c7b5cSNicolas Royer { 890bd3c7b5cSNicolas Royer return atmel_aes_crypt(req, 891bd3c7b5cSNicolas Royer AES_FLAGS_ENCRYPT | AES_FLAGS_CFB | AES_FLAGS_CFB32); 892bd3c7b5cSNicolas Royer } 893bd3c7b5cSNicolas Royer 894bd3c7b5cSNicolas Royer static int atmel_aes_cfb32_decrypt(struct ablkcipher_request *req) 895bd3c7b5cSNicolas Royer { 896bd3c7b5cSNicolas Royer return atmel_aes_crypt(req, 897bd3c7b5cSNicolas Royer AES_FLAGS_CFB | AES_FLAGS_CFB32); 898bd3c7b5cSNicolas Royer } 899bd3c7b5cSNicolas Royer 900bd3c7b5cSNicolas Royer static int atmel_aes_cfb16_encrypt(struct ablkcipher_request *req) 901bd3c7b5cSNicolas Royer { 902bd3c7b5cSNicolas Royer return atmel_aes_crypt(req, 903bd3c7b5cSNicolas Royer AES_FLAGS_ENCRYPT | AES_FLAGS_CFB | AES_FLAGS_CFB16); 904bd3c7b5cSNicolas Royer } 905bd3c7b5cSNicolas Royer 906bd3c7b5cSNicolas Royer static int atmel_aes_cfb16_decrypt(struct ablkcipher_request *req) 907bd3c7b5cSNicolas Royer { 908bd3c7b5cSNicolas Royer return atmel_aes_crypt(req, 909bd3c7b5cSNicolas Royer AES_FLAGS_CFB | AES_FLAGS_CFB16); 910bd3c7b5cSNicolas Royer } 911bd3c7b5cSNicolas Royer 912bd3c7b5cSNicolas Royer static int atmel_aes_cfb8_encrypt(struct ablkcipher_request *req) 913bd3c7b5cSNicolas Royer { 914bd3c7b5cSNicolas Royer return atmel_aes_crypt(req, 915bd3c7b5cSNicolas Royer AES_FLAGS_ENCRYPT | AES_FLAGS_CFB | AES_FLAGS_CFB8); 916bd3c7b5cSNicolas Royer } 917bd3c7b5cSNicolas Royer 918bd3c7b5cSNicolas Royer static int atmel_aes_cfb8_decrypt(struct ablkcipher_request *req) 919bd3c7b5cSNicolas Royer { 920bd3c7b5cSNicolas Royer return atmel_aes_crypt(req, 921bd3c7b5cSNicolas Royer AES_FLAGS_CFB | AES_FLAGS_CFB8); 922bd3c7b5cSNicolas Royer } 923bd3c7b5cSNicolas Royer 924bd3c7b5cSNicolas Royer static int atmel_aes_ctr_encrypt(struct ablkcipher_request *req) 925bd3c7b5cSNicolas Royer { 926bd3c7b5cSNicolas Royer return atmel_aes_crypt(req, 927bd3c7b5cSNicolas Royer AES_FLAGS_ENCRYPT | AES_FLAGS_CTR); 928bd3c7b5cSNicolas Royer } 929bd3c7b5cSNicolas Royer 930bd3c7b5cSNicolas Royer static int atmel_aes_ctr_decrypt(struct ablkcipher_request *req) 931bd3c7b5cSNicolas Royer { 932bd3c7b5cSNicolas Royer return atmel_aes_crypt(req, 933bd3c7b5cSNicolas Royer AES_FLAGS_CTR); 934bd3c7b5cSNicolas Royer } 935bd3c7b5cSNicolas Royer 936bd3c7b5cSNicolas Royer static int atmel_aes_cra_init(struct crypto_tfm *tfm) 937bd3c7b5cSNicolas Royer { 938bd3c7b5cSNicolas Royer tfm->crt_ablkcipher.reqsize = sizeof(struct atmel_aes_reqctx); 939bd3c7b5cSNicolas Royer 940bd3c7b5cSNicolas Royer return 0; 941bd3c7b5cSNicolas Royer } 942bd3c7b5cSNicolas Royer 943bd3c7b5cSNicolas Royer static void atmel_aes_cra_exit(struct crypto_tfm *tfm) 944bd3c7b5cSNicolas Royer { 945bd3c7b5cSNicolas Royer } 946bd3c7b5cSNicolas Royer 947bd3c7b5cSNicolas Royer static struct crypto_alg aes_algs[] = { 948bd3c7b5cSNicolas Royer { 949bd3c7b5cSNicolas Royer .cra_name = "ecb(aes)", 950bd3c7b5cSNicolas Royer .cra_driver_name = "atmel-ecb-aes", 951bd3c7b5cSNicolas Royer .cra_priority = 100, 952bd3c7b5cSNicolas Royer .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, 953bd3c7b5cSNicolas Royer .cra_blocksize = AES_BLOCK_SIZE, 954bd3c7b5cSNicolas Royer .cra_ctxsize = sizeof(struct atmel_aes_ctx), 955cadc4ab8SNicolas Royer .cra_alignmask = 0xf, 956bd3c7b5cSNicolas Royer .cra_type = &crypto_ablkcipher_type, 957bd3c7b5cSNicolas Royer .cra_module = THIS_MODULE, 958bd3c7b5cSNicolas Royer .cra_init = atmel_aes_cra_init, 959bd3c7b5cSNicolas Royer .cra_exit = atmel_aes_cra_exit, 960bd3c7b5cSNicolas Royer .cra_u.ablkcipher = { 961bd3c7b5cSNicolas Royer .min_keysize = AES_MIN_KEY_SIZE, 962bd3c7b5cSNicolas Royer .max_keysize = AES_MAX_KEY_SIZE, 963bd3c7b5cSNicolas Royer .setkey = atmel_aes_setkey, 964bd3c7b5cSNicolas Royer .encrypt = atmel_aes_ecb_encrypt, 965bd3c7b5cSNicolas Royer .decrypt = atmel_aes_ecb_decrypt, 966bd3c7b5cSNicolas Royer } 967bd3c7b5cSNicolas Royer }, 968bd3c7b5cSNicolas Royer { 969bd3c7b5cSNicolas Royer .cra_name = "cbc(aes)", 970bd3c7b5cSNicolas Royer .cra_driver_name = "atmel-cbc-aes", 971bd3c7b5cSNicolas Royer .cra_priority = 100, 972bd3c7b5cSNicolas Royer .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, 973bd3c7b5cSNicolas Royer .cra_blocksize = AES_BLOCK_SIZE, 974bd3c7b5cSNicolas Royer .cra_ctxsize = sizeof(struct atmel_aes_ctx), 975cadc4ab8SNicolas Royer .cra_alignmask = 0xf, 976bd3c7b5cSNicolas Royer .cra_type = &crypto_ablkcipher_type, 977bd3c7b5cSNicolas Royer .cra_module = THIS_MODULE, 978bd3c7b5cSNicolas Royer .cra_init = atmel_aes_cra_init, 979bd3c7b5cSNicolas Royer .cra_exit = atmel_aes_cra_exit, 980bd3c7b5cSNicolas Royer .cra_u.ablkcipher = { 981bd3c7b5cSNicolas Royer .min_keysize = AES_MIN_KEY_SIZE, 982bd3c7b5cSNicolas Royer .max_keysize = AES_MAX_KEY_SIZE, 983bd3c7b5cSNicolas Royer .ivsize = AES_BLOCK_SIZE, 984bd3c7b5cSNicolas Royer .setkey = atmel_aes_setkey, 985bd3c7b5cSNicolas Royer .encrypt = atmel_aes_cbc_encrypt, 986bd3c7b5cSNicolas Royer .decrypt = atmel_aes_cbc_decrypt, 987bd3c7b5cSNicolas Royer } 988bd3c7b5cSNicolas Royer }, 989bd3c7b5cSNicolas Royer { 990bd3c7b5cSNicolas Royer .cra_name = "ofb(aes)", 991bd3c7b5cSNicolas Royer .cra_driver_name = "atmel-ofb-aes", 992bd3c7b5cSNicolas Royer .cra_priority = 100, 993bd3c7b5cSNicolas Royer .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, 994bd3c7b5cSNicolas Royer .cra_blocksize = AES_BLOCK_SIZE, 995bd3c7b5cSNicolas Royer .cra_ctxsize = sizeof(struct atmel_aes_ctx), 996cadc4ab8SNicolas Royer .cra_alignmask = 0xf, 997bd3c7b5cSNicolas Royer .cra_type = &crypto_ablkcipher_type, 998bd3c7b5cSNicolas Royer .cra_module = THIS_MODULE, 999bd3c7b5cSNicolas Royer .cra_init = atmel_aes_cra_init, 1000bd3c7b5cSNicolas Royer .cra_exit = atmel_aes_cra_exit, 1001bd3c7b5cSNicolas Royer .cra_u.ablkcipher = { 1002bd3c7b5cSNicolas Royer .min_keysize = AES_MIN_KEY_SIZE, 1003bd3c7b5cSNicolas Royer .max_keysize = AES_MAX_KEY_SIZE, 1004bd3c7b5cSNicolas Royer .ivsize = AES_BLOCK_SIZE, 1005bd3c7b5cSNicolas Royer .setkey = atmel_aes_setkey, 1006bd3c7b5cSNicolas Royer .encrypt = atmel_aes_ofb_encrypt, 1007bd3c7b5cSNicolas Royer .decrypt = atmel_aes_ofb_decrypt, 1008bd3c7b5cSNicolas Royer } 1009bd3c7b5cSNicolas Royer }, 1010bd3c7b5cSNicolas Royer { 1011bd3c7b5cSNicolas Royer .cra_name = "cfb(aes)", 1012bd3c7b5cSNicolas Royer .cra_driver_name = "atmel-cfb-aes", 1013bd3c7b5cSNicolas Royer .cra_priority = 100, 1014bd3c7b5cSNicolas Royer .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, 1015bd3c7b5cSNicolas Royer .cra_blocksize = AES_BLOCK_SIZE, 1016bd3c7b5cSNicolas Royer .cra_ctxsize = sizeof(struct atmel_aes_ctx), 1017cadc4ab8SNicolas Royer .cra_alignmask = 0xf, 1018bd3c7b5cSNicolas Royer .cra_type = &crypto_ablkcipher_type, 1019bd3c7b5cSNicolas Royer .cra_module = THIS_MODULE, 1020bd3c7b5cSNicolas Royer .cra_init = atmel_aes_cra_init, 1021bd3c7b5cSNicolas Royer .cra_exit = atmel_aes_cra_exit, 1022bd3c7b5cSNicolas Royer .cra_u.ablkcipher = { 1023bd3c7b5cSNicolas Royer .min_keysize = AES_MIN_KEY_SIZE, 1024bd3c7b5cSNicolas Royer .max_keysize = AES_MAX_KEY_SIZE, 1025bd3c7b5cSNicolas Royer .ivsize = AES_BLOCK_SIZE, 1026bd3c7b5cSNicolas Royer .setkey = atmel_aes_setkey, 1027bd3c7b5cSNicolas Royer .encrypt = atmel_aes_cfb_encrypt, 1028bd3c7b5cSNicolas Royer .decrypt = atmel_aes_cfb_decrypt, 1029bd3c7b5cSNicolas Royer } 1030bd3c7b5cSNicolas Royer }, 1031bd3c7b5cSNicolas Royer { 1032bd3c7b5cSNicolas Royer .cra_name = "cfb32(aes)", 1033bd3c7b5cSNicolas Royer .cra_driver_name = "atmel-cfb32-aes", 1034bd3c7b5cSNicolas Royer .cra_priority = 100, 1035bd3c7b5cSNicolas Royer .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, 1036bd3c7b5cSNicolas Royer .cra_blocksize = CFB32_BLOCK_SIZE, 1037bd3c7b5cSNicolas Royer .cra_ctxsize = sizeof(struct atmel_aes_ctx), 1038cadc4ab8SNicolas Royer .cra_alignmask = 0x3, 1039bd3c7b5cSNicolas Royer .cra_type = &crypto_ablkcipher_type, 1040bd3c7b5cSNicolas Royer .cra_module = THIS_MODULE, 1041bd3c7b5cSNicolas Royer .cra_init = atmel_aes_cra_init, 1042bd3c7b5cSNicolas Royer .cra_exit = atmel_aes_cra_exit, 1043bd3c7b5cSNicolas Royer .cra_u.ablkcipher = { 1044bd3c7b5cSNicolas Royer .min_keysize = AES_MIN_KEY_SIZE, 1045bd3c7b5cSNicolas Royer .max_keysize = AES_MAX_KEY_SIZE, 1046bd3c7b5cSNicolas Royer .ivsize = AES_BLOCK_SIZE, 1047bd3c7b5cSNicolas Royer .setkey = atmel_aes_setkey, 1048bd3c7b5cSNicolas Royer .encrypt = atmel_aes_cfb32_encrypt, 1049bd3c7b5cSNicolas Royer .decrypt = atmel_aes_cfb32_decrypt, 1050bd3c7b5cSNicolas Royer } 1051bd3c7b5cSNicolas Royer }, 1052bd3c7b5cSNicolas Royer { 1053bd3c7b5cSNicolas Royer .cra_name = "cfb16(aes)", 1054bd3c7b5cSNicolas Royer .cra_driver_name = "atmel-cfb16-aes", 1055bd3c7b5cSNicolas Royer .cra_priority = 100, 1056bd3c7b5cSNicolas Royer .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, 1057bd3c7b5cSNicolas Royer .cra_blocksize = CFB16_BLOCK_SIZE, 1058bd3c7b5cSNicolas Royer .cra_ctxsize = sizeof(struct atmel_aes_ctx), 1059cadc4ab8SNicolas Royer .cra_alignmask = 0x1, 1060bd3c7b5cSNicolas Royer .cra_type = &crypto_ablkcipher_type, 1061bd3c7b5cSNicolas Royer .cra_module = THIS_MODULE, 1062bd3c7b5cSNicolas Royer .cra_init = atmel_aes_cra_init, 1063bd3c7b5cSNicolas Royer .cra_exit = atmel_aes_cra_exit, 1064bd3c7b5cSNicolas Royer .cra_u.ablkcipher = { 1065bd3c7b5cSNicolas Royer .min_keysize = AES_MIN_KEY_SIZE, 1066bd3c7b5cSNicolas Royer .max_keysize = AES_MAX_KEY_SIZE, 1067bd3c7b5cSNicolas Royer .ivsize = AES_BLOCK_SIZE, 1068bd3c7b5cSNicolas Royer .setkey = atmel_aes_setkey, 1069bd3c7b5cSNicolas Royer .encrypt = atmel_aes_cfb16_encrypt, 1070bd3c7b5cSNicolas Royer .decrypt = atmel_aes_cfb16_decrypt, 1071bd3c7b5cSNicolas Royer } 1072bd3c7b5cSNicolas Royer }, 1073bd3c7b5cSNicolas Royer { 1074bd3c7b5cSNicolas Royer .cra_name = "cfb8(aes)", 1075bd3c7b5cSNicolas Royer .cra_driver_name = "atmel-cfb8-aes", 1076bd3c7b5cSNicolas Royer .cra_priority = 100, 1077bd3c7b5cSNicolas Royer .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, 1078e5d8c961SLeilei Zhao .cra_blocksize = CFB8_BLOCK_SIZE, 1079bd3c7b5cSNicolas Royer .cra_ctxsize = sizeof(struct atmel_aes_ctx), 1080bd3c7b5cSNicolas Royer .cra_alignmask = 0x0, 1081bd3c7b5cSNicolas Royer .cra_type = &crypto_ablkcipher_type, 1082bd3c7b5cSNicolas Royer .cra_module = THIS_MODULE, 1083bd3c7b5cSNicolas Royer .cra_init = atmel_aes_cra_init, 1084bd3c7b5cSNicolas Royer .cra_exit = atmel_aes_cra_exit, 1085bd3c7b5cSNicolas Royer .cra_u.ablkcipher = { 1086bd3c7b5cSNicolas Royer .min_keysize = AES_MIN_KEY_SIZE, 1087bd3c7b5cSNicolas Royer .max_keysize = AES_MAX_KEY_SIZE, 1088bd3c7b5cSNicolas Royer .ivsize = AES_BLOCK_SIZE, 1089bd3c7b5cSNicolas Royer .setkey = atmel_aes_setkey, 1090bd3c7b5cSNicolas Royer .encrypt = atmel_aes_cfb8_encrypt, 1091bd3c7b5cSNicolas Royer .decrypt = atmel_aes_cfb8_decrypt, 1092bd3c7b5cSNicolas Royer } 1093bd3c7b5cSNicolas Royer }, 1094bd3c7b5cSNicolas Royer { 1095bd3c7b5cSNicolas Royer .cra_name = "ctr(aes)", 1096bd3c7b5cSNicolas Royer .cra_driver_name = "atmel-ctr-aes", 1097bd3c7b5cSNicolas Royer .cra_priority = 100, 1098bd3c7b5cSNicolas Royer .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, 1099bd3c7b5cSNicolas Royer .cra_blocksize = AES_BLOCK_SIZE, 1100bd3c7b5cSNicolas Royer .cra_ctxsize = sizeof(struct atmel_aes_ctx), 1101cadc4ab8SNicolas Royer .cra_alignmask = 0xf, 1102bd3c7b5cSNicolas Royer .cra_type = &crypto_ablkcipher_type, 1103bd3c7b5cSNicolas Royer .cra_module = THIS_MODULE, 1104bd3c7b5cSNicolas Royer .cra_init = atmel_aes_cra_init, 1105bd3c7b5cSNicolas Royer .cra_exit = atmel_aes_cra_exit, 1106bd3c7b5cSNicolas Royer .cra_u.ablkcipher = { 1107bd3c7b5cSNicolas Royer .min_keysize = AES_MIN_KEY_SIZE, 1108bd3c7b5cSNicolas Royer .max_keysize = AES_MAX_KEY_SIZE, 1109bd3c7b5cSNicolas Royer .ivsize = AES_BLOCK_SIZE, 1110bd3c7b5cSNicolas Royer .setkey = atmel_aes_setkey, 1111bd3c7b5cSNicolas Royer .encrypt = atmel_aes_ctr_encrypt, 1112bd3c7b5cSNicolas Royer .decrypt = atmel_aes_ctr_decrypt, 1113bd3c7b5cSNicolas Royer } 1114bd3c7b5cSNicolas Royer }, 1115bd3c7b5cSNicolas Royer }; 1116bd3c7b5cSNicolas Royer 1117cadc4ab8SNicolas Royer static struct crypto_alg aes_cfb64_alg = { 1118bd3c7b5cSNicolas Royer .cra_name = "cfb64(aes)", 1119bd3c7b5cSNicolas Royer .cra_driver_name = "atmel-cfb64-aes", 1120bd3c7b5cSNicolas Royer .cra_priority = 100, 1121bd3c7b5cSNicolas Royer .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, 1122bd3c7b5cSNicolas Royer .cra_blocksize = CFB64_BLOCK_SIZE, 1123bd3c7b5cSNicolas Royer .cra_ctxsize = sizeof(struct atmel_aes_ctx), 1124cadc4ab8SNicolas Royer .cra_alignmask = 0x7, 1125bd3c7b5cSNicolas Royer .cra_type = &crypto_ablkcipher_type, 1126bd3c7b5cSNicolas Royer .cra_module = THIS_MODULE, 1127bd3c7b5cSNicolas Royer .cra_init = atmel_aes_cra_init, 1128bd3c7b5cSNicolas Royer .cra_exit = atmel_aes_cra_exit, 1129bd3c7b5cSNicolas Royer .cra_u.ablkcipher = { 1130bd3c7b5cSNicolas Royer .min_keysize = AES_MIN_KEY_SIZE, 1131bd3c7b5cSNicolas Royer .max_keysize = AES_MAX_KEY_SIZE, 1132bd3c7b5cSNicolas Royer .ivsize = AES_BLOCK_SIZE, 1133bd3c7b5cSNicolas Royer .setkey = atmel_aes_setkey, 1134bd3c7b5cSNicolas Royer .encrypt = atmel_aes_cfb64_encrypt, 1135bd3c7b5cSNicolas Royer .decrypt = atmel_aes_cfb64_decrypt, 1136bd3c7b5cSNicolas Royer } 1137bd3c7b5cSNicolas Royer }; 1138bd3c7b5cSNicolas Royer 1139bd3c7b5cSNicolas Royer static void atmel_aes_queue_task(unsigned long data) 1140bd3c7b5cSNicolas Royer { 1141bd3c7b5cSNicolas Royer struct atmel_aes_dev *dd = (struct atmel_aes_dev *)data; 1142bd3c7b5cSNicolas Royer 1143bd3c7b5cSNicolas Royer atmel_aes_handle_queue(dd, NULL); 1144bd3c7b5cSNicolas Royer } 1145bd3c7b5cSNicolas Royer 1146bd3c7b5cSNicolas Royer static void atmel_aes_done_task(unsigned long data) 1147bd3c7b5cSNicolas Royer { 1148bd3c7b5cSNicolas Royer struct atmel_aes_dev *dd = (struct atmel_aes_dev *) data; 1149bd3c7b5cSNicolas Royer int err; 1150bd3c7b5cSNicolas Royer 1151bd3c7b5cSNicolas Royer if (!(dd->flags & AES_FLAGS_DMA)) { 1152bd3c7b5cSNicolas Royer atmel_aes_read_n(dd, AES_ODATAR(0), (u32 *) dd->buf_out, 1153bd3c7b5cSNicolas Royer dd->bufcnt >> 2); 1154bd3c7b5cSNicolas Royer 1155bd3c7b5cSNicolas Royer if (sg_copy_from_buffer(dd->out_sg, dd->nb_out_sg, 1156bd3c7b5cSNicolas Royer dd->buf_out, dd->bufcnt)) 1157bd3c7b5cSNicolas Royer err = 0; 1158bd3c7b5cSNicolas Royer else 1159bd3c7b5cSNicolas Royer err = -EINVAL; 1160bd3c7b5cSNicolas Royer 1161bd3c7b5cSNicolas Royer goto cpu_end; 1162bd3c7b5cSNicolas Royer } 1163bd3c7b5cSNicolas Royer 1164bd3c7b5cSNicolas Royer err = atmel_aes_crypt_dma_stop(dd); 1165bd3c7b5cSNicolas Royer 1166bd3c7b5cSNicolas Royer err = dd->err ? : err; 1167bd3c7b5cSNicolas Royer 1168bd3c7b5cSNicolas Royer if (dd->total && !err) { 1169cadc4ab8SNicolas Royer if (dd->flags & AES_FLAGS_FAST) { 1170cadc4ab8SNicolas Royer dd->in_sg = sg_next(dd->in_sg); 1171cadc4ab8SNicolas Royer dd->out_sg = sg_next(dd->out_sg); 1172cadc4ab8SNicolas Royer if (!dd->in_sg || !dd->out_sg) 1173cadc4ab8SNicolas Royer err = -EINVAL; 1174cadc4ab8SNicolas Royer } 1175cadc4ab8SNicolas Royer if (!err) 1176bd3c7b5cSNicolas Royer err = atmel_aes_crypt_dma_start(dd); 1177bd3c7b5cSNicolas Royer if (!err) 1178bd3c7b5cSNicolas Royer return; /* DMA started. Not fininishing. */ 1179bd3c7b5cSNicolas Royer } 1180bd3c7b5cSNicolas Royer 1181bd3c7b5cSNicolas Royer cpu_end: 1182bd3c7b5cSNicolas Royer atmel_aes_finish_req(dd, err); 1183bd3c7b5cSNicolas Royer atmel_aes_handle_queue(dd, NULL); 1184bd3c7b5cSNicolas Royer } 1185bd3c7b5cSNicolas Royer 1186bd3c7b5cSNicolas Royer static irqreturn_t atmel_aes_irq(int irq, void *dev_id) 1187bd3c7b5cSNicolas Royer { 1188bd3c7b5cSNicolas Royer struct atmel_aes_dev *aes_dd = dev_id; 1189bd3c7b5cSNicolas Royer u32 reg; 1190bd3c7b5cSNicolas Royer 1191bd3c7b5cSNicolas Royer reg = atmel_aes_read(aes_dd, AES_ISR); 1192bd3c7b5cSNicolas Royer if (reg & atmel_aes_read(aes_dd, AES_IMR)) { 1193bd3c7b5cSNicolas Royer atmel_aes_write(aes_dd, AES_IDR, reg); 1194bd3c7b5cSNicolas Royer if (AES_FLAGS_BUSY & aes_dd->flags) 1195bd3c7b5cSNicolas Royer tasklet_schedule(&aes_dd->done_task); 1196bd3c7b5cSNicolas Royer else 1197bd3c7b5cSNicolas Royer dev_warn(aes_dd->dev, "AES interrupt when no active requests.\n"); 1198bd3c7b5cSNicolas Royer return IRQ_HANDLED; 1199bd3c7b5cSNicolas Royer } 1200bd3c7b5cSNicolas Royer 1201bd3c7b5cSNicolas Royer return IRQ_NONE; 1202bd3c7b5cSNicolas Royer } 1203bd3c7b5cSNicolas Royer 1204bd3c7b5cSNicolas Royer static void atmel_aes_unregister_algs(struct atmel_aes_dev *dd) 1205bd3c7b5cSNicolas Royer { 1206bd3c7b5cSNicolas Royer int i; 1207bd3c7b5cSNicolas Royer 1208bd3c7b5cSNicolas Royer for (i = 0; i < ARRAY_SIZE(aes_algs); i++) 1209bd3c7b5cSNicolas Royer crypto_unregister_alg(&aes_algs[i]); 1210cadc4ab8SNicolas Royer if (dd->caps.has_cfb64) 1211cadc4ab8SNicolas Royer crypto_unregister_alg(&aes_cfb64_alg); 1212bd3c7b5cSNicolas Royer } 1213bd3c7b5cSNicolas Royer 1214bd3c7b5cSNicolas Royer static int atmel_aes_register_algs(struct atmel_aes_dev *dd) 1215bd3c7b5cSNicolas Royer { 1216bd3c7b5cSNicolas Royer int err, i, j; 1217bd3c7b5cSNicolas Royer 1218bd3c7b5cSNicolas Royer for (i = 0; i < ARRAY_SIZE(aes_algs); i++) { 1219bd3c7b5cSNicolas Royer err = crypto_register_alg(&aes_algs[i]); 1220bd3c7b5cSNicolas Royer if (err) 1221bd3c7b5cSNicolas Royer goto err_aes_algs; 1222bd3c7b5cSNicolas Royer } 1223bd3c7b5cSNicolas Royer 1224cadc4ab8SNicolas Royer if (dd->caps.has_cfb64) { 1225cadc4ab8SNicolas Royer err = crypto_register_alg(&aes_cfb64_alg); 1226bd3c7b5cSNicolas Royer if (err) 1227bd3c7b5cSNicolas Royer goto err_aes_cfb64_alg; 1228bd3c7b5cSNicolas Royer } 1229bd3c7b5cSNicolas Royer 1230bd3c7b5cSNicolas Royer return 0; 1231bd3c7b5cSNicolas Royer 1232bd3c7b5cSNicolas Royer err_aes_cfb64_alg: 1233bd3c7b5cSNicolas Royer i = ARRAY_SIZE(aes_algs); 1234bd3c7b5cSNicolas Royer err_aes_algs: 1235bd3c7b5cSNicolas Royer for (j = 0; j < i; j++) 1236bd3c7b5cSNicolas Royer crypto_unregister_alg(&aes_algs[j]); 1237bd3c7b5cSNicolas Royer 1238bd3c7b5cSNicolas Royer return err; 1239bd3c7b5cSNicolas Royer } 1240bd3c7b5cSNicolas Royer 1241cadc4ab8SNicolas Royer static void atmel_aes_get_cap(struct atmel_aes_dev *dd) 1242cadc4ab8SNicolas Royer { 1243cadc4ab8SNicolas Royer dd->caps.has_dualbuff = 0; 1244cadc4ab8SNicolas Royer dd->caps.has_cfb64 = 0; 1245cadc4ab8SNicolas Royer dd->caps.max_burst_size = 1; 1246cadc4ab8SNicolas Royer 1247cadc4ab8SNicolas Royer /* keep only major version number */ 1248cadc4ab8SNicolas Royer switch (dd->hw_version & 0xff0) { 1249cf1f0d12SLeilei Zhao case 0x200: 1250cf1f0d12SLeilei Zhao dd->caps.has_dualbuff = 1; 1251cf1f0d12SLeilei Zhao dd->caps.has_cfb64 = 1; 1252cf1f0d12SLeilei Zhao dd->caps.max_burst_size = 4; 1253cf1f0d12SLeilei Zhao break; 1254cadc4ab8SNicolas Royer case 0x130: 1255cadc4ab8SNicolas Royer dd->caps.has_dualbuff = 1; 1256cadc4ab8SNicolas Royer dd->caps.has_cfb64 = 1; 1257cadc4ab8SNicolas Royer dd->caps.max_burst_size = 4; 1258cadc4ab8SNicolas Royer break; 1259cadc4ab8SNicolas Royer case 0x120: 1260cadc4ab8SNicolas Royer break; 1261cadc4ab8SNicolas Royer default: 1262cadc4ab8SNicolas Royer dev_warn(dd->dev, 1263cadc4ab8SNicolas Royer "Unmanaged aes version, set minimum capabilities\n"); 1264cadc4ab8SNicolas Royer break; 1265cadc4ab8SNicolas Royer } 1266cadc4ab8SNicolas Royer } 1267cadc4ab8SNicolas Royer 1268be943c7dSNicolas Ferre #if defined(CONFIG_OF) 1269be943c7dSNicolas Ferre static const struct of_device_id atmel_aes_dt_ids[] = { 1270be943c7dSNicolas Ferre { .compatible = "atmel,at91sam9g46-aes" }, 1271be943c7dSNicolas Ferre { /* sentinel */ } 1272be943c7dSNicolas Ferre }; 1273be943c7dSNicolas Ferre MODULE_DEVICE_TABLE(of, atmel_aes_dt_ids); 1274be943c7dSNicolas Ferre 1275be943c7dSNicolas Ferre static struct crypto_platform_data *atmel_aes_of_init(struct platform_device *pdev) 1276be943c7dSNicolas Ferre { 1277be943c7dSNicolas Ferre struct device_node *np = pdev->dev.of_node; 1278be943c7dSNicolas Ferre struct crypto_platform_data *pdata; 1279be943c7dSNicolas Ferre 1280be943c7dSNicolas Ferre if (!np) { 1281be943c7dSNicolas Ferre dev_err(&pdev->dev, "device node not found\n"); 1282be943c7dSNicolas Ferre return ERR_PTR(-EINVAL); 1283be943c7dSNicolas Ferre } 1284be943c7dSNicolas Ferre 1285be943c7dSNicolas Ferre pdata = devm_kzalloc(&pdev->dev, sizeof(*pdata), GFP_KERNEL); 1286be943c7dSNicolas Ferre if (!pdata) { 1287be943c7dSNicolas Ferre dev_err(&pdev->dev, "could not allocate memory for pdata\n"); 1288be943c7dSNicolas Ferre return ERR_PTR(-ENOMEM); 1289be943c7dSNicolas Ferre } 1290be943c7dSNicolas Ferre 1291be943c7dSNicolas Ferre pdata->dma_slave = devm_kzalloc(&pdev->dev, 1292be943c7dSNicolas Ferre sizeof(*(pdata->dma_slave)), 1293be943c7dSNicolas Ferre GFP_KERNEL); 1294be943c7dSNicolas Ferre if (!pdata->dma_slave) { 1295be943c7dSNicolas Ferre dev_err(&pdev->dev, "could not allocate memory for dma_slave\n"); 1296be943c7dSNicolas Ferre devm_kfree(&pdev->dev, pdata); 1297be943c7dSNicolas Ferre return ERR_PTR(-ENOMEM); 1298be943c7dSNicolas Ferre } 1299be943c7dSNicolas Ferre 1300be943c7dSNicolas Ferre return pdata; 1301be943c7dSNicolas Ferre } 1302be943c7dSNicolas Ferre #else 1303be943c7dSNicolas Ferre static inline struct crypto_platform_data *atmel_aes_of_init(struct platform_device *pdev) 1304be943c7dSNicolas Ferre { 1305be943c7dSNicolas Ferre return ERR_PTR(-EINVAL); 1306be943c7dSNicolas Ferre } 1307be943c7dSNicolas Ferre #endif 1308be943c7dSNicolas Ferre 130949cfe4dbSGreg Kroah-Hartman static int atmel_aes_probe(struct platform_device *pdev) 1310bd3c7b5cSNicolas Royer { 1311bd3c7b5cSNicolas Royer struct atmel_aes_dev *aes_dd; 1312cadc4ab8SNicolas Royer struct crypto_platform_data *pdata; 1313bd3c7b5cSNicolas Royer struct device *dev = &pdev->dev; 1314bd3c7b5cSNicolas Royer struct resource *aes_res; 1315bd3c7b5cSNicolas Royer unsigned long aes_phys_size; 1316bd3c7b5cSNicolas Royer int err; 1317bd3c7b5cSNicolas Royer 1318bd3c7b5cSNicolas Royer pdata = pdev->dev.platform_data; 1319bd3c7b5cSNicolas Royer if (!pdata) { 1320be943c7dSNicolas Ferre pdata = atmel_aes_of_init(pdev); 1321be943c7dSNicolas Ferre if (IS_ERR(pdata)) { 1322be943c7dSNicolas Ferre err = PTR_ERR(pdata); 1323be943c7dSNicolas Ferre goto aes_dd_err; 1324be943c7dSNicolas Ferre } 1325be943c7dSNicolas Ferre } 1326be943c7dSNicolas Ferre 1327be943c7dSNicolas Ferre if (!pdata->dma_slave) { 1328bd3c7b5cSNicolas Royer err = -ENXIO; 1329bd3c7b5cSNicolas Royer goto aes_dd_err; 1330bd3c7b5cSNicolas Royer } 1331bd3c7b5cSNicolas Royer 1332bd3c7b5cSNicolas Royer aes_dd = kzalloc(sizeof(struct atmel_aes_dev), GFP_KERNEL); 1333bd3c7b5cSNicolas Royer if (aes_dd == NULL) { 1334bd3c7b5cSNicolas Royer dev_err(dev, "unable to alloc data struct.\n"); 1335bd3c7b5cSNicolas Royer err = -ENOMEM; 1336bd3c7b5cSNicolas Royer goto aes_dd_err; 1337bd3c7b5cSNicolas Royer } 1338bd3c7b5cSNicolas Royer 1339bd3c7b5cSNicolas Royer aes_dd->dev = dev; 1340bd3c7b5cSNicolas Royer 1341bd3c7b5cSNicolas Royer platform_set_drvdata(pdev, aes_dd); 1342bd3c7b5cSNicolas Royer 1343bd3c7b5cSNicolas Royer INIT_LIST_HEAD(&aes_dd->list); 1344bd3c7b5cSNicolas Royer 1345bd3c7b5cSNicolas Royer tasklet_init(&aes_dd->done_task, atmel_aes_done_task, 1346bd3c7b5cSNicolas Royer (unsigned long)aes_dd); 1347bd3c7b5cSNicolas Royer tasklet_init(&aes_dd->queue_task, atmel_aes_queue_task, 1348bd3c7b5cSNicolas Royer (unsigned long)aes_dd); 1349bd3c7b5cSNicolas Royer 1350bd3c7b5cSNicolas Royer crypto_init_queue(&aes_dd->queue, ATMEL_AES_QUEUE_LENGTH); 1351bd3c7b5cSNicolas Royer 1352bd3c7b5cSNicolas Royer aes_dd->irq = -1; 1353bd3c7b5cSNicolas Royer 1354bd3c7b5cSNicolas Royer /* Get the base address */ 1355bd3c7b5cSNicolas Royer aes_res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 1356bd3c7b5cSNicolas Royer if (!aes_res) { 1357bd3c7b5cSNicolas Royer dev_err(dev, "no MEM resource info\n"); 1358bd3c7b5cSNicolas Royer err = -ENODEV; 1359bd3c7b5cSNicolas Royer goto res_err; 1360bd3c7b5cSNicolas Royer } 1361bd3c7b5cSNicolas Royer aes_dd->phys_base = aes_res->start; 1362bd3c7b5cSNicolas Royer aes_phys_size = resource_size(aes_res); 1363bd3c7b5cSNicolas Royer 1364bd3c7b5cSNicolas Royer /* Get the IRQ */ 1365bd3c7b5cSNicolas Royer aes_dd->irq = platform_get_irq(pdev, 0); 1366bd3c7b5cSNicolas Royer if (aes_dd->irq < 0) { 1367bd3c7b5cSNicolas Royer dev_err(dev, "no IRQ resource info\n"); 1368bd3c7b5cSNicolas Royer err = aes_dd->irq; 1369bd3c7b5cSNicolas Royer goto aes_irq_err; 1370bd3c7b5cSNicolas Royer } 1371bd3c7b5cSNicolas Royer 1372bd3c7b5cSNicolas Royer err = request_irq(aes_dd->irq, atmel_aes_irq, IRQF_SHARED, "atmel-aes", 1373bd3c7b5cSNicolas Royer aes_dd); 1374bd3c7b5cSNicolas Royer if (err) { 1375bd3c7b5cSNicolas Royer dev_err(dev, "unable to request aes irq.\n"); 1376bd3c7b5cSNicolas Royer goto aes_irq_err; 1377bd3c7b5cSNicolas Royer } 1378bd3c7b5cSNicolas Royer 1379bd3c7b5cSNicolas Royer /* Initializing the clock */ 1380cadc4ab8SNicolas Royer aes_dd->iclk = clk_get(&pdev->dev, "aes_clk"); 1381bd3c7b5cSNicolas Royer if (IS_ERR(aes_dd->iclk)) { 1382be208356SColin Ian King dev_err(dev, "clock initialization failed.\n"); 1383bd3c7b5cSNicolas Royer err = PTR_ERR(aes_dd->iclk); 1384bd3c7b5cSNicolas Royer goto clk_err; 1385bd3c7b5cSNicolas Royer } 1386bd3c7b5cSNicolas Royer 1387bd3c7b5cSNicolas Royer aes_dd->io_base = ioremap(aes_dd->phys_base, aes_phys_size); 1388bd3c7b5cSNicolas Royer if (!aes_dd->io_base) { 1389bd3c7b5cSNicolas Royer dev_err(dev, "can't ioremap\n"); 1390bd3c7b5cSNicolas Royer err = -ENOMEM; 1391bd3c7b5cSNicolas Royer goto aes_io_err; 1392bd3c7b5cSNicolas Royer } 1393bd3c7b5cSNicolas Royer 1394cadc4ab8SNicolas Royer atmel_aes_hw_version_init(aes_dd); 1395cadc4ab8SNicolas Royer 1396cadc4ab8SNicolas Royer atmel_aes_get_cap(aes_dd); 1397cadc4ab8SNicolas Royer 1398cadc4ab8SNicolas Royer err = atmel_aes_buff_init(aes_dd); 1399cadc4ab8SNicolas Royer if (err) 1400cadc4ab8SNicolas Royer goto err_aes_buff; 1401cadc4ab8SNicolas Royer 1402cadc4ab8SNicolas Royer err = atmel_aes_dma_init(aes_dd, pdata); 1403bd3c7b5cSNicolas Royer if (err) 1404bd3c7b5cSNicolas Royer goto err_aes_dma; 1405bd3c7b5cSNicolas Royer 1406bd3c7b5cSNicolas Royer spin_lock(&atmel_aes.lock); 1407bd3c7b5cSNicolas Royer list_add_tail(&aes_dd->list, &atmel_aes.dev_list); 1408bd3c7b5cSNicolas Royer spin_unlock(&atmel_aes.lock); 1409bd3c7b5cSNicolas Royer 1410bd3c7b5cSNicolas Royer err = atmel_aes_register_algs(aes_dd); 1411bd3c7b5cSNicolas Royer if (err) 1412bd3c7b5cSNicolas Royer goto err_algs; 1413bd3c7b5cSNicolas Royer 1414be943c7dSNicolas Ferre dev_info(dev, "Atmel AES - Using %s, %s for DMA transfers\n", 1415be943c7dSNicolas Ferre dma_chan_name(aes_dd->dma_lch_in.chan), 1416be943c7dSNicolas Ferre dma_chan_name(aes_dd->dma_lch_out.chan)); 1417bd3c7b5cSNicolas Royer 1418bd3c7b5cSNicolas Royer return 0; 1419bd3c7b5cSNicolas Royer 1420bd3c7b5cSNicolas Royer err_algs: 1421bd3c7b5cSNicolas Royer spin_lock(&atmel_aes.lock); 1422bd3c7b5cSNicolas Royer list_del(&aes_dd->list); 1423bd3c7b5cSNicolas Royer spin_unlock(&atmel_aes.lock); 1424bd3c7b5cSNicolas Royer atmel_aes_dma_cleanup(aes_dd); 1425bd3c7b5cSNicolas Royer err_aes_dma: 1426cadc4ab8SNicolas Royer atmel_aes_buff_cleanup(aes_dd); 1427cadc4ab8SNicolas Royer err_aes_buff: 1428bd3c7b5cSNicolas Royer iounmap(aes_dd->io_base); 1429bd3c7b5cSNicolas Royer aes_io_err: 1430bd3c7b5cSNicolas Royer clk_put(aes_dd->iclk); 1431bd3c7b5cSNicolas Royer clk_err: 1432bd3c7b5cSNicolas Royer free_irq(aes_dd->irq, aes_dd); 1433bd3c7b5cSNicolas Royer aes_irq_err: 1434bd3c7b5cSNicolas Royer res_err: 1435bd3c7b5cSNicolas Royer tasklet_kill(&aes_dd->done_task); 1436bd3c7b5cSNicolas Royer tasklet_kill(&aes_dd->queue_task); 1437bd3c7b5cSNicolas Royer kfree(aes_dd); 1438bd3c7b5cSNicolas Royer aes_dd = NULL; 1439bd3c7b5cSNicolas Royer aes_dd_err: 1440bd3c7b5cSNicolas Royer dev_err(dev, "initialization failed.\n"); 1441bd3c7b5cSNicolas Royer 1442bd3c7b5cSNicolas Royer return err; 1443bd3c7b5cSNicolas Royer } 1444bd3c7b5cSNicolas Royer 144549cfe4dbSGreg Kroah-Hartman static int atmel_aes_remove(struct platform_device *pdev) 1446bd3c7b5cSNicolas Royer { 1447bd3c7b5cSNicolas Royer static struct atmel_aes_dev *aes_dd; 1448bd3c7b5cSNicolas Royer 1449bd3c7b5cSNicolas Royer aes_dd = platform_get_drvdata(pdev); 1450bd3c7b5cSNicolas Royer if (!aes_dd) 1451bd3c7b5cSNicolas Royer return -ENODEV; 1452bd3c7b5cSNicolas Royer spin_lock(&atmel_aes.lock); 1453bd3c7b5cSNicolas Royer list_del(&aes_dd->list); 1454bd3c7b5cSNicolas Royer spin_unlock(&atmel_aes.lock); 1455bd3c7b5cSNicolas Royer 1456bd3c7b5cSNicolas Royer atmel_aes_unregister_algs(aes_dd); 1457bd3c7b5cSNicolas Royer 1458bd3c7b5cSNicolas Royer tasklet_kill(&aes_dd->done_task); 1459bd3c7b5cSNicolas Royer tasklet_kill(&aes_dd->queue_task); 1460bd3c7b5cSNicolas Royer 1461bd3c7b5cSNicolas Royer atmel_aes_dma_cleanup(aes_dd); 1462bd3c7b5cSNicolas Royer 1463bd3c7b5cSNicolas Royer iounmap(aes_dd->io_base); 1464bd3c7b5cSNicolas Royer 1465bd3c7b5cSNicolas Royer clk_put(aes_dd->iclk); 1466bd3c7b5cSNicolas Royer 1467bd3c7b5cSNicolas Royer if (aes_dd->irq > 0) 1468bd3c7b5cSNicolas Royer free_irq(aes_dd->irq, aes_dd); 1469bd3c7b5cSNicolas Royer 1470bd3c7b5cSNicolas Royer kfree(aes_dd); 1471bd3c7b5cSNicolas Royer aes_dd = NULL; 1472bd3c7b5cSNicolas Royer 1473bd3c7b5cSNicolas Royer return 0; 1474bd3c7b5cSNicolas Royer } 1475bd3c7b5cSNicolas Royer 1476bd3c7b5cSNicolas Royer static struct platform_driver atmel_aes_driver = { 1477bd3c7b5cSNicolas Royer .probe = atmel_aes_probe, 147849cfe4dbSGreg Kroah-Hartman .remove = atmel_aes_remove, 1479bd3c7b5cSNicolas Royer .driver = { 1480bd3c7b5cSNicolas Royer .name = "atmel_aes", 1481be943c7dSNicolas Ferre .of_match_table = of_match_ptr(atmel_aes_dt_ids), 1482bd3c7b5cSNicolas Royer }, 1483bd3c7b5cSNicolas Royer }; 1484bd3c7b5cSNicolas Royer 1485bd3c7b5cSNicolas Royer module_platform_driver(atmel_aes_driver); 1486bd3c7b5cSNicolas Royer 1487bd3c7b5cSNicolas Royer MODULE_DESCRIPTION("Atmel AES hw acceleration support."); 1488bd3c7b5cSNicolas Royer MODULE_LICENSE("GPL v2"); 1489bd3c7b5cSNicolas Royer MODULE_AUTHOR("Nicolas Royer - Eukréa Electromatique"); 1490