1 /* 2 * Cryptographic API. 3 * 4 * Support for SAHARA cryptographic accelerator. 5 * 6 * Copyright (c) 2014 Steffen Trumtrar <s.trumtrar@pengutronix.de> 7 * Copyright (c) 2013 Vista Silicon S.L. 8 * Author: Javier Martin <javier.martin@vista-silicon.com> 9 * 10 * This program is free software; you can redistribute it and/or modify 11 * it under the terms of the GNU General Public License version 2 as published 12 * by the Free Software Foundation. 13 * 14 * Based on omap-aes.c and tegra-aes.c 15 */ 16 17 #include <crypto/algapi.h> 18 #include <crypto/aes.h> 19 #include <crypto/hash.h> 20 #include <crypto/internal/hash.h> 21 #include <crypto/scatterwalk.h> 22 #include <crypto/sha.h> 23 24 #include <linux/clk.h> 25 #include <linux/crypto.h> 26 #include <linux/interrupt.h> 27 #include <linux/io.h> 28 #include <linux/irq.h> 29 #include <linux/kernel.h> 30 #include <linux/kthread.h> 31 #include <linux/module.h> 32 #include <linux/mutex.h> 33 #include <linux/of.h> 34 #include <linux/of_device.h> 35 #include <linux/platform_device.h> 36 37 #define SHA_BUFFER_LEN PAGE_SIZE 38 #define SAHARA_MAX_SHA_BLOCK_SIZE SHA256_BLOCK_SIZE 39 40 #define SAHARA_NAME "sahara" 41 #define SAHARA_VERSION_3 3 42 #define SAHARA_VERSION_4 4 43 #define SAHARA_TIMEOUT_MS 1000 44 #define SAHARA_MAX_HW_DESC 2 45 #define SAHARA_MAX_HW_LINK 20 46 47 #define FLAGS_MODE_MASK 0x000f 48 #define FLAGS_ENCRYPT BIT(0) 49 #define FLAGS_CBC BIT(1) 50 #define FLAGS_NEW_KEY BIT(3) 51 52 #define SAHARA_HDR_BASE 0x00800000 53 #define SAHARA_HDR_SKHA_ALG_AES 0 54 #define SAHARA_HDR_SKHA_OP_ENC (1 << 2) 55 #define SAHARA_HDR_SKHA_MODE_ECB (0 << 3) 56 #define SAHARA_HDR_SKHA_MODE_CBC (1 << 3) 57 #define SAHARA_HDR_FORM_DATA (5 << 16) 58 #define SAHARA_HDR_FORM_KEY (8 << 16) 59 #define SAHARA_HDR_LLO (1 << 24) 60 #define SAHARA_HDR_CHA_SKHA (1 << 28) 61 #define SAHARA_HDR_CHA_MDHA (2 << 28) 62 #define SAHARA_HDR_PARITY_BIT (1 << 31) 63 64 #define SAHARA_HDR_MDHA_SET_MODE_MD_KEY 0x20880000 65 #define SAHARA_HDR_MDHA_SET_MODE_HASH 0x208D0000 66 #define SAHARA_HDR_MDHA_HASH 0xA0850000 67 #define SAHARA_HDR_MDHA_STORE_DIGEST 0x20820000 68 #define SAHARA_HDR_MDHA_ALG_SHA1 0 69 #define SAHARA_HDR_MDHA_ALG_MD5 1 70 #define SAHARA_HDR_MDHA_ALG_SHA256 2 71 #define SAHARA_HDR_MDHA_ALG_SHA224 3 72 #define SAHARA_HDR_MDHA_PDATA (1 << 2) 73 #define SAHARA_HDR_MDHA_HMAC (1 << 3) 74 #define SAHARA_HDR_MDHA_INIT (1 << 5) 75 #define SAHARA_HDR_MDHA_IPAD (1 << 6) 76 #define SAHARA_HDR_MDHA_OPAD (1 << 7) 77 #define SAHARA_HDR_MDHA_SWAP (1 << 8) 78 #define SAHARA_HDR_MDHA_MAC_FULL (1 << 9) 79 #define SAHARA_HDR_MDHA_SSL (1 << 10) 80 81 /* SAHARA can only process one request at a time */ 82 #define SAHARA_QUEUE_LENGTH 1 83 84 #define SAHARA_REG_VERSION 0x00 85 #define SAHARA_REG_DAR 0x04 86 #define SAHARA_REG_CONTROL 0x08 87 #define SAHARA_CONTROL_SET_THROTTLE(x) (((x) & 0xff) << 24) 88 #define SAHARA_CONTROL_SET_MAXBURST(x) (((x) & 0xff) << 16) 89 #define SAHARA_CONTROL_RNG_AUTORSD (1 << 7) 90 #define SAHARA_CONTROL_ENABLE_INT (1 << 4) 91 #define SAHARA_REG_CMD 0x0C 92 #define SAHARA_CMD_RESET (1 << 0) 93 #define SAHARA_CMD_CLEAR_INT (1 << 8) 94 #define SAHARA_CMD_CLEAR_ERR (1 << 9) 95 #define SAHARA_CMD_SINGLE_STEP (1 << 10) 96 #define SAHARA_CMD_MODE_BATCH (1 << 16) 97 #define SAHARA_CMD_MODE_DEBUG (1 << 18) 98 #define SAHARA_REG_STATUS 0x10 99 #define SAHARA_STATUS_GET_STATE(x) ((x) & 0x7) 100 #define SAHARA_STATE_IDLE 0 101 #define SAHARA_STATE_BUSY 1 102 #define SAHARA_STATE_ERR 2 103 #define SAHARA_STATE_FAULT 3 104 #define SAHARA_STATE_COMPLETE 4 105 #define SAHARA_STATE_COMP_FLAG (1 << 2) 106 #define SAHARA_STATUS_DAR_FULL (1 << 3) 107 #define SAHARA_STATUS_ERROR (1 << 4) 108 #define SAHARA_STATUS_SECURE (1 << 5) 109 #define SAHARA_STATUS_FAIL (1 << 6) 110 #define SAHARA_STATUS_INIT (1 << 7) 111 #define SAHARA_STATUS_RNG_RESEED (1 << 8) 112 #define SAHARA_STATUS_ACTIVE_RNG (1 << 9) 113 #define SAHARA_STATUS_ACTIVE_MDHA (1 << 10) 114 #define SAHARA_STATUS_ACTIVE_SKHA (1 << 11) 115 #define SAHARA_STATUS_MODE_BATCH (1 << 16) 116 #define SAHARA_STATUS_MODE_DEDICATED (1 << 17) 117 #define SAHARA_STATUS_MODE_DEBUG (1 << 18) 118 #define SAHARA_STATUS_GET_ISTATE(x) (((x) >> 24) & 0xff) 119 #define SAHARA_REG_ERRSTATUS 0x14 120 #define SAHARA_ERRSTATUS_GET_SOURCE(x) ((x) & 0xf) 121 #define SAHARA_ERRSOURCE_CHA 14 122 #define SAHARA_ERRSOURCE_DMA 15 123 #define SAHARA_ERRSTATUS_DMA_DIR (1 << 8) 124 #define SAHARA_ERRSTATUS_GET_DMASZ(x)(((x) >> 9) & 0x3) 125 #define SAHARA_ERRSTATUS_GET_DMASRC(x) (((x) >> 13) & 0x7) 126 #define SAHARA_ERRSTATUS_GET_CHASRC(x) (((x) >> 16) & 0xfff) 127 #define SAHARA_ERRSTATUS_GET_CHAERR(x) (((x) >> 28) & 0x3) 128 #define SAHARA_REG_FADDR 0x18 129 #define SAHARA_REG_CDAR 0x1C 130 #define SAHARA_REG_IDAR 0x20 131 132 struct sahara_hw_desc { 133 u32 hdr; 134 u32 len1; 135 dma_addr_t p1; 136 u32 len2; 137 dma_addr_t p2; 138 dma_addr_t next; 139 }; 140 141 struct sahara_hw_link { 142 u32 len; 143 dma_addr_t p; 144 dma_addr_t next; 145 }; 146 147 struct sahara_ctx { 148 unsigned long flags; 149 150 /* AES-specific context */ 151 int keylen; 152 u8 key[AES_KEYSIZE_128]; 153 struct crypto_ablkcipher *fallback; 154 155 /* SHA-specific context */ 156 struct crypto_shash *shash_fallback; 157 }; 158 159 struct sahara_aes_reqctx { 160 unsigned long mode; 161 }; 162 163 /* 164 * struct sahara_sha_reqctx - private data per request 165 * @buf: holds data for requests smaller than block_size 166 * @rembuf: used to prepare one block_size-aligned request 167 * @context: hw-specific context for request. Digest is extracted from this 168 * @mode: specifies what type of hw-descriptor needs to be built 169 * @digest_size: length of digest for this request 170 * @context_size: length of hw-context for this request. 171 * Always digest_size + 4 172 * @buf_cnt: number of bytes saved in buf 173 * @sg_in_idx: number of hw links 174 * @in_sg: scatterlist for input data 175 * @in_sg_chain: scatterlists for chained input data 176 * @total: total number of bytes for transfer 177 * @last: is this the last block 178 * @first: is this the first block 179 * @active: inside a transfer 180 */ 181 struct sahara_sha_reqctx { 182 u8 buf[SAHARA_MAX_SHA_BLOCK_SIZE]; 183 u8 rembuf[SAHARA_MAX_SHA_BLOCK_SIZE]; 184 u8 context[SHA256_DIGEST_SIZE + 4]; 185 struct mutex mutex; 186 unsigned int mode; 187 unsigned int digest_size; 188 unsigned int context_size; 189 unsigned int buf_cnt; 190 unsigned int sg_in_idx; 191 struct scatterlist *in_sg; 192 struct scatterlist in_sg_chain[2]; 193 size_t total; 194 unsigned int last; 195 unsigned int first; 196 unsigned int active; 197 }; 198 199 struct sahara_dev { 200 struct device *device; 201 unsigned int version; 202 void __iomem *regs_base; 203 struct clk *clk_ipg; 204 struct clk *clk_ahb; 205 struct mutex queue_mutex; 206 struct task_struct *kthread; 207 struct completion dma_completion; 208 209 struct sahara_ctx *ctx; 210 spinlock_t lock; 211 struct crypto_queue queue; 212 unsigned long flags; 213 214 struct sahara_hw_desc *hw_desc[SAHARA_MAX_HW_DESC]; 215 dma_addr_t hw_phys_desc[SAHARA_MAX_HW_DESC]; 216 217 u8 *key_base; 218 dma_addr_t key_phys_base; 219 220 u8 *iv_base; 221 dma_addr_t iv_phys_base; 222 223 u8 *context_base; 224 dma_addr_t context_phys_base; 225 226 struct sahara_hw_link *hw_link[SAHARA_MAX_HW_LINK]; 227 dma_addr_t hw_phys_link[SAHARA_MAX_HW_LINK]; 228 229 size_t total; 230 struct scatterlist *in_sg; 231 unsigned int nb_in_sg; 232 struct scatterlist *out_sg; 233 unsigned int nb_out_sg; 234 235 u32 error; 236 }; 237 238 static struct sahara_dev *dev_ptr; 239 240 static inline void sahara_write(struct sahara_dev *dev, u32 data, u32 reg) 241 { 242 writel(data, dev->regs_base + reg); 243 } 244 245 static inline unsigned int sahara_read(struct sahara_dev *dev, u32 reg) 246 { 247 return readl(dev->regs_base + reg); 248 } 249 250 static u32 sahara_aes_key_hdr(struct sahara_dev *dev) 251 { 252 u32 hdr = SAHARA_HDR_BASE | SAHARA_HDR_SKHA_ALG_AES | 253 SAHARA_HDR_FORM_KEY | SAHARA_HDR_LLO | 254 SAHARA_HDR_CHA_SKHA | SAHARA_HDR_PARITY_BIT; 255 256 if (dev->flags & FLAGS_CBC) { 257 hdr |= SAHARA_HDR_SKHA_MODE_CBC; 258 hdr ^= SAHARA_HDR_PARITY_BIT; 259 } 260 261 if (dev->flags & FLAGS_ENCRYPT) { 262 hdr |= SAHARA_HDR_SKHA_OP_ENC; 263 hdr ^= SAHARA_HDR_PARITY_BIT; 264 } 265 266 return hdr; 267 } 268 269 static u32 sahara_aes_data_link_hdr(struct sahara_dev *dev) 270 { 271 return SAHARA_HDR_BASE | SAHARA_HDR_FORM_DATA | 272 SAHARA_HDR_CHA_SKHA | SAHARA_HDR_PARITY_BIT; 273 } 274 275 static const char *sahara_err_src[16] = { 276 "No error", 277 "Header error", 278 "Descriptor length error", 279 "Descriptor length or pointer error", 280 "Link length error", 281 "Link pointer error", 282 "Input buffer error", 283 "Output buffer error", 284 "Output buffer starvation", 285 "Internal state fault", 286 "General descriptor problem", 287 "Reserved", 288 "Descriptor address error", 289 "Link address error", 290 "CHA error", 291 "DMA error" 292 }; 293 294 static const char *sahara_err_dmasize[4] = { 295 "Byte transfer", 296 "Half-word transfer", 297 "Word transfer", 298 "Reserved" 299 }; 300 301 static const char *sahara_err_dmasrc[8] = { 302 "No error", 303 "AHB bus error", 304 "Internal IP bus error", 305 "Parity error", 306 "DMA crosses 256 byte boundary", 307 "DMA is busy", 308 "Reserved", 309 "DMA HW error" 310 }; 311 312 static const char *sahara_cha_errsrc[12] = { 313 "Input buffer non-empty", 314 "Illegal address", 315 "Illegal mode", 316 "Illegal data size", 317 "Illegal key size", 318 "Write during processing", 319 "CTX read during processing", 320 "HW error", 321 "Input buffer disabled/underflow", 322 "Output buffer disabled/overflow", 323 "DES key parity error", 324 "Reserved" 325 }; 326 327 static const char *sahara_cha_err[4] = { "No error", "SKHA", "MDHA", "RNG" }; 328 329 static void sahara_decode_error(struct sahara_dev *dev, unsigned int error) 330 { 331 u8 source = SAHARA_ERRSTATUS_GET_SOURCE(error); 332 u16 chasrc = ffs(SAHARA_ERRSTATUS_GET_CHASRC(error)); 333 334 dev_err(dev->device, "%s: Error Register = 0x%08x\n", __func__, error); 335 336 dev_err(dev->device, " - %s.\n", sahara_err_src[source]); 337 338 if (source == SAHARA_ERRSOURCE_DMA) { 339 if (error & SAHARA_ERRSTATUS_DMA_DIR) 340 dev_err(dev->device, " * DMA read.\n"); 341 else 342 dev_err(dev->device, " * DMA write.\n"); 343 344 dev_err(dev->device, " * %s.\n", 345 sahara_err_dmasize[SAHARA_ERRSTATUS_GET_DMASZ(error)]); 346 dev_err(dev->device, " * %s.\n", 347 sahara_err_dmasrc[SAHARA_ERRSTATUS_GET_DMASRC(error)]); 348 } else if (source == SAHARA_ERRSOURCE_CHA) { 349 dev_err(dev->device, " * %s.\n", 350 sahara_cha_errsrc[chasrc]); 351 dev_err(dev->device, " * %s.\n", 352 sahara_cha_err[SAHARA_ERRSTATUS_GET_CHAERR(error)]); 353 } 354 dev_err(dev->device, "\n"); 355 } 356 357 static const char *sahara_state[4] = { "Idle", "Busy", "Error", "HW Fault" }; 358 359 static void sahara_decode_status(struct sahara_dev *dev, unsigned int status) 360 { 361 u8 state; 362 363 if (!IS_ENABLED(DEBUG)) 364 return; 365 366 state = SAHARA_STATUS_GET_STATE(status); 367 368 dev_dbg(dev->device, "%s: Status Register = 0x%08x\n", 369 __func__, status); 370 371 dev_dbg(dev->device, " - State = %d:\n", state); 372 if (state & SAHARA_STATE_COMP_FLAG) 373 dev_dbg(dev->device, " * Descriptor completed. IRQ pending.\n"); 374 375 dev_dbg(dev->device, " * %s.\n", 376 sahara_state[state & ~SAHARA_STATE_COMP_FLAG]); 377 378 if (status & SAHARA_STATUS_DAR_FULL) 379 dev_dbg(dev->device, " - DAR Full.\n"); 380 if (status & SAHARA_STATUS_ERROR) 381 dev_dbg(dev->device, " - Error.\n"); 382 if (status & SAHARA_STATUS_SECURE) 383 dev_dbg(dev->device, " - Secure.\n"); 384 if (status & SAHARA_STATUS_FAIL) 385 dev_dbg(dev->device, " - Fail.\n"); 386 if (status & SAHARA_STATUS_RNG_RESEED) 387 dev_dbg(dev->device, " - RNG Reseed Request.\n"); 388 if (status & SAHARA_STATUS_ACTIVE_RNG) 389 dev_dbg(dev->device, " - RNG Active.\n"); 390 if (status & SAHARA_STATUS_ACTIVE_MDHA) 391 dev_dbg(dev->device, " - MDHA Active.\n"); 392 if (status & SAHARA_STATUS_ACTIVE_SKHA) 393 dev_dbg(dev->device, " - SKHA Active.\n"); 394 395 if (status & SAHARA_STATUS_MODE_BATCH) 396 dev_dbg(dev->device, " - Batch Mode.\n"); 397 else if (status & SAHARA_STATUS_MODE_DEDICATED) 398 dev_dbg(dev->device, " - Decidated Mode.\n"); 399 else if (status & SAHARA_STATUS_MODE_DEBUG) 400 dev_dbg(dev->device, " - Debug Mode.\n"); 401 402 dev_dbg(dev->device, " - Internal state = 0x%02x\n", 403 SAHARA_STATUS_GET_ISTATE(status)); 404 405 dev_dbg(dev->device, "Current DAR: 0x%08x\n", 406 sahara_read(dev, SAHARA_REG_CDAR)); 407 dev_dbg(dev->device, "Initial DAR: 0x%08x\n\n", 408 sahara_read(dev, SAHARA_REG_IDAR)); 409 } 410 411 static void sahara_dump_descriptors(struct sahara_dev *dev) 412 { 413 int i; 414 415 if (!IS_ENABLED(DEBUG)) 416 return; 417 418 for (i = 0; i < SAHARA_MAX_HW_DESC; i++) { 419 dev_dbg(dev->device, "Descriptor (%d) (0x%08x):\n", 420 i, dev->hw_phys_desc[i]); 421 dev_dbg(dev->device, "\thdr = 0x%08x\n", dev->hw_desc[i]->hdr); 422 dev_dbg(dev->device, "\tlen1 = %u\n", dev->hw_desc[i]->len1); 423 dev_dbg(dev->device, "\tp1 = 0x%08x\n", dev->hw_desc[i]->p1); 424 dev_dbg(dev->device, "\tlen2 = %u\n", dev->hw_desc[i]->len2); 425 dev_dbg(dev->device, "\tp2 = 0x%08x\n", dev->hw_desc[i]->p2); 426 dev_dbg(dev->device, "\tnext = 0x%08x\n", 427 dev->hw_desc[i]->next); 428 } 429 dev_dbg(dev->device, "\n"); 430 } 431 432 static void sahara_dump_links(struct sahara_dev *dev) 433 { 434 int i; 435 436 if (!IS_ENABLED(DEBUG)) 437 return; 438 439 for (i = 0; i < SAHARA_MAX_HW_LINK; i++) { 440 dev_dbg(dev->device, "Link (%d) (0x%08x):\n", 441 i, dev->hw_phys_link[i]); 442 dev_dbg(dev->device, "\tlen = %u\n", dev->hw_link[i]->len); 443 dev_dbg(dev->device, "\tp = 0x%08x\n", dev->hw_link[i]->p); 444 dev_dbg(dev->device, "\tnext = 0x%08x\n", 445 dev->hw_link[i]->next); 446 } 447 dev_dbg(dev->device, "\n"); 448 } 449 450 static int sahara_hw_descriptor_create(struct sahara_dev *dev) 451 { 452 struct sahara_ctx *ctx = dev->ctx; 453 struct scatterlist *sg; 454 int ret; 455 int i, j; 456 int idx = 0; 457 458 /* Copy new key if necessary */ 459 if (ctx->flags & FLAGS_NEW_KEY) { 460 memcpy(dev->key_base, ctx->key, ctx->keylen); 461 ctx->flags &= ~FLAGS_NEW_KEY; 462 463 if (dev->flags & FLAGS_CBC) { 464 dev->hw_desc[idx]->len1 = AES_BLOCK_SIZE; 465 dev->hw_desc[idx]->p1 = dev->iv_phys_base; 466 } else { 467 dev->hw_desc[idx]->len1 = 0; 468 dev->hw_desc[idx]->p1 = 0; 469 } 470 dev->hw_desc[idx]->len2 = ctx->keylen; 471 dev->hw_desc[idx]->p2 = dev->key_phys_base; 472 dev->hw_desc[idx]->next = dev->hw_phys_desc[1]; 473 474 dev->hw_desc[idx]->hdr = sahara_aes_key_hdr(dev); 475 476 idx++; 477 } 478 479 dev->nb_in_sg = sg_nents_for_len(dev->in_sg, dev->total); 480 dev->nb_out_sg = sg_nents_for_len(dev->out_sg, dev->total); 481 if ((dev->nb_in_sg + dev->nb_out_sg) > SAHARA_MAX_HW_LINK) { 482 dev_err(dev->device, "not enough hw links (%d)\n", 483 dev->nb_in_sg + dev->nb_out_sg); 484 return -EINVAL; 485 } 486 487 ret = dma_map_sg(dev->device, dev->in_sg, dev->nb_in_sg, 488 DMA_TO_DEVICE); 489 if (ret != dev->nb_in_sg) { 490 dev_err(dev->device, "couldn't map in sg\n"); 491 goto unmap_in; 492 } 493 ret = dma_map_sg(dev->device, dev->out_sg, dev->nb_out_sg, 494 DMA_FROM_DEVICE); 495 if (ret != dev->nb_out_sg) { 496 dev_err(dev->device, "couldn't map out sg\n"); 497 goto unmap_out; 498 } 499 500 /* Create input links */ 501 dev->hw_desc[idx]->p1 = dev->hw_phys_link[0]; 502 sg = dev->in_sg; 503 for (i = 0; i < dev->nb_in_sg; i++) { 504 dev->hw_link[i]->len = sg->length; 505 dev->hw_link[i]->p = sg->dma_address; 506 if (i == (dev->nb_in_sg - 1)) { 507 dev->hw_link[i]->next = 0; 508 } else { 509 dev->hw_link[i]->next = dev->hw_phys_link[i + 1]; 510 sg = sg_next(sg); 511 } 512 } 513 514 /* Create output links */ 515 dev->hw_desc[idx]->p2 = dev->hw_phys_link[i]; 516 sg = dev->out_sg; 517 for (j = i; j < dev->nb_out_sg + i; j++) { 518 dev->hw_link[j]->len = sg->length; 519 dev->hw_link[j]->p = sg->dma_address; 520 if (j == (dev->nb_out_sg + i - 1)) { 521 dev->hw_link[j]->next = 0; 522 } else { 523 dev->hw_link[j]->next = dev->hw_phys_link[j + 1]; 524 sg = sg_next(sg); 525 } 526 } 527 528 /* Fill remaining fields of hw_desc[1] */ 529 dev->hw_desc[idx]->hdr = sahara_aes_data_link_hdr(dev); 530 dev->hw_desc[idx]->len1 = dev->total; 531 dev->hw_desc[idx]->len2 = dev->total; 532 dev->hw_desc[idx]->next = 0; 533 534 sahara_dump_descriptors(dev); 535 sahara_dump_links(dev); 536 537 sahara_write(dev, dev->hw_phys_desc[0], SAHARA_REG_DAR); 538 539 return 0; 540 541 unmap_out: 542 dma_unmap_sg(dev->device, dev->out_sg, dev->nb_out_sg, 543 DMA_TO_DEVICE); 544 unmap_in: 545 dma_unmap_sg(dev->device, dev->in_sg, dev->nb_in_sg, 546 DMA_FROM_DEVICE); 547 548 return -EINVAL; 549 } 550 551 static int sahara_aes_process(struct ablkcipher_request *req) 552 { 553 struct sahara_dev *dev = dev_ptr; 554 struct sahara_ctx *ctx; 555 struct sahara_aes_reqctx *rctx; 556 int ret; 557 unsigned long timeout; 558 559 /* Request is ready to be dispatched by the device */ 560 dev_dbg(dev->device, 561 "dispatch request (nbytes=%d, src=%p, dst=%p)\n", 562 req->nbytes, req->src, req->dst); 563 564 /* assign new request to device */ 565 dev->total = req->nbytes; 566 dev->in_sg = req->src; 567 dev->out_sg = req->dst; 568 569 rctx = ablkcipher_request_ctx(req); 570 ctx = crypto_ablkcipher_ctx(crypto_ablkcipher_reqtfm(req)); 571 rctx->mode &= FLAGS_MODE_MASK; 572 dev->flags = (dev->flags & ~FLAGS_MODE_MASK) | rctx->mode; 573 574 if ((dev->flags & FLAGS_CBC) && req->info) 575 memcpy(dev->iv_base, req->info, AES_KEYSIZE_128); 576 577 /* assign new context to device */ 578 dev->ctx = ctx; 579 580 reinit_completion(&dev->dma_completion); 581 582 ret = sahara_hw_descriptor_create(dev); 583 if (ret) 584 return -EINVAL; 585 586 timeout = wait_for_completion_timeout(&dev->dma_completion, 587 msecs_to_jiffies(SAHARA_TIMEOUT_MS)); 588 if (!timeout) { 589 dev_err(dev->device, "AES timeout\n"); 590 return -ETIMEDOUT; 591 } 592 593 dma_unmap_sg(dev->device, dev->out_sg, dev->nb_out_sg, 594 DMA_TO_DEVICE); 595 dma_unmap_sg(dev->device, dev->in_sg, dev->nb_in_sg, 596 DMA_FROM_DEVICE); 597 598 return 0; 599 } 600 601 static int sahara_aes_setkey(struct crypto_ablkcipher *tfm, const u8 *key, 602 unsigned int keylen) 603 { 604 struct sahara_ctx *ctx = crypto_ablkcipher_ctx(tfm); 605 int ret; 606 607 ctx->keylen = keylen; 608 609 /* SAHARA only supports 128bit keys */ 610 if (keylen == AES_KEYSIZE_128) { 611 memcpy(ctx->key, key, keylen); 612 ctx->flags |= FLAGS_NEW_KEY; 613 return 0; 614 } 615 616 if (keylen != AES_KEYSIZE_128 && 617 keylen != AES_KEYSIZE_192 && keylen != AES_KEYSIZE_256) 618 return -EINVAL; 619 620 /* 621 * The requested key size is not supported by HW, do a fallback. 622 */ 623 ctx->fallback->base.crt_flags &= ~CRYPTO_TFM_REQ_MASK; 624 ctx->fallback->base.crt_flags |= 625 (tfm->base.crt_flags & CRYPTO_TFM_REQ_MASK); 626 627 ret = crypto_ablkcipher_setkey(ctx->fallback, key, keylen); 628 if (ret) { 629 struct crypto_tfm *tfm_aux = crypto_ablkcipher_tfm(tfm); 630 631 tfm_aux->crt_flags &= ~CRYPTO_TFM_RES_MASK; 632 tfm_aux->crt_flags |= 633 (ctx->fallback->base.crt_flags & CRYPTO_TFM_RES_MASK); 634 } 635 return ret; 636 } 637 638 static int sahara_aes_crypt(struct ablkcipher_request *req, unsigned long mode) 639 { 640 struct sahara_aes_reqctx *rctx = ablkcipher_request_ctx(req); 641 struct sahara_dev *dev = dev_ptr; 642 int err = 0; 643 644 dev_dbg(dev->device, "nbytes: %d, enc: %d, cbc: %d\n", 645 req->nbytes, !!(mode & FLAGS_ENCRYPT), !!(mode & FLAGS_CBC)); 646 647 if (!IS_ALIGNED(req->nbytes, AES_BLOCK_SIZE)) { 648 dev_err(dev->device, 649 "request size is not exact amount of AES blocks\n"); 650 return -EINVAL; 651 } 652 653 rctx->mode = mode; 654 655 mutex_lock(&dev->queue_mutex); 656 err = ablkcipher_enqueue_request(&dev->queue, req); 657 mutex_unlock(&dev->queue_mutex); 658 659 wake_up_process(dev->kthread); 660 661 return err; 662 } 663 664 static int sahara_aes_ecb_encrypt(struct ablkcipher_request *req) 665 { 666 struct crypto_tfm *tfm = 667 crypto_ablkcipher_tfm(crypto_ablkcipher_reqtfm(req)); 668 struct sahara_ctx *ctx = crypto_ablkcipher_ctx( 669 crypto_ablkcipher_reqtfm(req)); 670 int err; 671 672 if (unlikely(ctx->keylen != AES_KEYSIZE_128)) { 673 ablkcipher_request_set_tfm(req, ctx->fallback); 674 err = crypto_ablkcipher_encrypt(req); 675 ablkcipher_request_set_tfm(req, __crypto_ablkcipher_cast(tfm)); 676 return err; 677 } 678 679 return sahara_aes_crypt(req, FLAGS_ENCRYPT); 680 } 681 682 static int sahara_aes_ecb_decrypt(struct ablkcipher_request *req) 683 { 684 struct crypto_tfm *tfm = 685 crypto_ablkcipher_tfm(crypto_ablkcipher_reqtfm(req)); 686 struct sahara_ctx *ctx = crypto_ablkcipher_ctx( 687 crypto_ablkcipher_reqtfm(req)); 688 int err; 689 690 if (unlikely(ctx->keylen != AES_KEYSIZE_128)) { 691 ablkcipher_request_set_tfm(req, ctx->fallback); 692 err = crypto_ablkcipher_decrypt(req); 693 ablkcipher_request_set_tfm(req, __crypto_ablkcipher_cast(tfm)); 694 return err; 695 } 696 697 return sahara_aes_crypt(req, 0); 698 } 699 700 static int sahara_aes_cbc_encrypt(struct ablkcipher_request *req) 701 { 702 struct crypto_tfm *tfm = 703 crypto_ablkcipher_tfm(crypto_ablkcipher_reqtfm(req)); 704 struct sahara_ctx *ctx = crypto_ablkcipher_ctx( 705 crypto_ablkcipher_reqtfm(req)); 706 int err; 707 708 if (unlikely(ctx->keylen != AES_KEYSIZE_128)) { 709 ablkcipher_request_set_tfm(req, ctx->fallback); 710 err = crypto_ablkcipher_encrypt(req); 711 ablkcipher_request_set_tfm(req, __crypto_ablkcipher_cast(tfm)); 712 return err; 713 } 714 715 return sahara_aes_crypt(req, FLAGS_ENCRYPT | FLAGS_CBC); 716 } 717 718 static int sahara_aes_cbc_decrypt(struct ablkcipher_request *req) 719 { 720 struct crypto_tfm *tfm = 721 crypto_ablkcipher_tfm(crypto_ablkcipher_reqtfm(req)); 722 struct sahara_ctx *ctx = crypto_ablkcipher_ctx( 723 crypto_ablkcipher_reqtfm(req)); 724 int err; 725 726 if (unlikely(ctx->keylen != AES_KEYSIZE_128)) { 727 ablkcipher_request_set_tfm(req, ctx->fallback); 728 err = crypto_ablkcipher_decrypt(req); 729 ablkcipher_request_set_tfm(req, __crypto_ablkcipher_cast(tfm)); 730 return err; 731 } 732 733 return sahara_aes_crypt(req, FLAGS_CBC); 734 } 735 736 static int sahara_aes_cra_init(struct crypto_tfm *tfm) 737 { 738 const char *name = crypto_tfm_alg_name(tfm); 739 struct sahara_ctx *ctx = crypto_tfm_ctx(tfm); 740 741 ctx->fallback = crypto_alloc_ablkcipher(name, 0, 742 CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK); 743 if (IS_ERR(ctx->fallback)) { 744 pr_err("Error allocating fallback algo %s\n", name); 745 return PTR_ERR(ctx->fallback); 746 } 747 748 tfm->crt_ablkcipher.reqsize = sizeof(struct sahara_aes_reqctx); 749 750 return 0; 751 } 752 753 static void sahara_aes_cra_exit(struct crypto_tfm *tfm) 754 { 755 struct sahara_ctx *ctx = crypto_tfm_ctx(tfm); 756 757 if (ctx->fallback) 758 crypto_free_ablkcipher(ctx->fallback); 759 ctx->fallback = NULL; 760 } 761 762 static u32 sahara_sha_init_hdr(struct sahara_dev *dev, 763 struct sahara_sha_reqctx *rctx) 764 { 765 u32 hdr = 0; 766 767 hdr = rctx->mode; 768 769 if (rctx->first) { 770 hdr |= SAHARA_HDR_MDHA_SET_MODE_HASH; 771 hdr |= SAHARA_HDR_MDHA_INIT; 772 } else { 773 hdr |= SAHARA_HDR_MDHA_SET_MODE_MD_KEY; 774 } 775 776 if (rctx->last) 777 hdr |= SAHARA_HDR_MDHA_PDATA; 778 779 if (hweight_long(hdr) % 2 == 0) 780 hdr |= SAHARA_HDR_PARITY_BIT; 781 782 return hdr; 783 } 784 785 static int sahara_sha_hw_links_create(struct sahara_dev *dev, 786 struct sahara_sha_reqctx *rctx, 787 int start) 788 { 789 struct scatterlist *sg; 790 unsigned int i; 791 int ret; 792 793 dev->in_sg = rctx->in_sg; 794 795 dev->nb_in_sg = sg_nents_for_len(dev->in_sg, rctx->total); 796 if ((dev->nb_in_sg) > SAHARA_MAX_HW_LINK) { 797 dev_err(dev->device, "not enough hw links (%d)\n", 798 dev->nb_in_sg + dev->nb_out_sg); 799 return -EINVAL; 800 } 801 802 sg = dev->in_sg; 803 ret = dma_map_sg(dev->device, dev->in_sg, dev->nb_in_sg, DMA_TO_DEVICE); 804 if (!ret) 805 return -EFAULT; 806 807 for (i = start; i < dev->nb_in_sg + start; i++) { 808 dev->hw_link[i]->len = sg->length; 809 dev->hw_link[i]->p = sg->dma_address; 810 if (i == (dev->nb_in_sg + start - 1)) { 811 dev->hw_link[i]->next = 0; 812 } else { 813 dev->hw_link[i]->next = dev->hw_phys_link[i + 1]; 814 sg = sg_next(sg); 815 } 816 } 817 818 return i; 819 } 820 821 static int sahara_sha_hw_data_descriptor_create(struct sahara_dev *dev, 822 struct sahara_sha_reqctx *rctx, 823 struct ahash_request *req, 824 int index) 825 { 826 unsigned result_len; 827 int i = index; 828 829 if (rctx->first) 830 /* Create initial descriptor: #8*/ 831 dev->hw_desc[index]->hdr = sahara_sha_init_hdr(dev, rctx); 832 else 833 /* Create hash descriptor: #10. Must follow #6. */ 834 dev->hw_desc[index]->hdr = SAHARA_HDR_MDHA_HASH; 835 836 dev->hw_desc[index]->len1 = rctx->total; 837 if (dev->hw_desc[index]->len1 == 0) { 838 /* if len1 is 0, p1 must be 0, too */ 839 dev->hw_desc[index]->p1 = 0; 840 rctx->sg_in_idx = 0; 841 } else { 842 /* Create input links */ 843 dev->hw_desc[index]->p1 = dev->hw_phys_link[index]; 844 i = sahara_sha_hw_links_create(dev, rctx, index); 845 846 rctx->sg_in_idx = index; 847 if (i < 0) 848 return i; 849 } 850 851 dev->hw_desc[index]->p2 = dev->hw_phys_link[i]; 852 853 /* Save the context for the next operation */ 854 result_len = rctx->context_size; 855 dev->hw_link[i]->p = dev->context_phys_base; 856 857 dev->hw_link[i]->len = result_len; 858 dev->hw_desc[index]->len2 = result_len; 859 860 dev->hw_link[i]->next = 0; 861 862 return 0; 863 } 864 865 /* 866 * Load descriptor aka #6 867 * 868 * To load a previously saved context back to the MDHA unit 869 * 870 * p1: Saved Context 871 * p2: NULL 872 * 873 */ 874 static int sahara_sha_hw_context_descriptor_create(struct sahara_dev *dev, 875 struct sahara_sha_reqctx *rctx, 876 struct ahash_request *req, 877 int index) 878 { 879 dev->hw_desc[index]->hdr = sahara_sha_init_hdr(dev, rctx); 880 881 dev->hw_desc[index]->len1 = rctx->context_size; 882 dev->hw_desc[index]->p1 = dev->hw_phys_link[index]; 883 dev->hw_desc[index]->len2 = 0; 884 dev->hw_desc[index]->p2 = 0; 885 886 dev->hw_link[index]->len = rctx->context_size; 887 dev->hw_link[index]->p = dev->context_phys_base; 888 dev->hw_link[index]->next = 0; 889 890 return 0; 891 } 892 893 static int sahara_walk_and_recalc(struct scatterlist *sg, unsigned int nbytes) 894 { 895 if (!sg || !sg->length) 896 return nbytes; 897 898 while (nbytes && sg) { 899 if (nbytes <= sg->length) { 900 sg->length = nbytes; 901 sg_mark_end(sg); 902 break; 903 } 904 nbytes -= sg->length; 905 sg = sg_next(sg); 906 } 907 908 return nbytes; 909 } 910 911 static int sahara_sha_prepare_request(struct ahash_request *req) 912 { 913 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 914 struct sahara_sha_reqctx *rctx = ahash_request_ctx(req); 915 unsigned int hash_later; 916 unsigned int block_size; 917 unsigned int len; 918 919 block_size = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm)); 920 921 /* append bytes from previous operation */ 922 len = rctx->buf_cnt + req->nbytes; 923 924 /* only the last transfer can be padded in hardware */ 925 if (!rctx->last && (len < block_size)) { 926 /* to few data, save for next operation */ 927 scatterwalk_map_and_copy(rctx->buf + rctx->buf_cnt, req->src, 928 0, req->nbytes, 0); 929 rctx->buf_cnt += req->nbytes; 930 931 return 0; 932 } 933 934 /* add data from previous operation first */ 935 if (rctx->buf_cnt) 936 memcpy(rctx->rembuf, rctx->buf, rctx->buf_cnt); 937 938 /* data must always be a multiple of block_size */ 939 hash_later = rctx->last ? 0 : len & (block_size - 1); 940 if (hash_later) { 941 unsigned int offset = req->nbytes - hash_later; 942 /* Save remaining bytes for later use */ 943 scatterwalk_map_and_copy(rctx->buf, req->src, offset, 944 hash_later, 0); 945 } 946 947 /* nbytes should now be multiple of blocksize */ 948 req->nbytes = req->nbytes - hash_later; 949 950 sahara_walk_and_recalc(req->src, req->nbytes); 951 952 /* have data from previous operation and current */ 953 if (rctx->buf_cnt && req->nbytes) { 954 sg_init_table(rctx->in_sg_chain, 2); 955 sg_set_buf(rctx->in_sg_chain, rctx->rembuf, rctx->buf_cnt); 956 957 sg_chain(rctx->in_sg_chain, 2, req->src); 958 959 rctx->total = req->nbytes + rctx->buf_cnt; 960 rctx->in_sg = rctx->in_sg_chain; 961 962 req->src = rctx->in_sg_chain; 963 /* only data from previous operation */ 964 } else if (rctx->buf_cnt) { 965 if (req->src) 966 rctx->in_sg = req->src; 967 else 968 rctx->in_sg = rctx->in_sg_chain; 969 /* buf was copied into rembuf above */ 970 sg_init_one(rctx->in_sg, rctx->rembuf, rctx->buf_cnt); 971 rctx->total = rctx->buf_cnt; 972 /* no data from previous operation */ 973 } else { 974 rctx->in_sg = req->src; 975 rctx->total = req->nbytes; 976 req->src = rctx->in_sg; 977 } 978 979 /* on next call, we only have the remaining data in the buffer */ 980 rctx->buf_cnt = hash_later; 981 982 return -EINPROGRESS; 983 } 984 985 static int sahara_sha_process(struct ahash_request *req) 986 { 987 struct sahara_dev *dev = dev_ptr; 988 struct sahara_sha_reqctx *rctx = ahash_request_ctx(req); 989 int ret; 990 unsigned long timeout; 991 992 ret = sahara_sha_prepare_request(req); 993 if (!ret) 994 return ret; 995 996 if (rctx->first) { 997 sahara_sha_hw_data_descriptor_create(dev, rctx, req, 0); 998 dev->hw_desc[0]->next = 0; 999 rctx->first = 0; 1000 } else { 1001 memcpy(dev->context_base, rctx->context, rctx->context_size); 1002 1003 sahara_sha_hw_context_descriptor_create(dev, rctx, req, 0); 1004 dev->hw_desc[0]->next = dev->hw_phys_desc[1]; 1005 sahara_sha_hw_data_descriptor_create(dev, rctx, req, 1); 1006 dev->hw_desc[1]->next = 0; 1007 } 1008 1009 sahara_dump_descriptors(dev); 1010 sahara_dump_links(dev); 1011 1012 reinit_completion(&dev->dma_completion); 1013 1014 sahara_write(dev, dev->hw_phys_desc[0], SAHARA_REG_DAR); 1015 1016 timeout = wait_for_completion_timeout(&dev->dma_completion, 1017 msecs_to_jiffies(SAHARA_TIMEOUT_MS)); 1018 if (!timeout) { 1019 dev_err(dev->device, "SHA timeout\n"); 1020 return -ETIMEDOUT; 1021 } 1022 1023 if (rctx->sg_in_idx) 1024 dma_unmap_sg(dev->device, dev->in_sg, dev->nb_in_sg, 1025 DMA_TO_DEVICE); 1026 1027 memcpy(rctx->context, dev->context_base, rctx->context_size); 1028 1029 if (req->result) 1030 memcpy(req->result, rctx->context, rctx->digest_size); 1031 1032 return 0; 1033 } 1034 1035 static int sahara_queue_manage(void *data) 1036 { 1037 struct sahara_dev *dev = (struct sahara_dev *)data; 1038 struct crypto_async_request *async_req; 1039 struct crypto_async_request *backlog; 1040 int ret = 0; 1041 1042 do { 1043 __set_current_state(TASK_INTERRUPTIBLE); 1044 1045 mutex_lock(&dev->queue_mutex); 1046 backlog = crypto_get_backlog(&dev->queue); 1047 async_req = crypto_dequeue_request(&dev->queue); 1048 mutex_unlock(&dev->queue_mutex); 1049 1050 if (backlog) 1051 backlog->complete(backlog, -EINPROGRESS); 1052 1053 if (async_req) { 1054 if (crypto_tfm_alg_type(async_req->tfm) == 1055 CRYPTO_ALG_TYPE_AHASH) { 1056 struct ahash_request *req = 1057 ahash_request_cast(async_req); 1058 1059 ret = sahara_sha_process(req); 1060 } else { 1061 struct ablkcipher_request *req = 1062 ablkcipher_request_cast(async_req); 1063 1064 ret = sahara_aes_process(req); 1065 } 1066 1067 async_req->complete(async_req, ret); 1068 1069 continue; 1070 } 1071 1072 schedule(); 1073 } while (!kthread_should_stop()); 1074 1075 return 0; 1076 } 1077 1078 static int sahara_sha_enqueue(struct ahash_request *req, int last) 1079 { 1080 struct sahara_sha_reqctx *rctx = ahash_request_ctx(req); 1081 struct sahara_dev *dev = dev_ptr; 1082 int ret; 1083 1084 if (!req->nbytes && !last) 1085 return 0; 1086 1087 mutex_lock(&rctx->mutex); 1088 rctx->last = last; 1089 1090 if (!rctx->active) { 1091 rctx->active = 1; 1092 rctx->first = 1; 1093 } 1094 1095 mutex_lock(&dev->queue_mutex); 1096 ret = crypto_enqueue_request(&dev->queue, &req->base); 1097 mutex_unlock(&dev->queue_mutex); 1098 1099 wake_up_process(dev->kthread); 1100 mutex_unlock(&rctx->mutex); 1101 1102 return ret; 1103 } 1104 1105 static int sahara_sha_init(struct ahash_request *req) 1106 { 1107 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 1108 struct sahara_sha_reqctx *rctx = ahash_request_ctx(req); 1109 1110 memset(rctx, 0, sizeof(*rctx)); 1111 1112 switch (crypto_ahash_digestsize(tfm)) { 1113 case SHA1_DIGEST_SIZE: 1114 rctx->mode |= SAHARA_HDR_MDHA_ALG_SHA1; 1115 rctx->digest_size = SHA1_DIGEST_SIZE; 1116 break; 1117 case SHA256_DIGEST_SIZE: 1118 rctx->mode |= SAHARA_HDR_MDHA_ALG_SHA256; 1119 rctx->digest_size = SHA256_DIGEST_SIZE; 1120 break; 1121 default: 1122 return -EINVAL; 1123 } 1124 1125 rctx->context_size = rctx->digest_size + 4; 1126 rctx->active = 0; 1127 1128 mutex_init(&rctx->mutex); 1129 1130 return 0; 1131 } 1132 1133 static int sahara_sha_update(struct ahash_request *req) 1134 { 1135 return sahara_sha_enqueue(req, 0); 1136 } 1137 1138 static int sahara_sha_final(struct ahash_request *req) 1139 { 1140 req->nbytes = 0; 1141 return sahara_sha_enqueue(req, 1); 1142 } 1143 1144 static int sahara_sha_finup(struct ahash_request *req) 1145 { 1146 return sahara_sha_enqueue(req, 1); 1147 } 1148 1149 static int sahara_sha_digest(struct ahash_request *req) 1150 { 1151 sahara_sha_init(req); 1152 1153 return sahara_sha_finup(req); 1154 } 1155 1156 static int sahara_sha_export(struct ahash_request *req, void *out) 1157 { 1158 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 1159 struct sahara_ctx *ctx = crypto_ahash_ctx(ahash); 1160 struct sahara_sha_reqctx *rctx = ahash_request_ctx(req); 1161 1162 memcpy(out, ctx, sizeof(struct sahara_ctx)); 1163 memcpy(out + sizeof(struct sahara_sha_reqctx), rctx, 1164 sizeof(struct sahara_sha_reqctx)); 1165 1166 return 0; 1167 } 1168 1169 static int sahara_sha_import(struct ahash_request *req, const void *in) 1170 { 1171 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 1172 struct sahara_ctx *ctx = crypto_ahash_ctx(ahash); 1173 struct sahara_sha_reqctx *rctx = ahash_request_ctx(req); 1174 1175 memcpy(ctx, in, sizeof(struct sahara_ctx)); 1176 memcpy(rctx, in + sizeof(struct sahara_sha_reqctx), 1177 sizeof(struct sahara_sha_reqctx)); 1178 1179 return 0; 1180 } 1181 1182 static int sahara_sha_cra_init(struct crypto_tfm *tfm) 1183 { 1184 const char *name = crypto_tfm_alg_name(tfm); 1185 struct sahara_ctx *ctx = crypto_tfm_ctx(tfm); 1186 1187 ctx->shash_fallback = crypto_alloc_shash(name, 0, 1188 CRYPTO_ALG_NEED_FALLBACK); 1189 if (IS_ERR(ctx->shash_fallback)) { 1190 pr_err("Error allocating fallback algo %s\n", name); 1191 return PTR_ERR(ctx->shash_fallback); 1192 } 1193 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), 1194 sizeof(struct sahara_sha_reqctx) + 1195 SHA_BUFFER_LEN + SHA256_BLOCK_SIZE); 1196 1197 return 0; 1198 } 1199 1200 static void sahara_sha_cra_exit(struct crypto_tfm *tfm) 1201 { 1202 struct sahara_ctx *ctx = crypto_tfm_ctx(tfm); 1203 1204 crypto_free_shash(ctx->shash_fallback); 1205 ctx->shash_fallback = NULL; 1206 } 1207 1208 static struct crypto_alg aes_algs[] = { 1209 { 1210 .cra_name = "ecb(aes)", 1211 .cra_driver_name = "sahara-ecb-aes", 1212 .cra_priority = 300, 1213 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | 1214 CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, 1215 .cra_blocksize = AES_BLOCK_SIZE, 1216 .cra_ctxsize = sizeof(struct sahara_ctx), 1217 .cra_alignmask = 0x0, 1218 .cra_type = &crypto_ablkcipher_type, 1219 .cra_module = THIS_MODULE, 1220 .cra_init = sahara_aes_cra_init, 1221 .cra_exit = sahara_aes_cra_exit, 1222 .cra_u.ablkcipher = { 1223 .min_keysize = AES_MIN_KEY_SIZE , 1224 .max_keysize = AES_MAX_KEY_SIZE, 1225 .setkey = sahara_aes_setkey, 1226 .encrypt = sahara_aes_ecb_encrypt, 1227 .decrypt = sahara_aes_ecb_decrypt, 1228 } 1229 }, { 1230 .cra_name = "cbc(aes)", 1231 .cra_driver_name = "sahara-cbc-aes", 1232 .cra_priority = 300, 1233 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | 1234 CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, 1235 .cra_blocksize = AES_BLOCK_SIZE, 1236 .cra_ctxsize = sizeof(struct sahara_ctx), 1237 .cra_alignmask = 0x0, 1238 .cra_type = &crypto_ablkcipher_type, 1239 .cra_module = THIS_MODULE, 1240 .cra_init = sahara_aes_cra_init, 1241 .cra_exit = sahara_aes_cra_exit, 1242 .cra_u.ablkcipher = { 1243 .min_keysize = AES_MIN_KEY_SIZE , 1244 .max_keysize = AES_MAX_KEY_SIZE, 1245 .ivsize = AES_BLOCK_SIZE, 1246 .setkey = sahara_aes_setkey, 1247 .encrypt = sahara_aes_cbc_encrypt, 1248 .decrypt = sahara_aes_cbc_decrypt, 1249 } 1250 } 1251 }; 1252 1253 static struct ahash_alg sha_v3_algs[] = { 1254 { 1255 .init = sahara_sha_init, 1256 .update = sahara_sha_update, 1257 .final = sahara_sha_final, 1258 .finup = sahara_sha_finup, 1259 .digest = sahara_sha_digest, 1260 .export = sahara_sha_export, 1261 .import = sahara_sha_import, 1262 .halg.digestsize = SHA1_DIGEST_SIZE, 1263 .halg.base = { 1264 .cra_name = "sha1", 1265 .cra_driver_name = "sahara-sha1", 1266 .cra_priority = 300, 1267 .cra_flags = CRYPTO_ALG_TYPE_AHASH | 1268 CRYPTO_ALG_ASYNC | 1269 CRYPTO_ALG_NEED_FALLBACK, 1270 .cra_blocksize = SHA1_BLOCK_SIZE, 1271 .cra_ctxsize = sizeof(struct sahara_ctx), 1272 .cra_alignmask = 0, 1273 .cra_module = THIS_MODULE, 1274 .cra_init = sahara_sha_cra_init, 1275 .cra_exit = sahara_sha_cra_exit, 1276 } 1277 }, 1278 }; 1279 1280 static struct ahash_alg sha_v4_algs[] = { 1281 { 1282 .init = sahara_sha_init, 1283 .update = sahara_sha_update, 1284 .final = sahara_sha_final, 1285 .finup = sahara_sha_finup, 1286 .digest = sahara_sha_digest, 1287 .export = sahara_sha_export, 1288 .import = sahara_sha_import, 1289 .halg.digestsize = SHA256_DIGEST_SIZE, 1290 .halg.base = { 1291 .cra_name = "sha256", 1292 .cra_driver_name = "sahara-sha256", 1293 .cra_priority = 300, 1294 .cra_flags = CRYPTO_ALG_TYPE_AHASH | 1295 CRYPTO_ALG_ASYNC | 1296 CRYPTO_ALG_NEED_FALLBACK, 1297 .cra_blocksize = SHA256_BLOCK_SIZE, 1298 .cra_ctxsize = sizeof(struct sahara_ctx), 1299 .cra_alignmask = 0, 1300 .cra_module = THIS_MODULE, 1301 .cra_init = sahara_sha_cra_init, 1302 .cra_exit = sahara_sha_cra_exit, 1303 } 1304 }, 1305 }; 1306 1307 static irqreturn_t sahara_irq_handler(int irq, void *data) 1308 { 1309 struct sahara_dev *dev = (struct sahara_dev *)data; 1310 unsigned int stat = sahara_read(dev, SAHARA_REG_STATUS); 1311 unsigned int err = sahara_read(dev, SAHARA_REG_ERRSTATUS); 1312 1313 sahara_write(dev, SAHARA_CMD_CLEAR_INT | SAHARA_CMD_CLEAR_ERR, 1314 SAHARA_REG_CMD); 1315 1316 sahara_decode_status(dev, stat); 1317 1318 if (SAHARA_STATUS_GET_STATE(stat) == SAHARA_STATE_BUSY) { 1319 return IRQ_NONE; 1320 } else if (SAHARA_STATUS_GET_STATE(stat) == SAHARA_STATE_COMPLETE) { 1321 dev->error = 0; 1322 } else { 1323 sahara_decode_error(dev, err); 1324 dev->error = -EINVAL; 1325 } 1326 1327 complete(&dev->dma_completion); 1328 1329 return IRQ_HANDLED; 1330 } 1331 1332 1333 static int sahara_register_algs(struct sahara_dev *dev) 1334 { 1335 int err; 1336 unsigned int i, j, k, l; 1337 1338 for (i = 0; i < ARRAY_SIZE(aes_algs); i++) { 1339 INIT_LIST_HEAD(&aes_algs[i].cra_list); 1340 err = crypto_register_alg(&aes_algs[i]); 1341 if (err) 1342 goto err_aes_algs; 1343 } 1344 1345 for (k = 0; k < ARRAY_SIZE(sha_v3_algs); k++) { 1346 err = crypto_register_ahash(&sha_v3_algs[k]); 1347 if (err) 1348 goto err_sha_v3_algs; 1349 } 1350 1351 if (dev->version > SAHARA_VERSION_3) 1352 for (l = 0; l < ARRAY_SIZE(sha_v4_algs); l++) { 1353 err = crypto_register_ahash(&sha_v4_algs[l]); 1354 if (err) 1355 goto err_sha_v4_algs; 1356 } 1357 1358 return 0; 1359 1360 err_sha_v4_algs: 1361 for (j = 0; j < l; j++) 1362 crypto_unregister_ahash(&sha_v4_algs[j]); 1363 1364 err_sha_v3_algs: 1365 for (j = 0; j < k; j++) 1366 crypto_unregister_ahash(&sha_v4_algs[j]); 1367 1368 err_aes_algs: 1369 for (j = 0; j < i; j++) 1370 crypto_unregister_alg(&aes_algs[j]); 1371 1372 return err; 1373 } 1374 1375 static void sahara_unregister_algs(struct sahara_dev *dev) 1376 { 1377 unsigned int i; 1378 1379 for (i = 0; i < ARRAY_SIZE(aes_algs); i++) 1380 crypto_unregister_alg(&aes_algs[i]); 1381 1382 for (i = 0; i < ARRAY_SIZE(sha_v4_algs); i++) 1383 crypto_unregister_ahash(&sha_v3_algs[i]); 1384 1385 if (dev->version > SAHARA_VERSION_3) 1386 for (i = 0; i < ARRAY_SIZE(sha_v4_algs); i++) 1387 crypto_unregister_ahash(&sha_v4_algs[i]); 1388 } 1389 1390 static struct platform_device_id sahara_platform_ids[] = { 1391 { .name = "sahara-imx27" }, 1392 { /* sentinel */ } 1393 }; 1394 MODULE_DEVICE_TABLE(platform, sahara_platform_ids); 1395 1396 static struct of_device_id sahara_dt_ids[] = { 1397 { .compatible = "fsl,imx53-sahara" }, 1398 { .compatible = "fsl,imx27-sahara" }, 1399 { /* sentinel */ } 1400 }; 1401 MODULE_DEVICE_TABLE(of, sahara_dt_ids); 1402 1403 static int sahara_probe(struct platform_device *pdev) 1404 { 1405 struct sahara_dev *dev; 1406 struct resource *res; 1407 u32 version; 1408 int irq; 1409 int err; 1410 int i; 1411 1412 dev = devm_kzalloc(&pdev->dev, sizeof(struct sahara_dev), GFP_KERNEL); 1413 if (dev == NULL) { 1414 dev_err(&pdev->dev, "unable to alloc data struct.\n"); 1415 return -ENOMEM; 1416 } 1417 1418 dev->device = &pdev->dev; 1419 platform_set_drvdata(pdev, dev); 1420 1421 /* Get the base address */ 1422 res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 1423 dev->regs_base = devm_ioremap_resource(&pdev->dev, res); 1424 if (IS_ERR(dev->regs_base)) 1425 return PTR_ERR(dev->regs_base); 1426 1427 /* Get the IRQ */ 1428 irq = platform_get_irq(pdev, 0); 1429 if (irq < 0) { 1430 dev_err(&pdev->dev, "failed to get irq resource\n"); 1431 return irq; 1432 } 1433 1434 err = devm_request_irq(&pdev->dev, irq, sahara_irq_handler, 1435 0, dev_name(&pdev->dev), dev); 1436 if (err) { 1437 dev_err(&pdev->dev, "failed to request irq\n"); 1438 return err; 1439 } 1440 1441 /* clocks */ 1442 dev->clk_ipg = devm_clk_get(&pdev->dev, "ipg"); 1443 if (IS_ERR(dev->clk_ipg)) { 1444 dev_err(&pdev->dev, "Could not get ipg clock\n"); 1445 return PTR_ERR(dev->clk_ipg); 1446 } 1447 1448 dev->clk_ahb = devm_clk_get(&pdev->dev, "ahb"); 1449 if (IS_ERR(dev->clk_ahb)) { 1450 dev_err(&pdev->dev, "Could not get ahb clock\n"); 1451 return PTR_ERR(dev->clk_ahb); 1452 } 1453 1454 /* Allocate HW descriptors */ 1455 dev->hw_desc[0] = dmam_alloc_coherent(&pdev->dev, 1456 SAHARA_MAX_HW_DESC * sizeof(struct sahara_hw_desc), 1457 &dev->hw_phys_desc[0], GFP_KERNEL); 1458 if (!dev->hw_desc[0]) { 1459 dev_err(&pdev->dev, "Could not allocate hw descriptors\n"); 1460 return -ENOMEM; 1461 } 1462 dev->hw_desc[1] = dev->hw_desc[0] + 1; 1463 dev->hw_phys_desc[1] = dev->hw_phys_desc[0] + 1464 sizeof(struct sahara_hw_desc); 1465 1466 /* Allocate space for iv and key */ 1467 dev->key_base = dmam_alloc_coherent(&pdev->dev, 2 * AES_KEYSIZE_128, 1468 &dev->key_phys_base, GFP_KERNEL); 1469 if (!dev->key_base) { 1470 dev_err(&pdev->dev, "Could not allocate memory for key\n"); 1471 return -ENOMEM; 1472 } 1473 dev->iv_base = dev->key_base + AES_KEYSIZE_128; 1474 dev->iv_phys_base = dev->key_phys_base + AES_KEYSIZE_128; 1475 1476 /* Allocate space for context: largest digest + message length field */ 1477 dev->context_base = dmam_alloc_coherent(&pdev->dev, 1478 SHA256_DIGEST_SIZE + 4, 1479 &dev->context_phys_base, GFP_KERNEL); 1480 if (!dev->context_base) { 1481 dev_err(&pdev->dev, "Could not allocate memory for MDHA context\n"); 1482 return -ENOMEM; 1483 } 1484 1485 /* Allocate space for HW links */ 1486 dev->hw_link[0] = dmam_alloc_coherent(&pdev->dev, 1487 SAHARA_MAX_HW_LINK * sizeof(struct sahara_hw_link), 1488 &dev->hw_phys_link[0], GFP_KERNEL); 1489 if (!dev->hw_link[0]) { 1490 dev_err(&pdev->dev, "Could not allocate hw links\n"); 1491 return -ENOMEM; 1492 } 1493 for (i = 1; i < SAHARA_MAX_HW_LINK; i++) { 1494 dev->hw_phys_link[i] = dev->hw_phys_link[i - 1] + 1495 sizeof(struct sahara_hw_link); 1496 dev->hw_link[i] = dev->hw_link[i - 1] + 1; 1497 } 1498 1499 crypto_init_queue(&dev->queue, SAHARA_QUEUE_LENGTH); 1500 1501 spin_lock_init(&dev->lock); 1502 mutex_init(&dev->queue_mutex); 1503 1504 dev_ptr = dev; 1505 1506 dev->kthread = kthread_run(sahara_queue_manage, dev, "sahara_crypto"); 1507 if (IS_ERR(dev->kthread)) { 1508 return PTR_ERR(dev->kthread); 1509 } 1510 1511 init_completion(&dev->dma_completion); 1512 1513 err = clk_prepare_enable(dev->clk_ipg); 1514 if (err) 1515 return err; 1516 err = clk_prepare_enable(dev->clk_ahb); 1517 if (err) 1518 goto clk_ipg_disable; 1519 1520 version = sahara_read(dev, SAHARA_REG_VERSION); 1521 if (of_device_is_compatible(pdev->dev.of_node, "fsl,imx27-sahara")) { 1522 if (version != SAHARA_VERSION_3) 1523 err = -ENODEV; 1524 } else if (of_device_is_compatible(pdev->dev.of_node, 1525 "fsl,imx53-sahara")) { 1526 if (((version >> 8) & 0xff) != SAHARA_VERSION_4) 1527 err = -ENODEV; 1528 version = (version >> 8) & 0xff; 1529 } 1530 if (err == -ENODEV) { 1531 dev_err(&pdev->dev, "SAHARA version %d not supported\n", 1532 version); 1533 goto err_algs; 1534 } 1535 1536 dev->version = version; 1537 1538 sahara_write(dev, SAHARA_CMD_RESET | SAHARA_CMD_MODE_BATCH, 1539 SAHARA_REG_CMD); 1540 sahara_write(dev, SAHARA_CONTROL_SET_THROTTLE(0) | 1541 SAHARA_CONTROL_SET_MAXBURST(8) | 1542 SAHARA_CONTROL_RNG_AUTORSD | 1543 SAHARA_CONTROL_ENABLE_INT, 1544 SAHARA_REG_CONTROL); 1545 1546 err = sahara_register_algs(dev); 1547 if (err) 1548 goto err_algs; 1549 1550 dev_info(&pdev->dev, "SAHARA version %d initialized\n", version); 1551 1552 return 0; 1553 1554 err_algs: 1555 kthread_stop(dev->kthread); 1556 dev_ptr = NULL; 1557 clk_disable_unprepare(dev->clk_ahb); 1558 clk_ipg_disable: 1559 clk_disable_unprepare(dev->clk_ipg); 1560 1561 return err; 1562 } 1563 1564 static int sahara_remove(struct platform_device *pdev) 1565 { 1566 struct sahara_dev *dev = platform_get_drvdata(pdev); 1567 1568 kthread_stop(dev->kthread); 1569 1570 sahara_unregister_algs(dev); 1571 1572 clk_disable_unprepare(dev->clk_ipg); 1573 clk_disable_unprepare(dev->clk_ahb); 1574 1575 dev_ptr = NULL; 1576 1577 return 0; 1578 } 1579 1580 static struct platform_driver sahara_driver = { 1581 .probe = sahara_probe, 1582 .remove = sahara_remove, 1583 .driver = { 1584 .name = SAHARA_NAME, 1585 .of_match_table = sahara_dt_ids, 1586 }, 1587 .id_table = sahara_platform_ids, 1588 }; 1589 1590 module_platform_driver(sahara_driver); 1591 1592 MODULE_LICENSE("GPL"); 1593 MODULE_AUTHOR("Javier Martin <javier.martin@vista-silicon.com>"); 1594 MODULE_AUTHOR("Steffen Trumtrar <s.trumtrar@pengutronix.de>"); 1595 MODULE_DESCRIPTION("SAHARA2 HW crypto accelerator"); 1596