1 /** 2 * AMCC SoC PPC4xx Crypto Driver 3 * 4 * Copyright (c) 2008 Applied Micro Circuits Corporation. 5 * All rights reserved. James Hsiao <jhsiao@amcc.com> 6 * 7 * This program is free software; you can redistribute it and/or modify 8 * it under the terms of the GNU General Public License as published by 9 * the Free Software Foundation; either version 2 of the License, or 10 * (at your option) any later version. 11 * 12 * This program is distributed in the hope that it will be useful, 13 * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 * GNU General Public License for more details. 16 * 17 * This file implements AMCC crypto offload Linux device driver for use with 18 * Linux CryptoAPI. 19 */ 20 21 #include <linux/kernel.h> 22 #include <linux/interrupt.h> 23 #include <linux/spinlock_types.h> 24 #include <linux/random.h> 25 #include <linux/scatterlist.h> 26 #include <linux/crypto.h> 27 #include <linux/dma-mapping.h> 28 #include <linux/platform_device.h> 29 #include <linux/init.h> 30 #include <linux/of_platform.h> 31 #include <asm/dcr.h> 32 #include <asm/dcr-regs.h> 33 #include <asm/cacheflush.h> 34 #include <crypto/aes.h> 35 #include <crypto/sha.h> 36 #include "crypto4xx_reg_def.h" 37 #include "crypto4xx_core.h" 38 #include "crypto4xx_sa.h" 39 40 #define PPC4XX_SEC_VERSION_STR "0.5" 41 42 /** 43 * PPC4xx Crypto Engine Initialization Routine 44 */ 45 static void crypto4xx_hw_init(struct crypto4xx_device *dev) 46 { 47 union ce_ring_size ring_size; 48 union ce_ring_contol ring_ctrl; 49 union ce_part_ring_size part_ring_size; 50 union ce_io_threshold io_threshold; 51 u32 rand_num; 52 union ce_pe_dma_cfg pe_dma_cfg; 53 54 writel(PPC4XX_BYTE_ORDER, dev->ce_base + CRYPTO4XX_BYTE_ORDER_CFG); 55 /* setup pe dma, include reset sg, pdr and pe, then release reset */ 56 pe_dma_cfg.w = 0; 57 pe_dma_cfg.bf.bo_sgpd_en = 1; 58 pe_dma_cfg.bf.bo_data_en = 0; 59 pe_dma_cfg.bf.bo_sa_en = 1; 60 pe_dma_cfg.bf.bo_pd_en = 1; 61 pe_dma_cfg.bf.dynamic_sa_en = 1; 62 pe_dma_cfg.bf.reset_sg = 1; 63 pe_dma_cfg.bf.reset_pdr = 1; 64 pe_dma_cfg.bf.reset_pe = 1; 65 writel(pe_dma_cfg.w, dev->ce_base + CRYPTO4XX_PE_DMA_CFG); 66 /* un reset pe,sg and pdr */ 67 pe_dma_cfg.bf.pe_mode = 0; 68 pe_dma_cfg.bf.reset_sg = 0; 69 pe_dma_cfg.bf.reset_pdr = 0; 70 pe_dma_cfg.bf.reset_pe = 0; 71 pe_dma_cfg.bf.bo_td_en = 0; 72 writel(pe_dma_cfg.w, dev->ce_base + CRYPTO4XX_PE_DMA_CFG); 73 writel(dev->pdr_pa, dev->ce_base + CRYPTO4XX_PDR_BASE); 74 writel(dev->pdr_pa, dev->ce_base + CRYPTO4XX_RDR_BASE); 75 writel(PPC4XX_PRNG_CTRL_AUTO_EN, dev->ce_base + CRYPTO4XX_PRNG_CTRL); 76 get_random_bytes(&rand_num, sizeof(rand_num)); 77 writel(rand_num, dev->ce_base + CRYPTO4XX_PRNG_SEED_L); 78 get_random_bytes(&rand_num, sizeof(rand_num)); 79 writel(rand_num, dev->ce_base + CRYPTO4XX_PRNG_SEED_H); 80 ring_size.w = 0; 81 ring_size.bf.ring_offset = PPC4XX_PD_SIZE; 82 ring_size.bf.ring_size = PPC4XX_NUM_PD; 83 writel(ring_size.w, dev->ce_base + CRYPTO4XX_RING_SIZE); 84 ring_ctrl.w = 0; 85 writel(ring_ctrl.w, dev->ce_base + CRYPTO4XX_RING_CTRL); 86 writel(PPC4XX_DC_3DES_EN, dev->ce_base + CRYPTO4XX_DEVICE_CTRL); 87 writel(dev->gdr_pa, dev->ce_base + CRYPTO4XX_GATH_RING_BASE); 88 writel(dev->sdr_pa, dev->ce_base + CRYPTO4XX_SCAT_RING_BASE); 89 part_ring_size.w = 0; 90 part_ring_size.bf.sdr_size = PPC4XX_SDR_SIZE; 91 part_ring_size.bf.gdr_size = PPC4XX_GDR_SIZE; 92 writel(part_ring_size.w, dev->ce_base + CRYPTO4XX_PART_RING_SIZE); 93 writel(PPC4XX_SD_BUFFER_SIZE, dev->ce_base + CRYPTO4XX_PART_RING_CFG); 94 io_threshold.w = 0; 95 io_threshold.bf.output_threshold = PPC4XX_OUTPUT_THRESHOLD; 96 io_threshold.bf.input_threshold = PPC4XX_INPUT_THRESHOLD; 97 writel(io_threshold.w, dev->ce_base + CRYPTO4XX_IO_THRESHOLD); 98 writel(0, dev->ce_base + CRYPTO4XX_PDR_BASE_UADDR); 99 writel(0, dev->ce_base + CRYPTO4XX_RDR_BASE_UADDR); 100 writel(0, dev->ce_base + CRYPTO4XX_PKT_SRC_UADDR); 101 writel(0, dev->ce_base + CRYPTO4XX_PKT_DEST_UADDR); 102 writel(0, dev->ce_base + CRYPTO4XX_SA_UADDR); 103 writel(0, dev->ce_base + CRYPTO4XX_GATH_RING_BASE_UADDR); 104 writel(0, dev->ce_base + CRYPTO4XX_SCAT_RING_BASE_UADDR); 105 /* un reset pe,sg and pdr */ 106 pe_dma_cfg.bf.pe_mode = 1; 107 pe_dma_cfg.bf.reset_sg = 0; 108 pe_dma_cfg.bf.reset_pdr = 0; 109 pe_dma_cfg.bf.reset_pe = 0; 110 pe_dma_cfg.bf.bo_td_en = 0; 111 writel(pe_dma_cfg.w, dev->ce_base + CRYPTO4XX_PE_DMA_CFG); 112 /*clear all pending interrupt*/ 113 writel(PPC4XX_INTERRUPT_CLR, dev->ce_base + CRYPTO4XX_INT_CLR); 114 writel(PPC4XX_INT_DESCR_CNT, dev->ce_base + CRYPTO4XX_INT_DESCR_CNT); 115 writel(PPC4XX_INT_DESCR_CNT, dev->ce_base + CRYPTO4XX_INT_DESCR_CNT); 116 writel(PPC4XX_INT_CFG, dev->ce_base + CRYPTO4XX_INT_CFG); 117 writel(PPC4XX_PD_DONE_INT, dev->ce_base + CRYPTO4XX_INT_EN); 118 } 119 120 int crypto4xx_alloc_sa(struct crypto4xx_ctx *ctx, u32 size) 121 { 122 ctx->sa_in = dma_alloc_coherent(ctx->dev->core_dev->device, size * 4, 123 &ctx->sa_in_dma_addr, GFP_ATOMIC); 124 if (ctx->sa_in == NULL) 125 return -ENOMEM; 126 127 ctx->sa_out = dma_alloc_coherent(ctx->dev->core_dev->device, size * 4, 128 &ctx->sa_out_dma_addr, GFP_ATOMIC); 129 if (ctx->sa_out == NULL) { 130 dma_free_coherent(ctx->dev->core_dev->device, 131 ctx->sa_len * 4, 132 ctx->sa_in, ctx->sa_in_dma_addr); 133 return -ENOMEM; 134 } 135 136 memset(ctx->sa_in, 0, size * 4); 137 memset(ctx->sa_out, 0, size * 4); 138 ctx->sa_len = size; 139 140 return 0; 141 } 142 143 void crypto4xx_free_sa(struct crypto4xx_ctx *ctx) 144 { 145 if (ctx->sa_in != NULL) 146 dma_free_coherent(ctx->dev->core_dev->device, ctx->sa_len * 4, 147 ctx->sa_in, ctx->sa_in_dma_addr); 148 if (ctx->sa_out != NULL) 149 dma_free_coherent(ctx->dev->core_dev->device, ctx->sa_len * 4, 150 ctx->sa_out, ctx->sa_out_dma_addr); 151 152 ctx->sa_in_dma_addr = 0; 153 ctx->sa_out_dma_addr = 0; 154 ctx->sa_len = 0; 155 } 156 157 u32 crypto4xx_alloc_state_record(struct crypto4xx_ctx *ctx) 158 { 159 ctx->state_record = dma_alloc_coherent(ctx->dev->core_dev->device, 160 sizeof(struct sa_state_record), 161 &ctx->state_record_dma_addr, GFP_ATOMIC); 162 if (!ctx->state_record_dma_addr) 163 return -ENOMEM; 164 memset(ctx->state_record, 0, sizeof(struct sa_state_record)); 165 166 return 0; 167 } 168 169 void crypto4xx_free_state_record(struct crypto4xx_ctx *ctx) 170 { 171 if (ctx->state_record != NULL) 172 dma_free_coherent(ctx->dev->core_dev->device, 173 sizeof(struct sa_state_record), 174 ctx->state_record, 175 ctx->state_record_dma_addr); 176 ctx->state_record_dma_addr = 0; 177 } 178 179 /** 180 * alloc memory for the gather ring 181 * no need to alloc buf for the ring 182 * gdr_tail, gdr_head and gdr_count are initialized by this function 183 */ 184 static u32 crypto4xx_build_pdr(struct crypto4xx_device *dev) 185 { 186 int i; 187 struct pd_uinfo *pd_uinfo; 188 dev->pdr = dma_alloc_coherent(dev->core_dev->device, 189 sizeof(struct ce_pd) * PPC4XX_NUM_PD, 190 &dev->pdr_pa, GFP_ATOMIC); 191 if (!dev->pdr) 192 return -ENOMEM; 193 194 dev->pdr_uinfo = kzalloc(sizeof(struct pd_uinfo) * PPC4XX_NUM_PD, 195 GFP_KERNEL); 196 if (!dev->pdr_uinfo) { 197 dma_free_coherent(dev->core_dev->device, 198 sizeof(struct ce_pd) * PPC4XX_NUM_PD, 199 dev->pdr, 200 dev->pdr_pa); 201 return -ENOMEM; 202 } 203 memset(dev->pdr, 0, sizeof(struct ce_pd) * PPC4XX_NUM_PD); 204 dev->shadow_sa_pool = dma_alloc_coherent(dev->core_dev->device, 205 256 * PPC4XX_NUM_PD, 206 &dev->shadow_sa_pool_pa, 207 GFP_ATOMIC); 208 if (!dev->shadow_sa_pool) 209 return -ENOMEM; 210 211 dev->shadow_sr_pool = dma_alloc_coherent(dev->core_dev->device, 212 sizeof(struct sa_state_record) * PPC4XX_NUM_PD, 213 &dev->shadow_sr_pool_pa, GFP_ATOMIC); 214 if (!dev->shadow_sr_pool) 215 return -ENOMEM; 216 for (i = 0; i < PPC4XX_NUM_PD; i++) { 217 pd_uinfo = (struct pd_uinfo *) (dev->pdr_uinfo + 218 sizeof(struct pd_uinfo) * i); 219 220 /* alloc 256 bytes which is enough for any kind of dynamic sa */ 221 pd_uinfo->sa_va = dev->shadow_sa_pool + 256 * i; 222 pd_uinfo->sa_pa = dev->shadow_sa_pool_pa + 256 * i; 223 224 /* alloc state record */ 225 pd_uinfo->sr_va = dev->shadow_sr_pool + 226 sizeof(struct sa_state_record) * i; 227 pd_uinfo->sr_pa = dev->shadow_sr_pool_pa + 228 sizeof(struct sa_state_record) * i; 229 } 230 231 return 0; 232 } 233 234 static void crypto4xx_destroy_pdr(struct crypto4xx_device *dev) 235 { 236 if (dev->pdr != NULL) 237 dma_free_coherent(dev->core_dev->device, 238 sizeof(struct ce_pd) * PPC4XX_NUM_PD, 239 dev->pdr, dev->pdr_pa); 240 if (dev->shadow_sa_pool) 241 dma_free_coherent(dev->core_dev->device, 256 * PPC4XX_NUM_PD, 242 dev->shadow_sa_pool, dev->shadow_sa_pool_pa); 243 if (dev->shadow_sr_pool) 244 dma_free_coherent(dev->core_dev->device, 245 sizeof(struct sa_state_record) * PPC4XX_NUM_PD, 246 dev->shadow_sr_pool, dev->shadow_sr_pool_pa); 247 248 kfree(dev->pdr_uinfo); 249 } 250 251 static u32 crypto4xx_get_pd_from_pdr_nolock(struct crypto4xx_device *dev) 252 { 253 u32 retval; 254 u32 tmp; 255 256 retval = dev->pdr_head; 257 tmp = (dev->pdr_head + 1) % PPC4XX_NUM_PD; 258 259 if (tmp == dev->pdr_tail) 260 return ERING_WAS_FULL; 261 262 dev->pdr_head = tmp; 263 264 return retval; 265 } 266 267 static u32 crypto4xx_put_pd_to_pdr(struct crypto4xx_device *dev, u32 idx) 268 { 269 struct pd_uinfo *pd_uinfo; 270 unsigned long flags; 271 272 pd_uinfo = (struct pd_uinfo *)(dev->pdr_uinfo + 273 sizeof(struct pd_uinfo) * idx); 274 spin_lock_irqsave(&dev->core_dev->lock, flags); 275 if (dev->pdr_tail != PPC4XX_LAST_PD) 276 dev->pdr_tail++; 277 else 278 dev->pdr_tail = 0; 279 pd_uinfo->state = PD_ENTRY_FREE; 280 spin_unlock_irqrestore(&dev->core_dev->lock, flags); 281 282 return 0; 283 } 284 285 static struct ce_pd *crypto4xx_get_pdp(struct crypto4xx_device *dev, 286 dma_addr_t *pd_dma, u32 idx) 287 { 288 *pd_dma = dev->pdr_pa + sizeof(struct ce_pd) * idx; 289 290 return dev->pdr + sizeof(struct ce_pd) * idx; 291 } 292 293 /** 294 * alloc memory for the gather ring 295 * no need to alloc buf for the ring 296 * gdr_tail, gdr_head and gdr_count are initialized by this function 297 */ 298 static u32 crypto4xx_build_gdr(struct crypto4xx_device *dev) 299 { 300 dev->gdr = dma_alloc_coherent(dev->core_dev->device, 301 sizeof(struct ce_gd) * PPC4XX_NUM_GD, 302 &dev->gdr_pa, GFP_ATOMIC); 303 if (!dev->gdr) 304 return -ENOMEM; 305 306 memset(dev->gdr, 0, sizeof(struct ce_gd) * PPC4XX_NUM_GD); 307 308 return 0; 309 } 310 311 static inline void crypto4xx_destroy_gdr(struct crypto4xx_device *dev) 312 { 313 dma_free_coherent(dev->core_dev->device, 314 sizeof(struct ce_gd) * PPC4XX_NUM_GD, 315 dev->gdr, dev->gdr_pa); 316 } 317 318 /* 319 * when this function is called. 320 * preemption or interrupt must be disabled 321 */ 322 u32 crypto4xx_get_n_gd(struct crypto4xx_device *dev, int n) 323 { 324 u32 retval; 325 u32 tmp; 326 if (n >= PPC4XX_NUM_GD) 327 return ERING_WAS_FULL; 328 329 retval = dev->gdr_head; 330 tmp = (dev->gdr_head + n) % PPC4XX_NUM_GD; 331 if (dev->gdr_head > dev->gdr_tail) { 332 if (tmp < dev->gdr_head && tmp >= dev->gdr_tail) 333 return ERING_WAS_FULL; 334 } else if (dev->gdr_head < dev->gdr_tail) { 335 if (tmp < dev->gdr_head || tmp >= dev->gdr_tail) 336 return ERING_WAS_FULL; 337 } 338 dev->gdr_head = tmp; 339 340 return retval; 341 } 342 343 static u32 crypto4xx_put_gd_to_gdr(struct crypto4xx_device *dev) 344 { 345 unsigned long flags; 346 347 spin_lock_irqsave(&dev->core_dev->lock, flags); 348 if (dev->gdr_tail == dev->gdr_head) { 349 spin_unlock_irqrestore(&dev->core_dev->lock, flags); 350 return 0; 351 } 352 353 if (dev->gdr_tail != PPC4XX_LAST_GD) 354 dev->gdr_tail++; 355 else 356 dev->gdr_tail = 0; 357 358 spin_unlock_irqrestore(&dev->core_dev->lock, flags); 359 360 return 0; 361 } 362 363 static inline struct ce_gd *crypto4xx_get_gdp(struct crypto4xx_device *dev, 364 dma_addr_t *gd_dma, u32 idx) 365 { 366 *gd_dma = dev->gdr_pa + sizeof(struct ce_gd) * idx; 367 368 return (struct ce_gd *) (dev->gdr + sizeof(struct ce_gd) * idx); 369 } 370 371 /** 372 * alloc memory for the scatter ring 373 * need to alloc buf for the ring 374 * sdr_tail, sdr_head and sdr_count are initialized by this function 375 */ 376 static u32 crypto4xx_build_sdr(struct crypto4xx_device *dev) 377 { 378 int i; 379 struct ce_sd *sd_array; 380 381 /* alloc memory for scatter descriptor ring */ 382 dev->sdr = dma_alloc_coherent(dev->core_dev->device, 383 sizeof(struct ce_sd) * PPC4XX_NUM_SD, 384 &dev->sdr_pa, GFP_ATOMIC); 385 if (!dev->sdr) 386 return -ENOMEM; 387 388 dev->scatter_buffer_size = PPC4XX_SD_BUFFER_SIZE; 389 dev->scatter_buffer_va = 390 dma_alloc_coherent(dev->core_dev->device, 391 dev->scatter_buffer_size * PPC4XX_NUM_SD, 392 &dev->scatter_buffer_pa, GFP_ATOMIC); 393 if (!dev->scatter_buffer_va) { 394 dma_free_coherent(dev->core_dev->device, 395 sizeof(struct ce_sd) * PPC4XX_NUM_SD, 396 dev->sdr, dev->sdr_pa); 397 return -ENOMEM; 398 } 399 400 sd_array = dev->sdr; 401 402 for (i = 0; i < PPC4XX_NUM_SD; i++) { 403 sd_array[i].ptr = dev->scatter_buffer_pa + 404 dev->scatter_buffer_size * i; 405 } 406 407 return 0; 408 } 409 410 static void crypto4xx_destroy_sdr(struct crypto4xx_device *dev) 411 { 412 if (dev->sdr != NULL) 413 dma_free_coherent(dev->core_dev->device, 414 sizeof(struct ce_sd) * PPC4XX_NUM_SD, 415 dev->sdr, dev->sdr_pa); 416 417 if (dev->scatter_buffer_va != NULL) 418 dma_free_coherent(dev->core_dev->device, 419 dev->scatter_buffer_size * PPC4XX_NUM_SD, 420 dev->scatter_buffer_va, 421 dev->scatter_buffer_pa); 422 } 423 424 /* 425 * when this function is called. 426 * preemption or interrupt must be disabled 427 */ 428 static u32 crypto4xx_get_n_sd(struct crypto4xx_device *dev, int n) 429 { 430 u32 retval; 431 u32 tmp; 432 433 if (n >= PPC4XX_NUM_SD) 434 return ERING_WAS_FULL; 435 436 retval = dev->sdr_head; 437 tmp = (dev->sdr_head + n) % PPC4XX_NUM_SD; 438 if (dev->sdr_head > dev->gdr_tail) { 439 if (tmp < dev->sdr_head && tmp >= dev->sdr_tail) 440 return ERING_WAS_FULL; 441 } else if (dev->sdr_head < dev->sdr_tail) { 442 if (tmp < dev->sdr_head || tmp >= dev->sdr_tail) 443 return ERING_WAS_FULL; 444 } /* the head = tail, or empty case is already take cared */ 445 dev->sdr_head = tmp; 446 447 return retval; 448 } 449 450 static u32 crypto4xx_put_sd_to_sdr(struct crypto4xx_device *dev) 451 { 452 unsigned long flags; 453 454 spin_lock_irqsave(&dev->core_dev->lock, flags); 455 if (dev->sdr_tail == dev->sdr_head) { 456 spin_unlock_irqrestore(&dev->core_dev->lock, flags); 457 return 0; 458 } 459 if (dev->sdr_tail != PPC4XX_LAST_SD) 460 dev->sdr_tail++; 461 else 462 dev->sdr_tail = 0; 463 spin_unlock_irqrestore(&dev->core_dev->lock, flags); 464 465 return 0; 466 } 467 468 static inline struct ce_sd *crypto4xx_get_sdp(struct crypto4xx_device *dev, 469 dma_addr_t *sd_dma, u32 idx) 470 { 471 *sd_dma = dev->sdr_pa + sizeof(struct ce_sd) * idx; 472 473 return (struct ce_sd *)(dev->sdr + sizeof(struct ce_sd) * idx); 474 } 475 476 static u32 crypto4xx_fill_one_page(struct crypto4xx_device *dev, 477 dma_addr_t *addr, u32 *length, 478 u32 *idx, u32 *offset, u32 *nbytes) 479 { 480 u32 len; 481 482 if (*length > dev->scatter_buffer_size) { 483 memcpy(phys_to_virt(*addr), 484 dev->scatter_buffer_va + 485 *idx * dev->scatter_buffer_size + *offset, 486 dev->scatter_buffer_size); 487 *offset = 0; 488 *length -= dev->scatter_buffer_size; 489 *nbytes -= dev->scatter_buffer_size; 490 if (*idx == PPC4XX_LAST_SD) 491 *idx = 0; 492 else 493 (*idx)++; 494 *addr = *addr + dev->scatter_buffer_size; 495 return 1; 496 } else if (*length < dev->scatter_buffer_size) { 497 memcpy(phys_to_virt(*addr), 498 dev->scatter_buffer_va + 499 *idx * dev->scatter_buffer_size + *offset, *length); 500 if ((*offset + *length) == dev->scatter_buffer_size) { 501 if (*idx == PPC4XX_LAST_SD) 502 *idx = 0; 503 else 504 (*idx)++; 505 *nbytes -= *length; 506 *offset = 0; 507 } else { 508 *nbytes -= *length; 509 *offset += *length; 510 } 511 512 return 0; 513 } else { 514 len = (*nbytes <= dev->scatter_buffer_size) ? 515 (*nbytes) : dev->scatter_buffer_size; 516 memcpy(phys_to_virt(*addr), 517 dev->scatter_buffer_va + 518 *idx * dev->scatter_buffer_size + *offset, 519 len); 520 *offset = 0; 521 *nbytes -= len; 522 523 if (*idx == PPC4XX_LAST_SD) 524 *idx = 0; 525 else 526 (*idx)++; 527 528 return 0; 529 } 530 } 531 532 static void crypto4xx_copy_pkt_to_dst(struct crypto4xx_device *dev, 533 struct ce_pd *pd, 534 struct pd_uinfo *pd_uinfo, 535 u32 nbytes, 536 struct scatterlist *dst) 537 { 538 dma_addr_t addr; 539 u32 this_sd; 540 u32 offset; 541 u32 len; 542 u32 i; 543 u32 sg_len; 544 struct scatterlist *sg; 545 546 this_sd = pd_uinfo->first_sd; 547 offset = 0; 548 i = 0; 549 550 while (nbytes) { 551 sg = &dst[i]; 552 sg_len = sg->length; 553 addr = dma_map_page(dev->core_dev->device, sg_page(sg), 554 sg->offset, sg->length, DMA_TO_DEVICE); 555 556 if (offset == 0) { 557 len = (nbytes <= sg->length) ? nbytes : sg->length; 558 while (crypto4xx_fill_one_page(dev, &addr, &len, 559 &this_sd, &offset, &nbytes)) 560 ; 561 if (!nbytes) 562 return; 563 i++; 564 } else { 565 len = (nbytes <= (dev->scatter_buffer_size - offset)) ? 566 nbytes : (dev->scatter_buffer_size - offset); 567 len = (sg->length < len) ? sg->length : len; 568 while (crypto4xx_fill_one_page(dev, &addr, &len, 569 &this_sd, &offset, &nbytes)) 570 ; 571 if (!nbytes) 572 return; 573 sg_len -= len; 574 if (sg_len) { 575 addr += len; 576 while (crypto4xx_fill_one_page(dev, &addr, 577 &sg_len, &this_sd, &offset, &nbytes)) 578 ; 579 } 580 i++; 581 } 582 } 583 } 584 585 static u32 crypto4xx_copy_digest_to_dst(struct pd_uinfo *pd_uinfo, 586 struct crypto4xx_ctx *ctx) 587 { 588 struct dynamic_sa_ctl *sa = (struct dynamic_sa_ctl *) ctx->sa_in; 589 struct sa_state_record *state_record = 590 (struct sa_state_record *) pd_uinfo->sr_va; 591 592 if (sa->sa_command_0.bf.hash_alg == SA_HASH_ALG_SHA1) { 593 memcpy((void *) pd_uinfo->dest_va, state_record->save_digest, 594 SA_HASH_ALG_SHA1_DIGEST_SIZE); 595 } 596 597 return 0; 598 } 599 600 static void crypto4xx_ret_sg_desc(struct crypto4xx_device *dev, 601 struct pd_uinfo *pd_uinfo) 602 { 603 int i; 604 if (pd_uinfo->num_gd) { 605 for (i = 0; i < pd_uinfo->num_gd; i++) 606 crypto4xx_put_gd_to_gdr(dev); 607 pd_uinfo->first_gd = 0xffffffff; 608 pd_uinfo->num_gd = 0; 609 } 610 if (pd_uinfo->num_sd) { 611 for (i = 0; i < pd_uinfo->num_sd; i++) 612 crypto4xx_put_sd_to_sdr(dev); 613 614 pd_uinfo->first_sd = 0xffffffff; 615 pd_uinfo->num_sd = 0; 616 } 617 } 618 619 static u32 crypto4xx_ablkcipher_done(struct crypto4xx_device *dev, 620 struct pd_uinfo *pd_uinfo, 621 struct ce_pd *pd) 622 { 623 struct crypto4xx_ctx *ctx; 624 struct ablkcipher_request *ablk_req; 625 struct scatterlist *dst; 626 dma_addr_t addr; 627 628 ablk_req = ablkcipher_request_cast(pd_uinfo->async_req); 629 ctx = crypto_tfm_ctx(ablk_req->base.tfm); 630 631 if (pd_uinfo->using_sd) { 632 crypto4xx_copy_pkt_to_dst(dev, pd, pd_uinfo, ablk_req->nbytes, 633 ablk_req->dst); 634 } else { 635 dst = pd_uinfo->dest_va; 636 addr = dma_map_page(dev->core_dev->device, sg_page(dst), 637 dst->offset, dst->length, DMA_FROM_DEVICE); 638 } 639 crypto4xx_ret_sg_desc(dev, pd_uinfo); 640 if (ablk_req->base.complete != NULL) 641 ablk_req->base.complete(&ablk_req->base, 0); 642 643 return 0; 644 } 645 646 static u32 crypto4xx_ahash_done(struct crypto4xx_device *dev, 647 struct pd_uinfo *pd_uinfo) 648 { 649 struct crypto4xx_ctx *ctx; 650 struct ahash_request *ahash_req; 651 652 ahash_req = ahash_request_cast(pd_uinfo->async_req); 653 ctx = crypto_tfm_ctx(ahash_req->base.tfm); 654 655 crypto4xx_copy_digest_to_dst(pd_uinfo, 656 crypto_tfm_ctx(ahash_req->base.tfm)); 657 crypto4xx_ret_sg_desc(dev, pd_uinfo); 658 /* call user provided callback function x */ 659 if (ahash_req->base.complete != NULL) 660 ahash_req->base.complete(&ahash_req->base, 0); 661 662 return 0; 663 } 664 665 static u32 crypto4xx_pd_done(struct crypto4xx_device *dev, u32 idx) 666 { 667 struct ce_pd *pd; 668 struct pd_uinfo *pd_uinfo; 669 670 pd = dev->pdr + sizeof(struct ce_pd)*idx; 671 pd_uinfo = dev->pdr_uinfo + sizeof(struct pd_uinfo)*idx; 672 if (crypto_tfm_alg_type(pd_uinfo->async_req->tfm) == 673 CRYPTO_ALG_TYPE_ABLKCIPHER) 674 return crypto4xx_ablkcipher_done(dev, pd_uinfo, pd); 675 else 676 return crypto4xx_ahash_done(dev, pd_uinfo); 677 } 678 679 /** 680 * Note: Only use this function to copy items that is word aligned. 681 */ 682 void crypto4xx_memcpy_le(unsigned int *dst, 683 const unsigned char *buf, 684 int len) 685 { 686 u8 *tmp; 687 for (; len >= 4; buf += 4, len -= 4) 688 *dst++ = cpu_to_le32(*(unsigned int *) buf); 689 690 tmp = (u8 *)dst; 691 switch (len) { 692 case 3: 693 *tmp++ = 0; 694 *tmp++ = *(buf+2); 695 *tmp++ = *(buf+1); 696 *tmp++ = *buf; 697 break; 698 case 2: 699 *tmp++ = 0; 700 *tmp++ = 0; 701 *tmp++ = *(buf+1); 702 *tmp++ = *buf; 703 break; 704 case 1: 705 *tmp++ = 0; 706 *tmp++ = 0; 707 *tmp++ = 0; 708 *tmp++ = *buf; 709 break; 710 default: 711 break; 712 } 713 } 714 715 static void crypto4xx_stop_all(struct crypto4xx_core_device *core_dev) 716 { 717 crypto4xx_destroy_pdr(core_dev->dev); 718 crypto4xx_destroy_gdr(core_dev->dev); 719 crypto4xx_destroy_sdr(core_dev->dev); 720 dev_set_drvdata(core_dev->device, NULL); 721 iounmap(core_dev->dev->ce_base); 722 kfree(core_dev->dev); 723 kfree(core_dev); 724 } 725 726 void crypto4xx_return_pd(struct crypto4xx_device *dev, 727 u32 pd_entry, struct ce_pd *pd, 728 struct pd_uinfo *pd_uinfo) 729 { 730 /* irq should be already disabled */ 731 dev->pdr_head = pd_entry; 732 pd->pd_ctl.w = 0; 733 pd->pd_ctl_len.w = 0; 734 pd_uinfo->state = PD_ENTRY_FREE; 735 } 736 737 /* 738 * derive number of elements in scatterlist 739 * Shamlessly copy from talitos.c 740 */ 741 static int get_sg_count(struct scatterlist *sg_list, int nbytes) 742 { 743 struct scatterlist *sg = sg_list; 744 int sg_nents = 0; 745 746 while (nbytes) { 747 sg_nents++; 748 if (sg->length > nbytes) 749 break; 750 nbytes -= sg->length; 751 sg = sg_next(sg); 752 } 753 754 return sg_nents; 755 } 756 757 static u32 get_next_gd(u32 current) 758 { 759 if (current != PPC4XX_LAST_GD) 760 return current + 1; 761 else 762 return 0; 763 } 764 765 static u32 get_next_sd(u32 current) 766 { 767 if (current != PPC4XX_LAST_SD) 768 return current + 1; 769 else 770 return 0; 771 } 772 773 u32 crypto4xx_build_pd(struct crypto_async_request *req, 774 struct crypto4xx_ctx *ctx, 775 struct scatterlist *src, 776 struct scatterlist *dst, 777 unsigned int datalen, 778 void *iv, u32 iv_len) 779 { 780 struct crypto4xx_device *dev = ctx->dev; 781 dma_addr_t addr, pd_dma, sd_dma, gd_dma; 782 struct dynamic_sa_ctl *sa; 783 struct scatterlist *sg; 784 struct ce_gd *gd; 785 struct ce_pd *pd; 786 u32 num_gd, num_sd; 787 u32 fst_gd = 0xffffffff; 788 u32 fst_sd = 0xffffffff; 789 u32 pd_entry; 790 unsigned long flags; 791 struct pd_uinfo *pd_uinfo = NULL; 792 unsigned int nbytes = datalen, idx; 793 unsigned int ivlen = 0; 794 u32 gd_idx = 0; 795 796 /* figure how many gd is needed */ 797 num_gd = get_sg_count(src, datalen); 798 if (num_gd == 1) 799 num_gd = 0; 800 801 /* figure how many sd is needed */ 802 if (sg_is_last(dst) || ctx->is_hash) { 803 num_sd = 0; 804 } else { 805 if (datalen > PPC4XX_SD_BUFFER_SIZE) { 806 num_sd = datalen / PPC4XX_SD_BUFFER_SIZE; 807 if (datalen % PPC4XX_SD_BUFFER_SIZE) 808 num_sd++; 809 } else { 810 num_sd = 1; 811 } 812 } 813 814 /* 815 * The follow section of code needs to be protected 816 * The gather ring and scatter ring needs to be consecutive 817 * In case of run out of any kind of descriptor, the descriptor 818 * already got must be return the original place. 819 */ 820 spin_lock_irqsave(&dev->core_dev->lock, flags); 821 if (num_gd) { 822 fst_gd = crypto4xx_get_n_gd(dev, num_gd); 823 if (fst_gd == ERING_WAS_FULL) { 824 spin_unlock_irqrestore(&dev->core_dev->lock, flags); 825 return -EAGAIN; 826 } 827 } 828 if (num_sd) { 829 fst_sd = crypto4xx_get_n_sd(dev, num_sd); 830 if (fst_sd == ERING_WAS_FULL) { 831 if (num_gd) 832 dev->gdr_head = fst_gd; 833 spin_unlock_irqrestore(&dev->core_dev->lock, flags); 834 return -EAGAIN; 835 } 836 } 837 pd_entry = crypto4xx_get_pd_from_pdr_nolock(dev); 838 if (pd_entry == ERING_WAS_FULL) { 839 if (num_gd) 840 dev->gdr_head = fst_gd; 841 if (num_sd) 842 dev->sdr_head = fst_sd; 843 spin_unlock_irqrestore(&dev->core_dev->lock, flags); 844 return -EAGAIN; 845 } 846 spin_unlock_irqrestore(&dev->core_dev->lock, flags); 847 848 pd_uinfo = (struct pd_uinfo *)(dev->pdr_uinfo + 849 sizeof(struct pd_uinfo) * pd_entry); 850 pd = crypto4xx_get_pdp(dev, &pd_dma, pd_entry); 851 pd_uinfo->async_req = req; 852 pd_uinfo->num_gd = num_gd; 853 pd_uinfo->num_sd = num_sd; 854 855 if (iv_len || ctx->is_hash) { 856 ivlen = iv_len; 857 pd->sa = pd_uinfo->sa_pa; 858 sa = (struct dynamic_sa_ctl *) pd_uinfo->sa_va; 859 if (ctx->direction == DIR_INBOUND) 860 memcpy(sa, ctx->sa_in, ctx->sa_len * 4); 861 else 862 memcpy(sa, ctx->sa_out, ctx->sa_len * 4); 863 864 memcpy((void *) sa + ctx->offset_to_sr_ptr, 865 &pd_uinfo->sr_pa, 4); 866 867 if (iv_len) 868 crypto4xx_memcpy_le(pd_uinfo->sr_va, iv, iv_len); 869 } else { 870 if (ctx->direction == DIR_INBOUND) { 871 pd->sa = ctx->sa_in_dma_addr; 872 sa = (struct dynamic_sa_ctl *) ctx->sa_in; 873 } else { 874 pd->sa = ctx->sa_out_dma_addr; 875 sa = (struct dynamic_sa_ctl *) ctx->sa_out; 876 } 877 } 878 pd->sa_len = ctx->sa_len; 879 if (num_gd) { 880 /* get first gd we are going to use */ 881 gd_idx = fst_gd; 882 pd_uinfo->first_gd = fst_gd; 883 pd_uinfo->num_gd = num_gd; 884 gd = crypto4xx_get_gdp(dev, &gd_dma, gd_idx); 885 pd->src = gd_dma; 886 /* enable gather */ 887 sa->sa_command_0.bf.gather = 1; 888 idx = 0; 889 src = &src[0]; 890 /* walk the sg, and setup gather array */ 891 while (nbytes) { 892 sg = &src[idx]; 893 addr = dma_map_page(dev->core_dev->device, sg_page(sg), 894 sg->offset, sg->length, DMA_TO_DEVICE); 895 gd->ptr = addr; 896 gd->ctl_len.len = sg->length; 897 gd->ctl_len.done = 0; 898 gd->ctl_len.ready = 1; 899 if (sg->length >= nbytes) 900 break; 901 nbytes -= sg->length; 902 gd_idx = get_next_gd(gd_idx); 903 gd = crypto4xx_get_gdp(dev, &gd_dma, gd_idx); 904 idx++; 905 } 906 } else { 907 pd->src = (u32)dma_map_page(dev->core_dev->device, sg_page(src), 908 src->offset, src->length, DMA_TO_DEVICE); 909 /* 910 * Disable gather in sa command 911 */ 912 sa->sa_command_0.bf.gather = 0; 913 /* 914 * Indicate gather array is not used 915 */ 916 pd_uinfo->first_gd = 0xffffffff; 917 pd_uinfo->num_gd = 0; 918 } 919 if (ctx->is_hash || sg_is_last(dst)) { 920 /* 921 * we know application give us dst a whole piece of memory 922 * no need to use scatter ring. 923 * In case of is_hash, the icv is always at end of src data. 924 */ 925 pd_uinfo->using_sd = 0; 926 pd_uinfo->first_sd = 0xffffffff; 927 pd_uinfo->num_sd = 0; 928 pd_uinfo->dest_va = dst; 929 sa->sa_command_0.bf.scatter = 0; 930 if (ctx->is_hash) 931 pd->dest = virt_to_phys((void *)dst); 932 else 933 pd->dest = (u32)dma_map_page(dev->core_dev->device, 934 sg_page(dst), dst->offset, 935 dst->length, DMA_TO_DEVICE); 936 } else { 937 struct ce_sd *sd = NULL; 938 u32 sd_idx = fst_sd; 939 nbytes = datalen; 940 sa->sa_command_0.bf.scatter = 1; 941 pd_uinfo->using_sd = 1; 942 pd_uinfo->dest_va = dst; 943 pd_uinfo->first_sd = fst_sd; 944 pd_uinfo->num_sd = num_sd; 945 sd = crypto4xx_get_sdp(dev, &sd_dma, sd_idx); 946 pd->dest = sd_dma; 947 /* setup scatter descriptor */ 948 sd->ctl.done = 0; 949 sd->ctl.rdy = 1; 950 /* sd->ptr should be setup by sd_init routine*/ 951 idx = 0; 952 if (nbytes >= PPC4XX_SD_BUFFER_SIZE) 953 nbytes -= PPC4XX_SD_BUFFER_SIZE; 954 else 955 nbytes = 0; 956 while (nbytes) { 957 sd_idx = get_next_sd(sd_idx); 958 sd = crypto4xx_get_sdp(dev, &sd_dma, sd_idx); 959 /* setup scatter descriptor */ 960 sd->ctl.done = 0; 961 sd->ctl.rdy = 1; 962 if (nbytes >= PPC4XX_SD_BUFFER_SIZE) 963 nbytes -= PPC4XX_SD_BUFFER_SIZE; 964 else 965 /* 966 * SD entry can hold PPC4XX_SD_BUFFER_SIZE, 967 * which is more than nbytes, so done. 968 */ 969 nbytes = 0; 970 } 971 } 972 973 sa->sa_command_1.bf.hash_crypto_offset = 0; 974 pd->pd_ctl.w = ctx->pd_ctl; 975 pd->pd_ctl_len.w = 0x00400000 | (ctx->bypass << 24) | datalen; 976 pd_uinfo->state = PD_ENTRY_INUSE; 977 wmb(); 978 /* write any value to push engine to read a pd */ 979 writel(1, dev->ce_base + CRYPTO4XX_INT_DESCR_RD); 980 return -EINPROGRESS; 981 } 982 983 /** 984 * Algorithm Registration Functions 985 */ 986 static int crypto4xx_alg_init(struct crypto_tfm *tfm) 987 { 988 struct crypto_alg *alg = tfm->__crt_alg; 989 struct crypto4xx_alg *amcc_alg = crypto_alg_to_crypto4xx_alg(alg); 990 struct crypto4xx_ctx *ctx = crypto_tfm_ctx(tfm); 991 992 ctx->dev = amcc_alg->dev; 993 ctx->sa_in = NULL; 994 ctx->sa_out = NULL; 995 ctx->sa_in_dma_addr = 0; 996 ctx->sa_out_dma_addr = 0; 997 ctx->sa_len = 0; 998 999 switch (alg->cra_flags & CRYPTO_ALG_TYPE_MASK) { 1000 default: 1001 tfm->crt_ablkcipher.reqsize = sizeof(struct crypto4xx_ctx); 1002 break; 1003 case CRYPTO_ALG_TYPE_AHASH: 1004 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), 1005 sizeof(struct crypto4xx_ctx)); 1006 break; 1007 } 1008 1009 return 0; 1010 } 1011 1012 static void crypto4xx_alg_exit(struct crypto_tfm *tfm) 1013 { 1014 struct crypto4xx_ctx *ctx = crypto_tfm_ctx(tfm); 1015 1016 crypto4xx_free_sa(ctx); 1017 crypto4xx_free_state_record(ctx); 1018 } 1019 1020 int crypto4xx_register_alg(struct crypto4xx_device *sec_dev, 1021 struct crypto4xx_alg_common *crypto_alg, 1022 int array_size) 1023 { 1024 struct crypto4xx_alg *alg; 1025 int i; 1026 int rc = 0; 1027 1028 for (i = 0; i < array_size; i++) { 1029 alg = kzalloc(sizeof(struct crypto4xx_alg), GFP_KERNEL); 1030 if (!alg) 1031 return -ENOMEM; 1032 1033 alg->alg = crypto_alg[i]; 1034 alg->dev = sec_dev; 1035 1036 switch (alg->alg.type) { 1037 case CRYPTO_ALG_TYPE_AHASH: 1038 rc = crypto_register_ahash(&alg->alg.u.hash); 1039 break; 1040 1041 default: 1042 rc = crypto_register_alg(&alg->alg.u.cipher); 1043 break; 1044 } 1045 1046 if (rc) { 1047 list_del(&alg->entry); 1048 kfree(alg); 1049 } else { 1050 list_add_tail(&alg->entry, &sec_dev->alg_list); 1051 } 1052 } 1053 1054 return 0; 1055 } 1056 1057 static void crypto4xx_unregister_alg(struct crypto4xx_device *sec_dev) 1058 { 1059 struct crypto4xx_alg *alg, *tmp; 1060 1061 list_for_each_entry_safe(alg, tmp, &sec_dev->alg_list, entry) { 1062 list_del(&alg->entry); 1063 switch (alg->alg.type) { 1064 case CRYPTO_ALG_TYPE_AHASH: 1065 crypto_unregister_ahash(&alg->alg.u.hash); 1066 break; 1067 1068 default: 1069 crypto_unregister_alg(&alg->alg.u.cipher); 1070 } 1071 kfree(alg); 1072 } 1073 } 1074 1075 static void crypto4xx_bh_tasklet_cb(unsigned long data) 1076 { 1077 struct device *dev = (struct device *)data; 1078 struct crypto4xx_core_device *core_dev = dev_get_drvdata(dev); 1079 struct pd_uinfo *pd_uinfo; 1080 struct ce_pd *pd; 1081 u32 tail; 1082 1083 while (core_dev->dev->pdr_head != core_dev->dev->pdr_tail) { 1084 tail = core_dev->dev->pdr_tail; 1085 pd_uinfo = core_dev->dev->pdr_uinfo + 1086 sizeof(struct pd_uinfo)*tail; 1087 pd = core_dev->dev->pdr + sizeof(struct ce_pd) * tail; 1088 if ((pd_uinfo->state == PD_ENTRY_INUSE) && 1089 pd->pd_ctl.bf.pe_done && 1090 !pd->pd_ctl.bf.host_ready) { 1091 pd->pd_ctl.bf.pe_done = 0; 1092 crypto4xx_pd_done(core_dev->dev, tail); 1093 crypto4xx_put_pd_to_pdr(core_dev->dev, tail); 1094 pd_uinfo->state = PD_ENTRY_FREE; 1095 } else { 1096 /* if tail not done, break */ 1097 break; 1098 } 1099 } 1100 } 1101 1102 /** 1103 * Top Half of isr. 1104 */ 1105 static irqreturn_t crypto4xx_ce_interrupt_handler(int irq, void *data) 1106 { 1107 struct device *dev = (struct device *)data; 1108 struct crypto4xx_core_device *core_dev = dev_get_drvdata(dev); 1109 1110 if (core_dev->dev->ce_base == 0) 1111 return 0; 1112 1113 writel(PPC4XX_INTERRUPT_CLR, 1114 core_dev->dev->ce_base + CRYPTO4XX_INT_CLR); 1115 tasklet_schedule(&core_dev->tasklet); 1116 1117 return IRQ_HANDLED; 1118 } 1119 1120 /** 1121 * Supported Crypto Algorithms 1122 */ 1123 struct crypto4xx_alg_common crypto4xx_alg[] = { 1124 /* Crypto AES modes */ 1125 { .type = CRYPTO_ALG_TYPE_ABLKCIPHER, .u.cipher = { 1126 .cra_name = "cbc(aes)", 1127 .cra_driver_name = "cbc-aes-ppc4xx", 1128 .cra_priority = CRYPTO4XX_CRYPTO_PRIORITY, 1129 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, 1130 .cra_blocksize = AES_BLOCK_SIZE, 1131 .cra_ctxsize = sizeof(struct crypto4xx_ctx), 1132 .cra_type = &crypto_ablkcipher_type, 1133 .cra_init = crypto4xx_alg_init, 1134 .cra_exit = crypto4xx_alg_exit, 1135 .cra_module = THIS_MODULE, 1136 .cra_u = { 1137 .ablkcipher = { 1138 .min_keysize = AES_MIN_KEY_SIZE, 1139 .max_keysize = AES_MAX_KEY_SIZE, 1140 .ivsize = AES_IV_SIZE, 1141 .setkey = crypto4xx_setkey_aes_cbc, 1142 .encrypt = crypto4xx_encrypt, 1143 .decrypt = crypto4xx_decrypt, 1144 } 1145 } 1146 }}, 1147 }; 1148 1149 /** 1150 * Module Initialization Routine 1151 */ 1152 static int __init crypto4xx_probe(struct of_device *ofdev, 1153 const struct of_device_id *match) 1154 { 1155 int rc; 1156 struct resource res; 1157 struct device *dev = &ofdev->dev; 1158 struct crypto4xx_core_device *core_dev; 1159 1160 rc = of_address_to_resource(ofdev->node, 0, &res); 1161 if (rc) 1162 return -ENODEV; 1163 1164 if (of_find_compatible_node(NULL, NULL, "amcc,ppc460ex-crypto")) { 1165 mtdcri(SDR0, PPC460EX_SDR0_SRST, 1166 mfdcri(SDR0, PPC460EX_SDR0_SRST) | PPC460EX_CE_RESET); 1167 mtdcri(SDR0, PPC460EX_SDR0_SRST, 1168 mfdcri(SDR0, PPC460EX_SDR0_SRST) & ~PPC460EX_CE_RESET); 1169 } else if (of_find_compatible_node(NULL, NULL, 1170 "amcc,ppc405ex-crypto")) { 1171 mtdcri(SDR0, PPC405EX_SDR0_SRST, 1172 mfdcri(SDR0, PPC405EX_SDR0_SRST) | PPC405EX_CE_RESET); 1173 mtdcri(SDR0, PPC405EX_SDR0_SRST, 1174 mfdcri(SDR0, PPC405EX_SDR0_SRST) & ~PPC405EX_CE_RESET); 1175 } else if (of_find_compatible_node(NULL, NULL, 1176 "amcc,ppc460sx-crypto")) { 1177 mtdcri(SDR0, PPC460SX_SDR0_SRST, 1178 mfdcri(SDR0, PPC460SX_SDR0_SRST) | PPC460SX_CE_RESET); 1179 mtdcri(SDR0, PPC460SX_SDR0_SRST, 1180 mfdcri(SDR0, PPC460SX_SDR0_SRST) & ~PPC460SX_CE_RESET); 1181 } else { 1182 printk(KERN_ERR "Crypto Function Not supported!\n"); 1183 return -EINVAL; 1184 } 1185 1186 core_dev = kzalloc(sizeof(struct crypto4xx_core_device), GFP_KERNEL); 1187 if (!core_dev) 1188 return -ENOMEM; 1189 1190 dev_set_drvdata(dev, core_dev); 1191 core_dev->ofdev = ofdev; 1192 core_dev->dev = kzalloc(sizeof(struct crypto4xx_device), GFP_KERNEL); 1193 if (!core_dev->dev) 1194 goto err_alloc_dev; 1195 1196 core_dev->dev->core_dev = core_dev; 1197 core_dev->device = dev; 1198 spin_lock_init(&core_dev->lock); 1199 INIT_LIST_HEAD(&core_dev->dev->alg_list); 1200 rc = crypto4xx_build_pdr(core_dev->dev); 1201 if (rc) 1202 goto err_build_pdr; 1203 1204 rc = crypto4xx_build_gdr(core_dev->dev); 1205 if (rc) 1206 goto err_build_gdr; 1207 1208 rc = crypto4xx_build_sdr(core_dev->dev); 1209 if (rc) 1210 goto err_build_sdr; 1211 1212 /* Init tasklet for bottom half processing */ 1213 tasklet_init(&core_dev->tasklet, crypto4xx_bh_tasklet_cb, 1214 (unsigned long) dev); 1215 1216 /* Register for Crypto isr, Crypto Engine IRQ */ 1217 core_dev->irq = irq_of_parse_and_map(ofdev->node, 0); 1218 rc = request_irq(core_dev->irq, crypto4xx_ce_interrupt_handler, 0, 1219 core_dev->dev->name, dev); 1220 if (rc) 1221 goto err_request_irq; 1222 1223 core_dev->dev->ce_base = of_iomap(ofdev->node, 0); 1224 if (!core_dev->dev->ce_base) { 1225 dev_err(dev, "failed to of_iomap\n"); 1226 goto err_iomap; 1227 } 1228 1229 /* need to setup pdr, rdr, gdr and sdr before this */ 1230 crypto4xx_hw_init(core_dev->dev); 1231 1232 /* Register security algorithms with Linux CryptoAPI */ 1233 rc = crypto4xx_register_alg(core_dev->dev, crypto4xx_alg, 1234 ARRAY_SIZE(crypto4xx_alg)); 1235 if (rc) 1236 goto err_start_dev; 1237 1238 return 0; 1239 1240 err_start_dev: 1241 iounmap(core_dev->dev->ce_base); 1242 err_iomap: 1243 free_irq(core_dev->irq, dev); 1244 irq_dispose_mapping(core_dev->irq); 1245 tasklet_kill(&core_dev->tasklet); 1246 err_request_irq: 1247 crypto4xx_destroy_sdr(core_dev->dev); 1248 err_build_sdr: 1249 crypto4xx_destroy_gdr(core_dev->dev); 1250 err_build_gdr: 1251 crypto4xx_destroy_pdr(core_dev->dev); 1252 err_build_pdr: 1253 kfree(core_dev->dev); 1254 err_alloc_dev: 1255 kfree(core_dev); 1256 1257 return rc; 1258 } 1259 1260 static int __exit crypto4xx_remove(struct of_device *ofdev) 1261 { 1262 struct device *dev = &ofdev->dev; 1263 struct crypto4xx_core_device *core_dev = dev_get_drvdata(dev); 1264 1265 free_irq(core_dev->irq, dev); 1266 irq_dispose_mapping(core_dev->irq); 1267 1268 tasklet_kill(&core_dev->tasklet); 1269 /* Un-register with Linux CryptoAPI */ 1270 crypto4xx_unregister_alg(core_dev->dev); 1271 /* Free all allocated memory */ 1272 crypto4xx_stop_all(core_dev); 1273 1274 return 0; 1275 } 1276 1277 static struct of_device_id crypto4xx_match[] = { 1278 { .compatible = "amcc,ppc4xx-crypto",}, 1279 { }, 1280 }; 1281 1282 static struct of_platform_driver crypto4xx_driver = { 1283 .name = "crypto4xx", 1284 .match_table = crypto4xx_match, 1285 .probe = crypto4xx_probe, 1286 .remove = crypto4xx_remove, 1287 }; 1288 1289 static int __init crypto4xx_init(void) 1290 { 1291 return of_register_platform_driver(&crypto4xx_driver); 1292 } 1293 1294 static void __exit crypto4xx_exit(void) 1295 { 1296 of_unregister_platform_driver(&crypto4xx_driver); 1297 } 1298 1299 module_init(crypto4xx_init); 1300 module_exit(crypto4xx_exit); 1301 1302 MODULE_LICENSE("GPL"); 1303 MODULE_AUTHOR("James Hsiao <jhsiao@amcc.com>"); 1304 MODULE_DESCRIPTION("Driver for AMCC PPC4xx crypto accelerator"); 1305 1306