1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * talitos - Freescale Integrated Security Engine (SEC) device driver 4 * 5 * Copyright (c) 2008-2011 Freescale Semiconductor, Inc. 6 * 7 * Scatterlist Crypto API glue code copied from files with the following: 8 * Copyright (c) 2006-2007 Herbert Xu <herbert@gondor.apana.org.au> 9 * 10 * Crypto algorithm registration code copied from hifn driver: 11 * 2007+ Copyright (c) Evgeniy Polyakov <johnpol@2ka.mipt.ru> 12 * All rights reserved. 13 */ 14 15 #include <linux/kernel.h> 16 #include <linux/module.h> 17 #include <linux/mod_devicetable.h> 18 #include <linux/device.h> 19 #include <linux/interrupt.h> 20 #include <linux/crypto.h> 21 #include <linux/hw_random.h> 22 #include <linux/of_address.h> 23 #include <linux/of_irq.h> 24 #include <linux/of_platform.h> 25 #include <linux/dma-mapping.h> 26 #include <linux/io.h> 27 #include <linux/spinlock.h> 28 #include <linux/rtnetlink.h> 29 #include <linux/slab.h> 30 31 #include <crypto/algapi.h> 32 #include <crypto/aes.h> 33 #include <crypto/internal/des.h> 34 #include <crypto/sha1.h> 35 #include <crypto/sha2.h> 36 #include <crypto/md5.h> 37 #include <crypto/internal/aead.h> 38 #include <crypto/authenc.h> 39 #include <crypto/internal/skcipher.h> 40 #include <crypto/hash.h> 41 #include <crypto/internal/hash.h> 42 #include <crypto/scatterwalk.h> 43 44 #include "talitos.h" 45 46 static void to_talitos_ptr(struct talitos_ptr *ptr, dma_addr_t dma_addr, 47 unsigned int len, bool is_sec1) 48 { 49 ptr->ptr = cpu_to_be32(lower_32_bits(dma_addr)); 50 if (is_sec1) { 51 ptr->len1 = cpu_to_be16(len); 52 } else { 53 ptr->len = cpu_to_be16(len); 54 ptr->eptr = upper_32_bits(dma_addr); 55 } 56 } 57 58 static void copy_talitos_ptr(struct talitos_ptr *dst_ptr, 59 struct talitos_ptr *src_ptr, bool is_sec1) 60 { 61 dst_ptr->ptr = src_ptr->ptr; 62 if (is_sec1) { 63 dst_ptr->len1 = src_ptr->len1; 64 } else { 65 dst_ptr->len = src_ptr->len; 66 dst_ptr->eptr = src_ptr->eptr; 67 } 68 } 69 70 static unsigned short from_talitos_ptr_len(struct talitos_ptr *ptr, 71 bool is_sec1) 72 { 73 if (is_sec1) 74 return be16_to_cpu(ptr->len1); 75 else 76 return be16_to_cpu(ptr->len); 77 } 78 79 static void to_talitos_ptr_ext_set(struct talitos_ptr *ptr, u8 val, 80 bool is_sec1) 81 { 82 if (!is_sec1) 83 ptr->j_extent = val; 84 } 85 86 static void to_talitos_ptr_ext_or(struct talitos_ptr *ptr, u8 val, bool is_sec1) 87 { 88 if (!is_sec1) 89 ptr->j_extent |= val; 90 } 91 92 /* 93 * map virtual single (contiguous) pointer to h/w descriptor pointer 94 */ 95 static void __map_single_talitos_ptr(struct device *dev, 96 struct talitos_ptr *ptr, 97 unsigned int len, void *data, 98 enum dma_data_direction dir, 99 unsigned long attrs) 100 { 101 dma_addr_t dma_addr = dma_map_single_attrs(dev, data, len, dir, attrs); 102 struct talitos_private *priv = dev_get_drvdata(dev); 103 bool is_sec1 = has_ftr_sec1(priv); 104 105 to_talitos_ptr(ptr, dma_addr, len, is_sec1); 106 } 107 108 static void map_single_talitos_ptr(struct device *dev, 109 struct talitos_ptr *ptr, 110 unsigned int len, void *data, 111 enum dma_data_direction dir) 112 { 113 __map_single_talitos_ptr(dev, ptr, len, data, dir, 0); 114 } 115 116 static void map_single_talitos_ptr_nosync(struct device *dev, 117 struct talitos_ptr *ptr, 118 unsigned int len, void *data, 119 enum dma_data_direction dir) 120 { 121 __map_single_talitos_ptr(dev, ptr, len, data, dir, 122 DMA_ATTR_SKIP_CPU_SYNC); 123 } 124 125 /* 126 * unmap bus single (contiguous) h/w descriptor pointer 127 */ 128 static void unmap_single_talitos_ptr(struct device *dev, 129 struct talitos_ptr *ptr, 130 enum dma_data_direction dir) 131 { 132 struct talitos_private *priv = dev_get_drvdata(dev); 133 bool is_sec1 = has_ftr_sec1(priv); 134 135 dma_unmap_single(dev, be32_to_cpu(ptr->ptr), 136 from_talitos_ptr_len(ptr, is_sec1), dir); 137 } 138 139 static int reset_channel(struct device *dev, int ch) 140 { 141 struct talitos_private *priv = dev_get_drvdata(dev); 142 unsigned int timeout = TALITOS_TIMEOUT; 143 bool is_sec1 = has_ftr_sec1(priv); 144 145 if (is_sec1) { 146 setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO, 147 TALITOS1_CCCR_LO_RESET); 148 149 while ((in_be32(priv->chan[ch].reg + TALITOS_CCCR_LO) & 150 TALITOS1_CCCR_LO_RESET) && --timeout) 151 cpu_relax(); 152 } else { 153 setbits32(priv->chan[ch].reg + TALITOS_CCCR, 154 TALITOS2_CCCR_RESET); 155 156 while ((in_be32(priv->chan[ch].reg + TALITOS_CCCR) & 157 TALITOS2_CCCR_RESET) && --timeout) 158 cpu_relax(); 159 } 160 161 if (timeout == 0) { 162 dev_err(dev, "failed to reset channel %d\n", ch); 163 return -EIO; 164 } 165 166 /* set 36-bit addressing, done writeback enable and done IRQ enable */ 167 setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO, TALITOS_CCCR_LO_EAE | 168 TALITOS_CCCR_LO_CDWE | TALITOS_CCCR_LO_CDIE); 169 /* enable chaining descriptors */ 170 if (is_sec1) 171 setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO, 172 TALITOS_CCCR_LO_NE); 173 174 /* and ICCR writeback, if available */ 175 if (priv->features & TALITOS_FTR_HW_AUTH_CHECK) 176 setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO, 177 TALITOS_CCCR_LO_IWSE); 178 179 return 0; 180 } 181 182 static int reset_device(struct device *dev) 183 { 184 struct talitos_private *priv = dev_get_drvdata(dev); 185 unsigned int timeout = TALITOS_TIMEOUT; 186 bool is_sec1 = has_ftr_sec1(priv); 187 u32 mcr = is_sec1 ? TALITOS1_MCR_SWR : TALITOS2_MCR_SWR; 188 189 setbits32(priv->reg + TALITOS_MCR, mcr); 190 191 while ((in_be32(priv->reg + TALITOS_MCR) & mcr) 192 && --timeout) 193 cpu_relax(); 194 195 if (priv->irq[1]) { 196 mcr = TALITOS_MCR_RCA1 | TALITOS_MCR_RCA3; 197 setbits32(priv->reg + TALITOS_MCR, mcr); 198 } 199 200 if (timeout == 0) { 201 dev_err(dev, "failed to reset device\n"); 202 return -EIO; 203 } 204 205 return 0; 206 } 207 208 /* 209 * Reset and initialize the device 210 */ 211 static int init_device(struct device *dev) 212 { 213 struct talitos_private *priv = dev_get_drvdata(dev); 214 int ch, err; 215 bool is_sec1 = has_ftr_sec1(priv); 216 217 /* 218 * Master reset 219 * errata documentation: warning: certain SEC interrupts 220 * are not fully cleared by writing the MCR:SWR bit, 221 * set bit twice to completely reset 222 */ 223 err = reset_device(dev); 224 if (err) 225 return err; 226 227 err = reset_device(dev); 228 if (err) 229 return err; 230 231 /* reset channels */ 232 for (ch = 0; ch < priv->num_channels; ch++) { 233 err = reset_channel(dev, ch); 234 if (err) 235 return err; 236 } 237 238 /* enable channel done and error interrupts */ 239 if (is_sec1) { 240 clrbits32(priv->reg + TALITOS_IMR, TALITOS1_IMR_INIT); 241 clrbits32(priv->reg + TALITOS_IMR_LO, TALITOS1_IMR_LO_INIT); 242 /* disable parity error check in DEU (erroneous? test vect.) */ 243 setbits32(priv->reg_deu + TALITOS_EUICR, TALITOS1_DEUICR_KPE); 244 } else { 245 setbits32(priv->reg + TALITOS_IMR, TALITOS2_IMR_INIT); 246 setbits32(priv->reg + TALITOS_IMR_LO, TALITOS2_IMR_LO_INIT); 247 } 248 249 /* disable integrity check error interrupts (use writeback instead) */ 250 if (priv->features & TALITOS_FTR_HW_AUTH_CHECK) 251 setbits32(priv->reg_mdeu + TALITOS_EUICR_LO, 252 TALITOS_MDEUICR_LO_ICE); 253 254 return 0; 255 } 256 257 /** 258 * talitos_submit - submits a descriptor to the device for processing 259 * @dev: the SEC device to be used 260 * @ch: the SEC device channel to be used 261 * @desc: the descriptor to be processed by the device 262 * @callback: whom to call when processing is complete 263 * @context: a handle for use by caller (optional) 264 * 265 * desc must contain valid dma-mapped (bus physical) address pointers. 266 * callback must check err and feedback in descriptor header 267 * for device processing status. 268 */ 269 static int talitos_submit(struct device *dev, int ch, struct talitos_desc *desc, 270 void (*callback)(struct device *dev, 271 struct talitos_desc *desc, 272 void *context, int error), 273 void *context) 274 { 275 struct talitos_private *priv = dev_get_drvdata(dev); 276 struct talitos_request *request; 277 unsigned long flags; 278 int head; 279 bool is_sec1 = has_ftr_sec1(priv); 280 281 spin_lock_irqsave(&priv->chan[ch].head_lock, flags); 282 283 if (!atomic_inc_not_zero(&priv->chan[ch].submit_count)) { 284 /* h/w fifo is full */ 285 spin_unlock_irqrestore(&priv->chan[ch].head_lock, flags); 286 return -EAGAIN; 287 } 288 289 head = priv->chan[ch].head; 290 request = &priv->chan[ch].fifo[head]; 291 292 /* map descriptor and save caller data */ 293 if (is_sec1) { 294 desc->hdr1 = desc->hdr; 295 request->dma_desc = dma_map_single(dev, &desc->hdr1, 296 TALITOS_DESC_SIZE, 297 DMA_BIDIRECTIONAL); 298 } else { 299 request->dma_desc = dma_map_single(dev, desc, 300 TALITOS_DESC_SIZE, 301 DMA_BIDIRECTIONAL); 302 } 303 request->callback = callback; 304 request->context = context; 305 306 /* increment fifo head */ 307 priv->chan[ch].head = (priv->chan[ch].head + 1) & (priv->fifo_len - 1); 308 309 smp_wmb(); 310 request->desc = desc; 311 312 /* GO! */ 313 wmb(); 314 out_be32(priv->chan[ch].reg + TALITOS_FF, 315 upper_32_bits(request->dma_desc)); 316 out_be32(priv->chan[ch].reg + TALITOS_FF_LO, 317 lower_32_bits(request->dma_desc)); 318 319 spin_unlock_irqrestore(&priv->chan[ch].head_lock, flags); 320 321 return -EINPROGRESS; 322 } 323 324 static __be32 get_request_hdr(struct talitos_request *request, bool is_sec1) 325 { 326 struct talitos_edesc *edesc; 327 328 if (!is_sec1) 329 return request->desc->hdr; 330 331 if (!request->desc->next_desc) 332 return request->desc->hdr1; 333 334 edesc = container_of(request->desc, struct talitos_edesc, desc); 335 336 return ((struct talitos_desc *)(edesc->buf + edesc->dma_len))->hdr1; 337 } 338 339 /* 340 * process what was done, notify callback of error if not 341 */ 342 static void flush_channel(struct device *dev, int ch, int error, int reset_ch) 343 { 344 struct talitos_private *priv = dev_get_drvdata(dev); 345 struct talitos_request *request, saved_req; 346 unsigned long flags; 347 int tail, status; 348 bool is_sec1 = has_ftr_sec1(priv); 349 350 spin_lock_irqsave(&priv->chan[ch].tail_lock, flags); 351 352 tail = priv->chan[ch].tail; 353 while (priv->chan[ch].fifo[tail].desc) { 354 __be32 hdr; 355 356 request = &priv->chan[ch].fifo[tail]; 357 358 /* descriptors with their done bits set don't get the error */ 359 rmb(); 360 hdr = get_request_hdr(request, is_sec1); 361 362 if ((hdr & DESC_HDR_DONE) == DESC_HDR_DONE) 363 status = 0; 364 else 365 if (!error) 366 break; 367 else 368 status = error; 369 370 dma_unmap_single(dev, request->dma_desc, 371 TALITOS_DESC_SIZE, 372 DMA_BIDIRECTIONAL); 373 374 /* copy entries so we can call callback outside lock */ 375 saved_req.desc = request->desc; 376 saved_req.callback = request->callback; 377 saved_req.context = request->context; 378 379 /* release request entry in fifo */ 380 smp_wmb(); 381 request->desc = NULL; 382 383 /* increment fifo tail */ 384 priv->chan[ch].tail = (tail + 1) & (priv->fifo_len - 1); 385 386 spin_unlock_irqrestore(&priv->chan[ch].tail_lock, flags); 387 388 atomic_dec(&priv->chan[ch].submit_count); 389 390 saved_req.callback(dev, saved_req.desc, saved_req.context, 391 status); 392 /* channel may resume processing in single desc error case */ 393 if (error && !reset_ch && status == error) 394 return; 395 spin_lock_irqsave(&priv->chan[ch].tail_lock, flags); 396 tail = priv->chan[ch].tail; 397 } 398 399 spin_unlock_irqrestore(&priv->chan[ch].tail_lock, flags); 400 } 401 402 /* 403 * process completed requests for channels that have done status 404 */ 405 #define DEF_TALITOS1_DONE(name, ch_done_mask) \ 406 static void talitos1_done_##name(unsigned long data) \ 407 { \ 408 struct device *dev = (struct device *)data; \ 409 struct talitos_private *priv = dev_get_drvdata(dev); \ 410 unsigned long flags; \ 411 \ 412 if (ch_done_mask & 0x10000000) \ 413 flush_channel(dev, 0, 0, 0); \ 414 if (ch_done_mask & 0x40000000) \ 415 flush_channel(dev, 1, 0, 0); \ 416 if (ch_done_mask & 0x00010000) \ 417 flush_channel(dev, 2, 0, 0); \ 418 if (ch_done_mask & 0x00040000) \ 419 flush_channel(dev, 3, 0, 0); \ 420 \ 421 /* At this point, all completed channels have been processed */ \ 422 /* Unmask done interrupts for channels completed later on. */ \ 423 spin_lock_irqsave(&priv->reg_lock, flags); \ 424 clrbits32(priv->reg + TALITOS_IMR, ch_done_mask); \ 425 clrbits32(priv->reg + TALITOS_IMR_LO, TALITOS1_IMR_LO_INIT); \ 426 spin_unlock_irqrestore(&priv->reg_lock, flags); \ 427 } 428 429 DEF_TALITOS1_DONE(4ch, TALITOS1_ISR_4CHDONE) 430 DEF_TALITOS1_DONE(ch0, TALITOS1_ISR_CH_0_DONE) 431 432 #define DEF_TALITOS2_DONE(name, ch_done_mask) \ 433 static void talitos2_done_##name(unsigned long data) \ 434 { \ 435 struct device *dev = (struct device *)data; \ 436 struct talitos_private *priv = dev_get_drvdata(dev); \ 437 unsigned long flags; \ 438 \ 439 if (ch_done_mask & 1) \ 440 flush_channel(dev, 0, 0, 0); \ 441 if (ch_done_mask & (1 << 2)) \ 442 flush_channel(dev, 1, 0, 0); \ 443 if (ch_done_mask & (1 << 4)) \ 444 flush_channel(dev, 2, 0, 0); \ 445 if (ch_done_mask & (1 << 6)) \ 446 flush_channel(dev, 3, 0, 0); \ 447 \ 448 /* At this point, all completed channels have been processed */ \ 449 /* Unmask done interrupts for channels completed later on. */ \ 450 spin_lock_irqsave(&priv->reg_lock, flags); \ 451 setbits32(priv->reg + TALITOS_IMR, ch_done_mask); \ 452 setbits32(priv->reg + TALITOS_IMR_LO, TALITOS2_IMR_LO_INIT); \ 453 spin_unlock_irqrestore(&priv->reg_lock, flags); \ 454 } 455 456 DEF_TALITOS2_DONE(4ch, TALITOS2_ISR_4CHDONE) 457 DEF_TALITOS2_DONE(ch0, TALITOS2_ISR_CH_0_DONE) 458 DEF_TALITOS2_DONE(ch0_2, TALITOS2_ISR_CH_0_2_DONE) 459 DEF_TALITOS2_DONE(ch1_3, TALITOS2_ISR_CH_1_3_DONE) 460 461 /* 462 * locate current (offending) descriptor 463 */ 464 static __be32 current_desc_hdr(struct device *dev, int ch) 465 { 466 struct talitos_private *priv = dev_get_drvdata(dev); 467 int tail, iter; 468 dma_addr_t cur_desc; 469 470 cur_desc = ((u64)in_be32(priv->chan[ch].reg + TALITOS_CDPR)) << 32; 471 cur_desc |= in_be32(priv->chan[ch].reg + TALITOS_CDPR_LO); 472 473 if (!cur_desc) { 474 dev_err(dev, "CDPR is NULL, giving up search for offending descriptor\n"); 475 return 0; 476 } 477 478 tail = priv->chan[ch].tail; 479 480 iter = tail; 481 while (priv->chan[ch].fifo[iter].dma_desc != cur_desc && 482 priv->chan[ch].fifo[iter].desc->next_desc != cpu_to_be32(cur_desc)) { 483 iter = (iter + 1) & (priv->fifo_len - 1); 484 if (iter == tail) { 485 dev_err(dev, "couldn't locate current descriptor\n"); 486 return 0; 487 } 488 } 489 490 if (priv->chan[ch].fifo[iter].desc->next_desc == cpu_to_be32(cur_desc)) { 491 struct talitos_edesc *edesc; 492 493 edesc = container_of(priv->chan[ch].fifo[iter].desc, 494 struct talitos_edesc, desc); 495 return ((struct talitos_desc *) 496 (edesc->buf + edesc->dma_len))->hdr; 497 } 498 499 return priv->chan[ch].fifo[iter].desc->hdr; 500 } 501 502 /* 503 * user diagnostics; report root cause of error based on execution unit status 504 */ 505 static void report_eu_error(struct device *dev, int ch, __be32 desc_hdr) 506 { 507 struct talitos_private *priv = dev_get_drvdata(dev); 508 int i; 509 510 if (!desc_hdr) 511 desc_hdr = cpu_to_be32(in_be32(priv->chan[ch].reg + TALITOS_DESCBUF)); 512 513 switch (desc_hdr & DESC_HDR_SEL0_MASK) { 514 case DESC_HDR_SEL0_AFEU: 515 dev_err(dev, "AFEUISR 0x%08x_%08x\n", 516 in_be32(priv->reg_afeu + TALITOS_EUISR), 517 in_be32(priv->reg_afeu + TALITOS_EUISR_LO)); 518 break; 519 case DESC_HDR_SEL0_DEU: 520 dev_err(dev, "DEUISR 0x%08x_%08x\n", 521 in_be32(priv->reg_deu + TALITOS_EUISR), 522 in_be32(priv->reg_deu + TALITOS_EUISR_LO)); 523 break; 524 case DESC_HDR_SEL0_MDEUA: 525 case DESC_HDR_SEL0_MDEUB: 526 dev_err(dev, "MDEUISR 0x%08x_%08x\n", 527 in_be32(priv->reg_mdeu + TALITOS_EUISR), 528 in_be32(priv->reg_mdeu + TALITOS_EUISR_LO)); 529 break; 530 case DESC_HDR_SEL0_RNG: 531 dev_err(dev, "RNGUISR 0x%08x_%08x\n", 532 in_be32(priv->reg_rngu + TALITOS_ISR), 533 in_be32(priv->reg_rngu + TALITOS_ISR_LO)); 534 break; 535 case DESC_HDR_SEL0_PKEU: 536 dev_err(dev, "PKEUISR 0x%08x_%08x\n", 537 in_be32(priv->reg_pkeu + TALITOS_EUISR), 538 in_be32(priv->reg_pkeu + TALITOS_EUISR_LO)); 539 break; 540 case DESC_HDR_SEL0_AESU: 541 dev_err(dev, "AESUISR 0x%08x_%08x\n", 542 in_be32(priv->reg_aesu + TALITOS_EUISR), 543 in_be32(priv->reg_aesu + TALITOS_EUISR_LO)); 544 break; 545 case DESC_HDR_SEL0_CRCU: 546 dev_err(dev, "CRCUISR 0x%08x_%08x\n", 547 in_be32(priv->reg_crcu + TALITOS_EUISR), 548 in_be32(priv->reg_crcu + TALITOS_EUISR_LO)); 549 break; 550 case DESC_HDR_SEL0_KEU: 551 dev_err(dev, "KEUISR 0x%08x_%08x\n", 552 in_be32(priv->reg_pkeu + TALITOS_EUISR), 553 in_be32(priv->reg_pkeu + TALITOS_EUISR_LO)); 554 break; 555 } 556 557 switch (desc_hdr & DESC_HDR_SEL1_MASK) { 558 case DESC_HDR_SEL1_MDEUA: 559 case DESC_HDR_SEL1_MDEUB: 560 dev_err(dev, "MDEUISR 0x%08x_%08x\n", 561 in_be32(priv->reg_mdeu + TALITOS_EUISR), 562 in_be32(priv->reg_mdeu + TALITOS_EUISR_LO)); 563 break; 564 case DESC_HDR_SEL1_CRCU: 565 dev_err(dev, "CRCUISR 0x%08x_%08x\n", 566 in_be32(priv->reg_crcu + TALITOS_EUISR), 567 in_be32(priv->reg_crcu + TALITOS_EUISR_LO)); 568 break; 569 } 570 571 for (i = 0; i < 8; i++) 572 dev_err(dev, "DESCBUF 0x%08x_%08x\n", 573 in_be32(priv->chan[ch].reg + TALITOS_DESCBUF + 8*i), 574 in_be32(priv->chan[ch].reg + TALITOS_DESCBUF_LO + 8*i)); 575 } 576 577 /* 578 * recover from error interrupts 579 */ 580 static void talitos_error(struct device *dev, u32 isr, u32 isr_lo) 581 { 582 struct talitos_private *priv = dev_get_drvdata(dev); 583 unsigned int timeout = TALITOS_TIMEOUT; 584 int ch, error, reset_dev = 0; 585 u32 v_lo; 586 bool is_sec1 = has_ftr_sec1(priv); 587 int reset_ch = is_sec1 ? 1 : 0; /* only SEC2 supports continuation */ 588 589 for (ch = 0; ch < priv->num_channels; ch++) { 590 /* skip channels without errors */ 591 if (is_sec1) { 592 /* bits 29, 31, 17, 19 */ 593 if (!(isr & (1 << (29 + (ch & 1) * 2 - (ch & 2) * 6)))) 594 continue; 595 } else { 596 if (!(isr & (1 << (ch * 2 + 1)))) 597 continue; 598 } 599 600 error = -EINVAL; 601 602 v_lo = in_be32(priv->chan[ch].reg + TALITOS_CCPSR_LO); 603 604 if (v_lo & TALITOS_CCPSR_LO_DOF) { 605 dev_err(dev, "double fetch fifo overflow error\n"); 606 error = -EAGAIN; 607 reset_ch = 1; 608 } 609 if (v_lo & TALITOS_CCPSR_LO_SOF) { 610 /* h/w dropped descriptor */ 611 dev_err(dev, "single fetch fifo overflow error\n"); 612 error = -EAGAIN; 613 } 614 if (v_lo & TALITOS_CCPSR_LO_MDTE) 615 dev_err(dev, "master data transfer error\n"); 616 if (v_lo & TALITOS_CCPSR_LO_SGDLZ) 617 dev_err(dev, is_sec1 ? "pointer not complete error\n" 618 : "s/g data length zero error\n"); 619 if (v_lo & TALITOS_CCPSR_LO_FPZ) 620 dev_err(dev, is_sec1 ? "parity error\n" 621 : "fetch pointer zero error\n"); 622 if (v_lo & TALITOS_CCPSR_LO_IDH) 623 dev_err(dev, "illegal descriptor header error\n"); 624 if (v_lo & TALITOS_CCPSR_LO_IEU) 625 dev_err(dev, is_sec1 ? "static assignment error\n" 626 : "invalid exec unit error\n"); 627 if (v_lo & TALITOS_CCPSR_LO_EU) 628 report_eu_error(dev, ch, current_desc_hdr(dev, ch)); 629 if (!is_sec1) { 630 if (v_lo & TALITOS_CCPSR_LO_GB) 631 dev_err(dev, "gather boundary error\n"); 632 if (v_lo & TALITOS_CCPSR_LO_GRL) 633 dev_err(dev, "gather return/length error\n"); 634 if (v_lo & TALITOS_CCPSR_LO_SB) 635 dev_err(dev, "scatter boundary error\n"); 636 if (v_lo & TALITOS_CCPSR_LO_SRL) 637 dev_err(dev, "scatter return/length error\n"); 638 } 639 640 flush_channel(dev, ch, error, reset_ch); 641 642 if (reset_ch) { 643 reset_channel(dev, ch); 644 } else { 645 setbits32(priv->chan[ch].reg + TALITOS_CCCR, 646 TALITOS2_CCCR_CONT); 647 setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO, 0); 648 while ((in_be32(priv->chan[ch].reg + TALITOS_CCCR) & 649 TALITOS2_CCCR_CONT) && --timeout) 650 cpu_relax(); 651 if (timeout == 0) { 652 dev_err(dev, "failed to restart channel %d\n", 653 ch); 654 reset_dev = 1; 655 } 656 } 657 } 658 if (reset_dev || (is_sec1 && isr & ~TALITOS1_ISR_4CHERR) || 659 (!is_sec1 && isr & ~TALITOS2_ISR_4CHERR) || isr_lo) { 660 if (is_sec1 && (isr_lo & TALITOS1_ISR_TEA_ERR)) 661 dev_err(dev, "TEA error: ISR 0x%08x_%08x\n", 662 isr, isr_lo); 663 else 664 dev_err(dev, "done overflow, internal time out, or " 665 "rngu error: ISR 0x%08x_%08x\n", isr, isr_lo); 666 667 /* purge request queues */ 668 for (ch = 0; ch < priv->num_channels; ch++) 669 flush_channel(dev, ch, -EIO, 1); 670 671 /* reset and reinitialize the device */ 672 init_device(dev); 673 } 674 } 675 676 #define DEF_TALITOS1_INTERRUPT(name, ch_done_mask, ch_err_mask, tlet) \ 677 static irqreturn_t talitos1_interrupt_##name(int irq, void *data) \ 678 { \ 679 struct device *dev = data; \ 680 struct talitos_private *priv = dev_get_drvdata(dev); \ 681 u32 isr, isr_lo; \ 682 unsigned long flags; \ 683 \ 684 spin_lock_irqsave(&priv->reg_lock, flags); \ 685 isr = in_be32(priv->reg + TALITOS_ISR); \ 686 isr_lo = in_be32(priv->reg + TALITOS_ISR_LO); \ 687 /* Acknowledge interrupt */ \ 688 out_be32(priv->reg + TALITOS_ICR, isr & (ch_done_mask | ch_err_mask)); \ 689 out_be32(priv->reg + TALITOS_ICR_LO, isr_lo); \ 690 \ 691 if (unlikely(isr & ch_err_mask || isr_lo & TALITOS1_IMR_LO_INIT)) { \ 692 spin_unlock_irqrestore(&priv->reg_lock, flags); \ 693 talitos_error(dev, isr & ch_err_mask, isr_lo); \ 694 } \ 695 else { \ 696 if (likely(isr & ch_done_mask)) { \ 697 /* mask further done interrupts. */ \ 698 setbits32(priv->reg + TALITOS_IMR, ch_done_mask); \ 699 /* done_task will unmask done interrupts at exit */ \ 700 tasklet_schedule(&priv->done_task[tlet]); \ 701 } \ 702 spin_unlock_irqrestore(&priv->reg_lock, flags); \ 703 } \ 704 \ 705 return (isr & (ch_done_mask | ch_err_mask) || isr_lo) ? IRQ_HANDLED : \ 706 IRQ_NONE; \ 707 } 708 709 DEF_TALITOS1_INTERRUPT(4ch, TALITOS1_ISR_4CHDONE, TALITOS1_ISR_4CHERR, 0) 710 711 #define DEF_TALITOS2_INTERRUPT(name, ch_done_mask, ch_err_mask, tlet) \ 712 static irqreturn_t talitos2_interrupt_##name(int irq, void *data) \ 713 { \ 714 struct device *dev = data; \ 715 struct talitos_private *priv = dev_get_drvdata(dev); \ 716 u32 isr, isr_lo; \ 717 unsigned long flags; \ 718 \ 719 spin_lock_irqsave(&priv->reg_lock, flags); \ 720 isr = in_be32(priv->reg + TALITOS_ISR); \ 721 isr_lo = in_be32(priv->reg + TALITOS_ISR_LO); \ 722 /* Acknowledge interrupt */ \ 723 out_be32(priv->reg + TALITOS_ICR, isr & (ch_done_mask | ch_err_mask)); \ 724 out_be32(priv->reg + TALITOS_ICR_LO, isr_lo); \ 725 \ 726 if (unlikely(isr & ch_err_mask || isr_lo)) { \ 727 spin_unlock_irqrestore(&priv->reg_lock, flags); \ 728 talitos_error(dev, isr & ch_err_mask, isr_lo); \ 729 } \ 730 else { \ 731 if (likely(isr & ch_done_mask)) { \ 732 /* mask further done interrupts. */ \ 733 clrbits32(priv->reg + TALITOS_IMR, ch_done_mask); \ 734 /* done_task will unmask done interrupts at exit */ \ 735 tasklet_schedule(&priv->done_task[tlet]); \ 736 } \ 737 spin_unlock_irqrestore(&priv->reg_lock, flags); \ 738 } \ 739 \ 740 return (isr & (ch_done_mask | ch_err_mask) || isr_lo) ? IRQ_HANDLED : \ 741 IRQ_NONE; \ 742 } 743 744 DEF_TALITOS2_INTERRUPT(4ch, TALITOS2_ISR_4CHDONE, TALITOS2_ISR_4CHERR, 0) 745 DEF_TALITOS2_INTERRUPT(ch0_2, TALITOS2_ISR_CH_0_2_DONE, TALITOS2_ISR_CH_0_2_ERR, 746 0) 747 DEF_TALITOS2_INTERRUPT(ch1_3, TALITOS2_ISR_CH_1_3_DONE, TALITOS2_ISR_CH_1_3_ERR, 748 1) 749 750 /* 751 * hwrng 752 */ 753 static int talitos_rng_data_present(struct hwrng *rng, int wait) 754 { 755 struct device *dev = (struct device *)rng->priv; 756 struct talitos_private *priv = dev_get_drvdata(dev); 757 u32 ofl; 758 int i; 759 760 for (i = 0; i < 20; i++) { 761 ofl = in_be32(priv->reg_rngu + TALITOS_EUSR_LO) & 762 TALITOS_RNGUSR_LO_OFL; 763 if (ofl || !wait) 764 break; 765 udelay(10); 766 } 767 768 return !!ofl; 769 } 770 771 static int talitos_rng_data_read(struct hwrng *rng, u32 *data) 772 { 773 struct device *dev = (struct device *)rng->priv; 774 struct talitos_private *priv = dev_get_drvdata(dev); 775 776 /* rng fifo requires 64-bit accesses */ 777 *data = in_be32(priv->reg_rngu + TALITOS_EU_FIFO); 778 *data = in_be32(priv->reg_rngu + TALITOS_EU_FIFO_LO); 779 780 return sizeof(u32); 781 } 782 783 static int talitos_rng_init(struct hwrng *rng) 784 { 785 struct device *dev = (struct device *)rng->priv; 786 struct talitos_private *priv = dev_get_drvdata(dev); 787 unsigned int timeout = TALITOS_TIMEOUT; 788 789 setbits32(priv->reg_rngu + TALITOS_EURCR_LO, TALITOS_RNGURCR_LO_SR); 790 while (!(in_be32(priv->reg_rngu + TALITOS_EUSR_LO) 791 & TALITOS_RNGUSR_LO_RD) 792 && --timeout) 793 cpu_relax(); 794 if (timeout == 0) { 795 dev_err(dev, "failed to reset rng hw\n"); 796 return -ENODEV; 797 } 798 799 /* start generating */ 800 setbits32(priv->reg_rngu + TALITOS_EUDSR_LO, 0); 801 802 return 0; 803 } 804 805 static int talitos_register_rng(struct device *dev) 806 { 807 struct talitos_private *priv = dev_get_drvdata(dev); 808 int err; 809 810 priv->rng.name = dev_driver_string(dev); 811 priv->rng.init = talitos_rng_init; 812 priv->rng.data_present = talitos_rng_data_present; 813 priv->rng.data_read = talitos_rng_data_read; 814 priv->rng.priv = (unsigned long)dev; 815 816 err = hwrng_register(&priv->rng); 817 if (!err) 818 priv->rng_registered = true; 819 820 return err; 821 } 822 823 static void talitos_unregister_rng(struct device *dev) 824 { 825 struct talitos_private *priv = dev_get_drvdata(dev); 826 827 if (!priv->rng_registered) 828 return; 829 830 hwrng_unregister(&priv->rng); 831 priv->rng_registered = false; 832 } 833 834 /* 835 * crypto alg 836 */ 837 #define TALITOS_CRA_PRIORITY 3000 838 /* 839 * Defines a priority for doing AEAD with descriptors type 840 * HMAC_SNOOP_NO_AFEA (HSNA) instead of type IPSEC_ESP 841 */ 842 #define TALITOS_CRA_PRIORITY_AEAD_HSNA (TALITOS_CRA_PRIORITY - 1) 843 #ifdef CONFIG_CRYPTO_DEV_TALITOS2 844 #define TALITOS_MAX_KEY_SIZE (AES_MAX_KEY_SIZE + SHA512_BLOCK_SIZE) 845 #else 846 #define TALITOS_MAX_KEY_SIZE (AES_MAX_KEY_SIZE + SHA256_BLOCK_SIZE) 847 #endif 848 #define TALITOS_MAX_IV_LENGTH 16 /* max of AES_BLOCK_SIZE, DES3_EDE_BLOCK_SIZE */ 849 850 struct talitos_ctx { 851 struct device *dev; 852 int ch; 853 __be32 desc_hdr_template; 854 u8 key[TALITOS_MAX_KEY_SIZE]; 855 u8 iv[TALITOS_MAX_IV_LENGTH]; 856 dma_addr_t dma_key; 857 unsigned int keylen; 858 unsigned int enckeylen; 859 unsigned int authkeylen; 860 }; 861 862 #define HASH_MAX_BLOCK_SIZE SHA512_BLOCK_SIZE 863 #define TALITOS_MDEU_MAX_CONTEXT_SIZE TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512 864 865 struct talitos_ahash_req_ctx { 866 u32 hw_context[TALITOS_MDEU_MAX_CONTEXT_SIZE / sizeof(u32)]; 867 unsigned int hw_context_size; 868 u8 buf[2][HASH_MAX_BLOCK_SIZE]; 869 int buf_idx; 870 unsigned int swinit; 871 unsigned int first; 872 unsigned int last; 873 unsigned int to_hash_later; 874 unsigned int nbuf; 875 struct scatterlist bufsl[2]; 876 struct scatterlist *psrc; 877 }; 878 879 struct talitos_export_state { 880 u32 hw_context[TALITOS_MDEU_MAX_CONTEXT_SIZE / sizeof(u32)]; 881 u8 buf[HASH_MAX_BLOCK_SIZE]; 882 unsigned int swinit; 883 unsigned int first; 884 unsigned int last; 885 unsigned int to_hash_later; 886 unsigned int nbuf; 887 }; 888 889 static int aead_setkey(struct crypto_aead *authenc, 890 const u8 *key, unsigned int keylen) 891 { 892 struct talitos_ctx *ctx = crypto_aead_ctx(authenc); 893 struct device *dev = ctx->dev; 894 struct crypto_authenc_keys keys; 895 896 if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) 897 goto badkey; 898 899 if (keys.authkeylen + keys.enckeylen > TALITOS_MAX_KEY_SIZE) 900 goto badkey; 901 902 if (ctx->keylen) 903 dma_unmap_single(dev, ctx->dma_key, ctx->keylen, DMA_TO_DEVICE); 904 905 memcpy(ctx->key, keys.authkey, keys.authkeylen); 906 memcpy(&ctx->key[keys.authkeylen], keys.enckey, keys.enckeylen); 907 908 ctx->keylen = keys.authkeylen + keys.enckeylen; 909 ctx->enckeylen = keys.enckeylen; 910 ctx->authkeylen = keys.authkeylen; 911 ctx->dma_key = dma_map_single(dev, ctx->key, ctx->keylen, 912 DMA_TO_DEVICE); 913 914 memzero_explicit(&keys, sizeof(keys)); 915 return 0; 916 917 badkey: 918 memzero_explicit(&keys, sizeof(keys)); 919 return -EINVAL; 920 } 921 922 static int aead_des3_setkey(struct crypto_aead *authenc, 923 const u8 *key, unsigned int keylen) 924 { 925 struct talitos_ctx *ctx = crypto_aead_ctx(authenc); 926 struct device *dev = ctx->dev; 927 struct crypto_authenc_keys keys; 928 int err; 929 930 err = crypto_authenc_extractkeys(&keys, key, keylen); 931 if (unlikely(err)) 932 goto out; 933 934 err = -EINVAL; 935 if (keys.authkeylen + keys.enckeylen > TALITOS_MAX_KEY_SIZE) 936 goto out; 937 938 err = verify_aead_des3_key(authenc, keys.enckey, keys.enckeylen); 939 if (err) 940 goto out; 941 942 if (ctx->keylen) 943 dma_unmap_single(dev, ctx->dma_key, ctx->keylen, DMA_TO_DEVICE); 944 945 memcpy(ctx->key, keys.authkey, keys.authkeylen); 946 memcpy(&ctx->key[keys.authkeylen], keys.enckey, keys.enckeylen); 947 948 ctx->keylen = keys.authkeylen + keys.enckeylen; 949 ctx->enckeylen = keys.enckeylen; 950 ctx->authkeylen = keys.authkeylen; 951 ctx->dma_key = dma_map_single(dev, ctx->key, ctx->keylen, 952 DMA_TO_DEVICE); 953 954 out: 955 memzero_explicit(&keys, sizeof(keys)); 956 return err; 957 } 958 959 static void talitos_sg_unmap(struct device *dev, 960 struct talitos_edesc *edesc, 961 struct scatterlist *src, 962 struct scatterlist *dst, 963 unsigned int len, unsigned int offset) 964 { 965 struct talitos_private *priv = dev_get_drvdata(dev); 966 bool is_sec1 = has_ftr_sec1(priv); 967 unsigned int src_nents = edesc->src_nents ? : 1; 968 unsigned int dst_nents = edesc->dst_nents ? : 1; 969 970 if (is_sec1 && dst && dst_nents > 1) { 971 dma_sync_single_for_device(dev, edesc->dma_link_tbl + offset, 972 len, DMA_FROM_DEVICE); 973 sg_pcopy_from_buffer(dst, dst_nents, edesc->buf + offset, len, 974 offset); 975 } 976 if (src != dst) { 977 if (src_nents == 1 || !is_sec1) 978 dma_unmap_sg(dev, src, src_nents, DMA_TO_DEVICE); 979 980 if (dst && (dst_nents == 1 || !is_sec1)) 981 dma_unmap_sg(dev, dst, dst_nents, DMA_FROM_DEVICE); 982 } else if (src_nents == 1 || !is_sec1) { 983 dma_unmap_sg(dev, src, src_nents, DMA_BIDIRECTIONAL); 984 } 985 } 986 987 static void ipsec_esp_unmap(struct device *dev, 988 struct talitos_edesc *edesc, 989 struct aead_request *areq, bool encrypt) 990 { 991 struct crypto_aead *aead = crypto_aead_reqtfm(areq); 992 struct talitos_ctx *ctx = crypto_aead_ctx(aead); 993 unsigned int ivsize = crypto_aead_ivsize(aead); 994 unsigned int authsize = crypto_aead_authsize(aead); 995 unsigned int cryptlen = areq->cryptlen - (encrypt ? 0 : authsize); 996 bool is_ipsec_esp = edesc->desc.hdr & DESC_HDR_TYPE_IPSEC_ESP; 997 struct talitos_ptr *civ_ptr = &edesc->desc.ptr[is_ipsec_esp ? 2 : 3]; 998 999 if (is_ipsec_esp) 1000 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[6], 1001 DMA_FROM_DEVICE); 1002 unmap_single_talitos_ptr(dev, civ_ptr, DMA_TO_DEVICE); 1003 1004 talitos_sg_unmap(dev, edesc, areq->src, areq->dst, 1005 cryptlen + authsize, areq->assoclen); 1006 1007 if (edesc->dma_len) 1008 dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len, 1009 DMA_BIDIRECTIONAL); 1010 1011 if (!is_ipsec_esp) { 1012 unsigned int dst_nents = edesc->dst_nents ? : 1; 1013 1014 sg_pcopy_to_buffer(areq->dst, dst_nents, ctx->iv, ivsize, 1015 areq->assoclen + cryptlen - ivsize); 1016 } 1017 } 1018 1019 /* 1020 * ipsec_esp descriptor callbacks 1021 */ 1022 static void ipsec_esp_encrypt_done(struct device *dev, 1023 struct talitos_desc *desc, void *context, 1024 int err) 1025 { 1026 struct aead_request *areq = context; 1027 struct crypto_aead *authenc = crypto_aead_reqtfm(areq); 1028 unsigned int ivsize = crypto_aead_ivsize(authenc); 1029 struct talitos_edesc *edesc; 1030 1031 edesc = container_of(desc, struct talitos_edesc, desc); 1032 1033 ipsec_esp_unmap(dev, edesc, areq, true); 1034 1035 dma_unmap_single(dev, edesc->iv_dma, ivsize, DMA_TO_DEVICE); 1036 1037 kfree(edesc); 1038 1039 aead_request_complete(areq, err); 1040 } 1041 1042 static void ipsec_esp_decrypt_swauth_done(struct device *dev, 1043 struct talitos_desc *desc, 1044 void *context, int err) 1045 { 1046 struct aead_request *req = context; 1047 struct crypto_aead *authenc = crypto_aead_reqtfm(req); 1048 unsigned int authsize = crypto_aead_authsize(authenc); 1049 struct talitos_edesc *edesc; 1050 char *oicv, *icv; 1051 1052 edesc = container_of(desc, struct talitos_edesc, desc); 1053 1054 ipsec_esp_unmap(dev, edesc, req, false); 1055 1056 if (!err) { 1057 /* auth check */ 1058 oicv = edesc->buf + edesc->dma_len; 1059 icv = oicv - authsize; 1060 1061 err = crypto_memneq(oicv, icv, authsize) ? -EBADMSG : 0; 1062 } 1063 1064 kfree(edesc); 1065 1066 aead_request_complete(req, err); 1067 } 1068 1069 static void ipsec_esp_decrypt_hwauth_done(struct device *dev, 1070 struct talitos_desc *desc, 1071 void *context, int err) 1072 { 1073 struct aead_request *req = context; 1074 struct talitos_edesc *edesc; 1075 1076 edesc = container_of(desc, struct talitos_edesc, desc); 1077 1078 ipsec_esp_unmap(dev, edesc, req, false); 1079 1080 /* check ICV auth status */ 1081 if (!err && ((desc->hdr_lo & DESC_HDR_LO_ICCR1_MASK) != 1082 DESC_HDR_LO_ICCR1_PASS)) 1083 err = -EBADMSG; 1084 1085 kfree(edesc); 1086 1087 aead_request_complete(req, err); 1088 } 1089 1090 /* 1091 * convert scatterlist to SEC h/w link table format 1092 * stop at cryptlen bytes 1093 */ 1094 static int sg_to_link_tbl_offset(struct scatterlist *sg, int sg_count, 1095 unsigned int offset, int datalen, int elen, 1096 struct talitos_ptr *link_tbl_ptr) 1097 { 1098 int n_sg = elen ? sg_count + 1 : sg_count; 1099 int count = 0; 1100 int cryptlen = datalen + elen; 1101 1102 while (cryptlen && sg && n_sg--) { 1103 unsigned int len = sg_dma_len(sg); 1104 1105 if (offset >= len) { 1106 offset -= len; 1107 goto next; 1108 } 1109 1110 len -= offset; 1111 1112 if (len > cryptlen) 1113 len = cryptlen; 1114 1115 if (datalen > 0 && len > datalen) { 1116 to_talitos_ptr(link_tbl_ptr + count, 1117 sg_dma_address(sg) + offset, datalen, 0); 1118 to_talitos_ptr_ext_set(link_tbl_ptr + count, 0, 0); 1119 count++; 1120 len -= datalen; 1121 offset += datalen; 1122 } 1123 to_talitos_ptr(link_tbl_ptr + count, 1124 sg_dma_address(sg) + offset, len, 0); 1125 to_talitos_ptr_ext_set(link_tbl_ptr + count, 0, 0); 1126 count++; 1127 cryptlen -= len; 1128 datalen -= len; 1129 offset = 0; 1130 1131 next: 1132 sg = sg_next(sg); 1133 } 1134 1135 /* tag end of link table */ 1136 if (count > 0) 1137 to_talitos_ptr_ext_set(link_tbl_ptr + count - 1, 1138 DESC_PTR_LNKTBL_RET, 0); 1139 1140 return count; 1141 } 1142 1143 static int talitos_sg_map_ext(struct device *dev, struct scatterlist *src, 1144 unsigned int len, struct talitos_edesc *edesc, 1145 struct talitos_ptr *ptr, int sg_count, 1146 unsigned int offset, int tbl_off, int elen, 1147 bool force) 1148 { 1149 struct talitos_private *priv = dev_get_drvdata(dev); 1150 bool is_sec1 = has_ftr_sec1(priv); 1151 1152 if (!src) { 1153 to_talitos_ptr(ptr, 0, 0, is_sec1); 1154 return 1; 1155 } 1156 to_talitos_ptr_ext_set(ptr, elen, is_sec1); 1157 if (sg_count == 1 && !force) { 1158 to_talitos_ptr(ptr, sg_dma_address(src) + offset, len, is_sec1); 1159 return sg_count; 1160 } 1161 if (is_sec1) { 1162 to_talitos_ptr(ptr, edesc->dma_link_tbl + offset, len, is_sec1); 1163 return sg_count; 1164 } 1165 sg_count = sg_to_link_tbl_offset(src, sg_count, offset, len, elen, 1166 &edesc->link_tbl[tbl_off]); 1167 if (sg_count == 1 && !force) { 1168 /* Only one segment now, so no link tbl needed*/ 1169 copy_talitos_ptr(ptr, &edesc->link_tbl[tbl_off], is_sec1); 1170 return sg_count; 1171 } 1172 to_talitos_ptr(ptr, edesc->dma_link_tbl + 1173 tbl_off * sizeof(struct talitos_ptr), len, is_sec1); 1174 to_talitos_ptr_ext_or(ptr, DESC_PTR_LNKTBL_JUMP, is_sec1); 1175 1176 return sg_count; 1177 } 1178 1179 static int talitos_sg_map(struct device *dev, struct scatterlist *src, 1180 unsigned int len, struct talitos_edesc *edesc, 1181 struct talitos_ptr *ptr, int sg_count, 1182 unsigned int offset, int tbl_off) 1183 { 1184 return talitos_sg_map_ext(dev, src, len, edesc, ptr, sg_count, offset, 1185 tbl_off, 0, false); 1186 } 1187 1188 /* 1189 * fill in and submit ipsec_esp descriptor 1190 */ 1191 static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq, 1192 bool encrypt, 1193 void (*callback)(struct device *dev, 1194 struct talitos_desc *desc, 1195 void *context, int error)) 1196 { 1197 struct crypto_aead *aead = crypto_aead_reqtfm(areq); 1198 unsigned int authsize = crypto_aead_authsize(aead); 1199 struct talitos_ctx *ctx = crypto_aead_ctx(aead); 1200 struct device *dev = ctx->dev; 1201 struct talitos_desc *desc = &edesc->desc; 1202 unsigned int cryptlen = areq->cryptlen - (encrypt ? 0 : authsize); 1203 unsigned int ivsize = crypto_aead_ivsize(aead); 1204 int tbl_off = 0; 1205 int sg_count, ret; 1206 int elen = 0; 1207 bool sync_needed = false; 1208 struct talitos_private *priv = dev_get_drvdata(dev); 1209 bool is_sec1 = has_ftr_sec1(priv); 1210 bool is_ipsec_esp = desc->hdr & DESC_HDR_TYPE_IPSEC_ESP; 1211 struct talitos_ptr *civ_ptr = &desc->ptr[is_ipsec_esp ? 2 : 3]; 1212 struct talitos_ptr *ckey_ptr = &desc->ptr[is_ipsec_esp ? 3 : 2]; 1213 dma_addr_t dma_icv = edesc->dma_link_tbl + edesc->dma_len - authsize; 1214 1215 /* hmac key */ 1216 to_talitos_ptr(&desc->ptr[0], ctx->dma_key, ctx->authkeylen, is_sec1); 1217 1218 sg_count = edesc->src_nents ?: 1; 1219 if (is_sec1 && sg_count > 1) 1220 sg_copy_to_buffer(areq->src, sg_count, edesc->buf, 1221 areq->assoclen + cryptlen); 1222 else 1223 sg_count = dma_map_sg(dev, areq->src, sg_count, 1224 (areq->src == areq->dst) ? 1225 DMA_BIDIRECTIONAL : DMA_TO_DEVICE); 1226 1227 /* hmac data */ 1228 ret = talitos_sg_map(dev, areq->src, areq->assoclen, edesc, 1229 &desc->ptr[1], sg_count, 0, tbl_off); 1230 1231 if (ret > 1) { 1232 tbl_off += ret; 1233 sync_needed = true; 1234 } 1235 1236 /* cipher iv */ 1237 to_talitos_ptr(civ_ptr, edesc->iv_dma, ivsize, is_sec1); 1238 1239 /* cipher key */ 1240 to_talitos_ptr(ckey_ptr, ctx->dma_key + ctx->authkeylen, 1241 ctx->enckeylen, is_sec1); 1242 1243 /* 1244 * cipher in 1245 * map and adjust cipher len to aead request cryptlen. 1246 * extent is bytes of HMAC postpended to ciphertext, 1247 * typically 12 for ipsec 1248 */ 1249 if (is_ipsec_esp && (desc->hdr & DESC_HDR_MODE1_MDEU_CICV)) 1250 elen = authsize; 1251 1252 ret = talitos_sg_map_ext(dev, areq->src, cryptlen, edesc, &desc->ptr[4], 1253 sg_count, areq->assoclen, tbl_off, elen, 1254 false); 1255 1256 if (ret > 1) { 1257 tbl_off += ret; 1258 sync_needed = true; 1259 } 1260 1261 /* cipher out */ 1262 if (areq->src != areq->dst) { 1263 sg_count = edesc->dst_nents ? : 1; 1264 if (!is_sec1 || sg_count == 1) 1265 dma_map_sg(dev, areq->dst, sg_count, DMA_FROM_DEVICE); 1266 } 1267 1268 if (is_ipsec_esp && encrypt) 1269 elen = authsize; 1270 else 1271 elen = 0; 1272 ret = talitos_sg_map_ext(dev, areq->dst, cryptlen, edesc, &desc->ptr[5], 1273 sg_count, areq->assoclen, tbl_off, elen, 1274 is_ipsec_esp && !encrypt); 1275 tbl_off += ret; 1276 1277 if (!encrypt && is_ipsec_esp) { 1278 struct talitos_ptr *tbl_ptr = &edesc->link_tbl[tbl_off]; 1279 1280 /* Add an entry to the link table for ICV data */ 1281 to_talitos_ptr_ext_set(tbl_ptr - 1, 0, is_sec1); 1282 to_talitos_ptr_ext_set(tbl_ptr, DESC_PTR_LNKTBL_RET, is_sec1); 1283 1284 /* icv data follows link tables */ 1285 to_talitos_ptr(tbl_ptr, dma_icv, authsize, is_sec1); 1286 to_talitos_ptr_ext_or(&desc->ptr[5], authsize, is_sec1); 1287 sync_needed = true; 1288 } else if (!encrypt) { 1289 to_talitos_ptr(&desc->ptr[6], dma_icv, authsize, is_sec1); 1290 sync_needed = true; 1291 } else if (!is_ipsec_esp) { 1292 talitos_sg_map(dev, areq->dst, authsize, edesc, &desc->ptr[6], 1293 sg_count, areq->assoclen + cryptlen, tbl_off); 1294 } 1295 1296 /* iv out */ 1297 if (is_ipsec_esp) 1298 map_single_talitos_ptr(dev, &desc->ptr[6], ivsize, ctx->iv, 1299 DMA_FROM_DEVICE); 1300 1301 if (sync_needed) 1302 dma_sync_single_for_device(dev, edesc->dma_link_tbl, 1303 edesc->dma_len, 1304 DMA_BIDIRECTIONAL); 1305 1306 ret = talitos_submit(dev, ctx->ch, desc, callback, areq); 1307 if (ret != -EINPROGRESS) { 1308 ipsec_esp_unmap(dev, edesc, areq, encrypt); 1309 kfree(edesc); 1310 } 1311 return ret; 1312 } 1313 1314 /* 1315 * allocate and map the extended descriptor 1316 */ 1317 static struct talitos_edesc *talitos_edesc_alloc(struct device *dev, 1318 struct scatterlist *src, 1319 struct scatterlist *dst, 1320 u8 *iv, 1321 unsigned int assoclen, 1322 unsigned int cryptlen, 1323 unsigned int authsize, 1324 unsigned int ivsize, 1325 int icv_stashing, 1326 u32 cryptoflags, 1327 bool encrypt) 1328 { 1329 struct talitos_edesc *edesc; 1330 int src_nents, dst_nents, alloc_len, dma_len, src_len, dst_len; 1331 dma_addr_t iv_dma = 0; 1332 gfp_t flags = cryptoflags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : 1333 GFP_ATOMIC; 1334 struct talitos_private *priv = dev_get_drvdata(dev); 1335 bool is_sec1 = has_ftr_sec1(priv); 1336 int max_len = is_sec1 ? TALITOS1_MAX_DATA_LEN : TALITOS2_MAX_DATA_LEN; 1337 1338 if (cryptlen + authsize > max_len) { 1339 dev_err(dev, "length exceeds h/w max limit\n"); 1340 return ERR_PTR(-EINVAL); 1341 } 1342 1343 if (!dst || dst == src) { 1344 src_len = assoclen + cryptlen + authsize; 1345 src_nents = sg_nents_for_len(src, src_len); 1346 if (src_nents < 0) { 1347 dev_err(dev, "Invalid number of src SG.\n"); 1348 return ERR_PTR(-EINVAL); 1349 } 1350 src_nents = (src_nents == 1) ? 0 : src_nents; 1351 dst_nents = dst ? src_nents : 0; 1352 dst_len = 0; 1353 } else { /* dst && dst != src*/ 1354 src_len = assoclen + cryptlen + (encrypt ? 0 : authsize); 1355 src_nents = sg_nents_for_len(src, src_len); 1356 if (src_nents < 0) { 1357 dev_err(dev, "Invalid number of src SG.\n"); 1358 return ERR_PTR(-EINVAL); 1359 } 1360 src_nents = (src_nents == 1) ? 0 : src_nents; 1361 dst_len = assoclen + cryptlen + (encrypt ? authsize : 0); 1362 dst_nents = sg_nents_for_len(dst, dst_len); 1363 if (dst_nents < 0) { 1364 dev_err(dev, "Invalid number of dst SG.\n"); 1365 return ERR_PTR(-EINVAL); 1366 } 1367 dst_nents = (dst_nents == 1) ? 0 : dst_nents; 1368 } 1369 1370 /* 1371 * allocate space for base edesc plus the link tables, 1372 * allowing for two separate entries for AD and generated ICV (+ 2), 1373 * and space for two sets of ICVs (stashed and generated) 1374 */ 1375 alloc_len = sizeof(struct talitos_edesc); 1376 if (src_nents || dst_nents || !encrypt) { 1377 if (is_sec1) 1378 dma_len = (src_nents ? src_len : 0) + 1379 (dst_nents ? dst_len : 0) + authsize; 1380 else 1381 dma_len = (src_nents + dst_nents + 2) * 1382 sizeof(struct talitos_ptr) + authsize; 1383 alloc_len += dma_len; 1384 } else { 1385 dma_len = 0; 1386 } 1387 alloc_len += icv_stashing ? authsize : 0; 1388 1389 /* if its a ahash, add space for a second desc next to the first one */ 1390 if (is_sec1 && !dst) 1391 alloc_len += sizeof(struct talitos_desc); 1392 alloc_len += ivsize; 1393 1394 edesc = kmalloc(alloc_len, GFP_DMA | flags); 1395 if (!edesc) 1396 return ERR_PTR(-ENOMEM); 1397 if (ivsize) { 1398 iv = memcpy(((u8 *)edesc) + alloc_len - ivsize, iv, ivsize); 1399 iv_dma = dma_map_single(dev, iv, ivsize, DMA_TO_DEVICE); 1400 } 1401 memset(&edesc->desc, 0, sizeof(edesc->desc)); 1402 1403 edesc->src_nents = src_nents; 1404 edesc->dst_nents = dst_nents; 1405 edesc->iv_dma = iv_dma; 1406 edesc->dma_len = dma_len; 1407 if (dma_len) 1408 edesc->dma_link_tbl = dma_map_single(dev, &edesc->link_tbl[0], 1409 edesc->dma_len, 1410 DMA_BIDIRECTIONAL); 1411 1412 return edesc; 1413 } 1414 1415 static struct talitos_edesc *aead_edesc_alloc(struct aead_request *areq, u8 *iv, 1416 int icv_stashing, bool encrypt) 1417 { 1418 struct crypto_aead *authenc = crypto_aead_reqtfm(areq); 1419 unsigned int authsize = crypto_aead_authsize(authenc); 1420 struct talitos_ctx *ctx = crypto_aead_ctx(authenc); 1421 unsigned int ivsize = crypto_aead_ivsize(authenc); 1422 unsigned int cryptlen = areq->cryptlen - (encrypt ? 0 : authsize); 1423 1424 return talitos_edesc_alloc(ctx->dev, areq->src, areq->dst, 1425 iv, areq->assoclen, cryptlen, 1426 authsize, ivsize, icv_stashing, 1427 areq->base.flags, encrypt); 1428 } 1429 1430 static int aead_encrypt(struct aead_request *req) 1431 { 1432 struct crypto_aead *authenc = crypto_aead_reqtfm(req); 1433 struct talitos_ctx *ctx = crypto_aead_ctx(authenc); 1434 struct talitos_edesc *edesc; 1435 1436 /* allocate extended descriptor */ 1437 edesc = aead_edesc_alloc(req, req->iv, 0, true); 1438 if (IS_ERR(edesc)) 1439 return PTR_ERR(edesc); 1440 1441 /* set encrypt */ 1442 edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT; 1443 1444 return ipsec_esp(edesc, req, true, ipsec_esp_encrypt_done); 1445 } 1446 1447 static int aead_decrypt(struct aead_request *req) 1448 { 1449 struct crypto_aead *authenc = crypto_aead_reqtfm(req); 1450 unsigned int authsize = crypto_aead_authsize(authenc); 1451 struct talitos_ctx *ctx = crypto_aead_ctx(authenc); 1452 struct talitos_private *priv = dev_get_drvdata(ctx->dev); 1453 struct talitos_edesc *edesc; 1454 void *icvdata; 1455 1456 /* allocate extended descriptor */ 1457 edesc = aead_edesc_alloc(req, req->iv, 1, false); 1458 if (IS_ERR(edesc)) 1459 return PTR_ERR(edesc); 1460 1461 if ((edesc->desc.hdr & DESC_HDR_TYPE_IPSEC_ESP) && 1462 (priv->features & TALITOS_FTR_HW_AUTH_CHECK) && 1463 ((!edesc->src_nents && !edesc->dst_nents) || 1464 priv->features & TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT)) { 1465 1466 /* decrypt and check the ICV */ 1467 edesc->desc.hdr = ctx->desc_hdr_template | 1468 DESC_HDR_DIR_INBOUND | 1469 DESC_HDR_MODE1_MDEU_CICV; 1470 1471 /* reset integrity check result bits */ 1472 1473 return ipsec_esp(edesc, req, false, 1474 ipsec_esp_decrypt_hwauth_done); 1475 } 1476 1477 /* Have to check the ICV with software */ 1478 edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_DIR_INBOUND; 1479 1480 /* stash incoming ICV for later cmp with ICV generated by the h/w */ 1481 icvdata = edesc->buf + edesc->dma_len; 1482 1483 sg_pcopy_to_buffer(req->src, edesc->src_nents ? : 1, icvdata, authsize, 1484 req->assoclen + req->cryptlen - authsize); 1485 1486 return ipsec_esp(edesc, req, false, ipsec_esp_decrypt_swauth_done); 1487 } 1488 1489 static int skcipher_setkey(struct crypto_skcipher *cipher, 1490 const u8 *key, unsigned int keylen) 1491 { 1492 struct talitos_ctx *ctx = crypto_skcipher_ctx(cipher); 1493 struct device *dev = ctx->dev; 1494 1495 if (ctx->keylen) 1496 dma_unmap_single(dev, ctx->dma_key, ctx->keylen, DMA_TO_DEVICE); 1497 1498 memcpy(&ctx->key, key, keylen); 1499 ctx->keylen = keylen; 1500 1501 ctx->dma_key = dma_map_single(dev, ctx->key, keylen, DMA_TO_DEVICE); 1502 1503 return 0; 1504 } 1505 1506 static int skcipher_des_setkey(struct crypto_skcipher *cipher, 1507 const u8 *key, unsigned int keylen) 1508 { 1509 return verify_skcipher_des_key(cipher, key) ?: 1510 skcipher_setkey(cipher, key, keylen); 1511 } 1512 1513 static int skcipher_des3_setkey(struct crypto_skcipher *cipher, 1514 const u8 *key, unsigned int keylen) 1515 { 1516 return verify_skcipher_des3_key(cipher, key) ?: 1517 skcipher_setkey(cipher, key, keylen); 1518 } 1519 1520 static int skcipher_aes_setkey(struct crypto_skcipher *cipher, 1521 const u8 *key, unsigned int keylen) 1522 { 1523 if (keylen == AES_KEYSIZE_128 || keylen == AES_KEYSIZE_192 || 1524 keylen == AES_KEYSIZE_256) 1525 return skcipher_setkey(cipher, key, keylen); 1526 1527 return -EINVAL; 1528 } 1529 1530 static void common_nonsnoop_unmap(struct device *dev, 1531 struct talitos_edesc *edesc, 1532 struct skcipher_request *areq) 1533 { 1534 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[5], DMA_FROM_DEVICE); 1535 1536 talitos_sg_unmap(dev, edesc, areq->src, areq->dst, areq->cryptlen, 0); 1537 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[1], DMA_TO_DEVICE); 1538 1539 if (edesc->dma_len) 1540 dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len, 1541 DMA_BIDIRECTIONAL); 1542 } 1543 1544 static void skcipher_done(struct device *dev, 1545 struct talitos_desc *desc, void *context, 1546 int err) 1547 { 1548 struct skcipher_request *areq = context; 1549 struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(areq); 1550 struct talitos_ctx *ctx = crypto_skcipher_ctx(cipher); 1551 unsigned int ivsize = crypto_skcipher_ivsize(cipher); 1552 struct talitos_edesc *edesc; 1553 1554 edesc = container_of(desc, struct talitos_edesc, desc); 1555 1556 common_nonsnoop_unmap(dev, edesc, areq); 1557 memcpy(areq->iv, ctx->iv, ivsize); 1558 1559 kfree(edesc); 1560 1561 areq->base.complete(&areq->base, err); 1562 } 1563 1564 static int common_nonsnoop(struct talitos_edesc *edesc, 1565 struct skcipher_request *areq, 1566 void (*callback) (struct device *dev, 1567 struct talitos_desc *desc, 1568 void *context, int error)) 1569 { 1570 struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(areq); 1571 struct talitos_ctx *ctx = crypto_skcipher_ctx(cipher); 1572 struct device *dev = ctx->dev; 1573 struct talitos_desc *desc = &edesc->desc; 1574 unsigned int cryptlen = areq->cryptlen; 1575 unsigned int ivsize = crypto_skcipher_ivsize(cipher); 1576 int sg_count, ret; 1577 bool sync_needed = false; 1578 struct talitos_private *priv = dev_get_drvdata(dev); 1579 bool is_sec1 = has_ftr_sec1(priv); 1580 1581 /* first DWORD empty */ 1582 1583 /* cipher iv */ 1584 to_talitos_ptr(&desc->ptr[1], edesc->iv_dma, ivsize, is_sec1); 1585 1586 /* cipher key */ 1587 to_talitos_ptr(&desc->ptr[2], ctx->dma_key, ctx->keylen, is_sec1); 1588 1589 sg_count = edesc->src_nents ?: 1; 1590 if (is_sec1 && sg_count > 1) 1591 sg_copy_to_buffer(areq->src, sg_count, edesc->buf, 1592 cryptlen); 1593 else 1594 sg_count = dma_map_sg(dev, areq->src, sg_count, 1595 (areq->src == areq->dst) ? 1596 DMA_BIDIRECTIONAL : DMA_TO_DEVICE); 1597 /* 1598 * cipher in 1599 */ 1600 sg_count = talitos_sg_map(dev, areq->src, cryptlen, edesc, 1601 &desc->ptr[3], sg_count, 0, 0); 1602 if (sg_count > 1) 1603 sync_needed = true; 1604 1605 /* cipher out */ 1606 if (areq->src != areq->dst) { 1607 sg_count = edesc->dst_nents ? : 1; 1608 if (!is_sec1 || sg_count == 1) 1609 dma_map_sg(dev, areq->dst, sg_count, DMA_FROM_DEVICE); 1610 } 1611 1612 ret = talitos_sg_map(dev, areq->dst, cryptlen, edesc, &desc->ptr[4], 1613 sg_count, 0, (edesc->src_nents + 1)); 1614 if (ret > 1) 1615 sync_needed = true; 1616 1617 /* iv out */ 1618 map_single_talitos_ptr(dev, &desc->ptr[5], ivsize, ctx->iv, 1619 DMA_FROM_DEVICE); 1620 1621 /* last DWORD empty */ 1622 1623 if (sync_needed) 1624 dma_sync_single_for_device(dev, edesc->dma_link_tbl, 1625 edesc->dma_len, DMA_BIDIRECTIONAL); 1626 1627 ret = talitos_submit(dev, ctx->ch, desc, callback, areq); 1628 if (ret != -EINPROGRESS) { 1629 common_nonsnoop_unmap(dev, edesc, areq); 1630 kfree(edesc); 1631 } 1632 return ret; 1633 } 1634 1635 static struct talitos_edesc *skcipher_edesc_alloc(struct skcipher_request * 1636 areq, bool encrypt) 1637 { 1638 struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(areq); 1639 struct talitos_ctx *ctx = crypto_skcipher_ctx(cipher); 1640 unsigned int ivsize = crypto_skcipher_ivsize(cipher); 1641 1642 return talitos_edesc_alloc(ctx->dev, areq->src, areq->dst, 1643 areq->iv, 0, areq->cryptlen, 0, ivsize, 0, 1644 areq->base.flags, encrypt); 1645 } 1646 1647 static int skcipher_encrypt(struct skcipher_request *areq) 1648 { 1649 struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(areq); 1650 struct talitos_ctx *ctx = crypto_skcipher_ctx(cipher); 1651 struct talitos_edesc *edesc; 1652 unsigned int blocksize = 1653 crypto_tfm_alg_blocksize(crypto_skcipher_tfm(cipher)); 1654 1655 if (!areq->cryptlen) 1656 return 0; 1657 1658 if (areq->cryptlen % blocksize) 1659 return -EINVAL; 1660 1661 /* allocate extended descriptor */ 1662 edesc = skcipher_edesc_alloc(areq, true); 1663 if (IS_ERR(edesc)) 1664 return PTR_ERR(edesc); 1665 1666 /* set encrypt */ 1667 edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT; 1668 1669 return common_nonsnoop(edesc, areq, skcipher_done); 1670 } 1671 1672 static int skcipher_decrypt(struct skcipher_request *areq) 1673 { 1674 struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(areq); 1675 struct talitos_ctx *ctx = crypto_skcipher_ctx(cipher); 1676 struct talitos_edesc *edesc; 1677 unsigned int blocksize = 1678 crypto_tfm_alg_blocksize(crypto_skcipher_tfm(cipher)); 1679 1680 if (!areq->cryptlen) 1681 return 0; 1682 1683 if (areq->cryptlen % blocksize) 1684 return -EINVAL; 1685 1686 /* allocate extended descriptor */ 1687 edesc = skcipher_edesc_alloc(areq, false); 1688 if (IS_ERR(edesc)) 1689 return PTR_ERR(edesc); 1690 1691 edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_DIR_INBOUND; 1692 1693 return common_nonsnoop(edesc, areq, skcipher_done); 1694 } 1695 1696 static void common_nonsnoop_hash_unmap(struct device *dev, 1697 struct talitos_edesc *edesc, 1698 struct ahash_request *areq) 1699 { 1700 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 1701 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); 1702 struct talitos_private *priv = dev_get_drvdata(dev); 1703 bool is_sec1 = has_ftr_sec1(priv); 1704 struct talitos_desc *desc = &edesc->desc; 1705 struct talitos_desc *desc2 = (struct talitos_desc *) 1706 (edesc->buf + edesc->dma_len); 1707 1708 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[5], DMA_FROM_DEVICE); 1709 if (desc->next_desc && 1710 desc->ptr[5].ptr != desc2->ptr[5].ptr) 1711 unmap_single_talitos_ptr(dev, &desc2->ptr[5], DMA_FROM_DEVICE); 1712 if (req_ctx->last) 1713 memcpy(areq->result, req_ctx->hw_context, 1714 crypto_ahash_digestsize(tfm)); 1715 1716 if (req_ctx->psrc) 1717 talitos_sg_unmap(dev, edesc, req_ctx->psrc, NULL, 0, 0); 1718 1719 /* When using hashctx-in, must unmap it. */ 1720 if (from_talitos_ptr_len(&edesc->desc.ptr[1], is_sec1)) 1721 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[1], 1722 DMA_TO_DEVICE); 1723 else if (desc->next_desc) 1724 unmap_single_talitos_ptr(dev, &desc2->ptr[1], 1725 DMA_TO_DEVICE); 1726 1727 if (is_sec1 && req_ctx->nbuf) 1728 unmap_single_talitos_ptr(dev, &desc->ptr[3], 1729 DMA_TO_DEVICE); 1730 1731 if (edesc->dma_len) 1732 dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len, 1733 DMA_BIDIRECTIONAL); 1734 1735 if (edesc->desc.next_desc) 1736 dma_unmap_single(dev, be32_to_cpu(edesc->desc.next_desc), 1737 TALITOS_DESC_SIZE, DMA_BIDIRECTIONAL); 1738 } 1739 1740 static void ahash_done(struct device *dev, 1741 struct talitos_desc *desc, void *context, 1742 int err) 1743 { 1744 struct ahash_request *areq = context; 1745 struct talitos_edesc *edesc = 1746 container_of(desc, struct talitos_edesc, desc); 1747 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 1748 1749 if (!req_ctx->last && req_ctx->to_hash_later) { 1750 /* Position any partial block for next update/final/finup */ 1751 req_ctx->buf_idx = (req_ctx->buf_idx + 1) & 1; 1752 req_ctx->nbuf = req_ctx->to_hash_later; 1753 } 1754 common_nonsnoop_hash_unmap(dev, edesc, areq); 1755 1756 kfree(edesc); 1757 1758 areq->base.complete(&areq->base, err); 1759 } 1760 1761 /* 1762 * SEC1 doesn't like hashing of 0 sized message, so we do the padding 1763 * ourself and submit a padded block 1764 */ 1765 static void talitos_handle_buggy_hash(struct talitos_ctx *ctx, 1766 struct talitos_edesc *edesc, 1767 struct talitos_ptr *ptr) 1768 { 1769 static u8 padded_hash[64] = { 1770 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1771 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1772 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1773 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1774 }; 1775 1776 pr_err_once("Bug in SEC1, padding ourself\n"); 1777 edesc->desc.hdr &= ~DESC_HDR_MODE0_MDEU_PAD; 1778 map_single_talitos_ptr(ctx->dev, ptr, sizeof(padded_hash), 1779 (char *)padded_hash, DMA_TO_DEVICE); 1780 } 1781 1782 static int common_nonsnoop_hash(struct talitos_edesc *edesc, 1783 struct ahash_request *areq, unsigned int length, 1784 void (*callback) (struct device *dev, 1785 struct talitos_desc *desc, 1786 void *context, int error)) 1787 { 1788 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); 1789 struct talitos_ctx *ctx = crypto_ahash_ctx(tfm); 1790 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 1791 struct device *dev = ctx->dev; 1792 struct talitos_desc *desc = &edesc->desc; 1793 int ret; 1794 bool sync_needed = false; 1795 struct talitos_private *priv = dev_get_drvdata(dev); 1796 bool is_sec1 = has_ftr_sec1(priv); 1797 int sg_count; 1798 1799 /* first DWORD empty */ 1800 1801 /* hash context in */ 1802 if (!req_ctx->first || req_ctx->swinit) { 1803 map_single_talitos_ptr_nosync(dev, &desc->ptr[1], 1804 req_ctx->hw_context_size, 1805 req_ctx->hw_context, 1806 DMA_TO_DEVICE); 1807 req_ctx->swinit = 0; 1808 } 1809 /* Indicate next op is not the first. */ 1810 req_ctx->first = 0; 1811 1812 /* HMAC key */ 1813 if (ctx->keylen) 1814 to_talitos_ptr(&desc->ptr[2], ctx->dma_key, ctx->keylen, 1815 is_sec1); 1816 1817 if (is_sec1 && req_ctx->nbuf) 1818 length -= req_ctx->nbuf; 1819 1820 sg_count = edesc->src_nents ?: 1; 1821 if (is_sec1 && sg_count > 1) 1822 sg_copy_to_buffer(req_ctx->psrc, sg_count, edesc->buf, length); 1823 else if (length) 1824 sg_count = dma_map_sg(dev, req_ctx->psrc, sg_count, 1825 DMA_TO_DEVICE); 1826 /* 1827 * data in 1828 */ 1829 if (is_sec1 && req_ctx->nbuf) { 1830 map_single_talitos_ptr(dev, &desc->ptr[3], req_ctx->nbuf, 1831 req_ctx->buf[req_ctx->buf_idx], 1832 DMA_TO_DEVICE); 1833 } else { 1834 sg_count = talitos_sg_map(dev, req_ctx->psrc, length, edesc, 1835 &desc->ptr[3], sg_count, 0, 0); 1836 if (sg_count > 1) 1837 sync_needed = true; 1838 } 1839 1840 /* fifth DWORD empty */ 1841 1842 /* hash/HMAC out -or- hash context out */ 1843 if (req_ctx->last) 1844 map_single_talitos_ptr(dev, &desc->ptr[5], 1845 crypto_ahash_digestsize(tfm), 1846 req_ctx->hw_context, DMA_FROM_DEVICE); 1847 else 1848 map_single_talitos_ptr_nosync(dev, &desc->ptr[5], 1849 req_ctx->hw_context_size, 1850 req_ctx->hw_context, 1851 DMA_FROM_DEVICE); 1852 1853 /* last DWORD empty */ 1854 1855 if (is_sec1 && from_talitos_ptr_len(&desc->ptr[3], true) == 0) 1856 talitos_handle_buggy_hash(ctx, edesc, &desc->ptr[3]); 1857 1858 if (is_sec1 && req_ctx->nbuf && length) { 1859 struct talitos_desc *desc2 = (struct talitos_desc *) 1860 (edesc->buf + edesc->dma_len); 1861 dma_addr_t next_desc; 1862 1863 memset(desc2, 0, sizeof(*desc2)); 1864 desc2->hdr = desc->hdr; 1865 desc2->hdr &= ~DESC_HDR_MODE0_MDEU_INIT; 1866 desc2->hdr1 = desc2->hdr; 1867 desc->hdr &= ~DESC_HDR_MODE0_MDEU_PAD; 1868 desc->hdr |= DESC_HDR_MODE0_MDEU_CONT; 1869 desc->hdr &= ~DESC_HDR_DONE_NOTIFY; 1870 1871 if (desc->ptr[1].ptr) 1872 copy_talitos_ptr(&desc2->ptr[1], &desc->ptr[1], 1873 is_sec1); 1874 else 1875 map_single_talitos_ptr_nosync(dev, &desc2->ptr[1], 1876 req_ctx->hw_context_size, 1877 req_ctx->hw_context, 1878 DMA_TO_DEVICE); 1879 copy_talitos_ptr(&desc2->ptr[2], &desc->ptr[2], is_sec1); 1880 sg_count = talitos_sg_map(dev, req_ctx->psrc, length, edesc, 1881 &desc2->ptr[3], sg_count, 0, 0); 1882 if (sg_count > 1) 1883 sync_needed = true; 1884 copy_talitos_ptr(&desc2->ptr[5], &desc->ptr[5], is_sec1); 1885 if (req_ctx->last) 1886 map_single_talitos_ptr_nosync(dev, &desc->ptr[5], 1887 req_ctx->hw_context_size, 1888 req_ctx->hw_context, 1889 DMA_FROM_DEVICE); 1890 1891 next_desc = dma_map_single(dev, &desc2->hdr1, TALITOS_DESC_SIZE, 1892 DMA_BIDIRECTIONAL); 1893 desc->next_desc = cpu_to_be32(next_desc); 1894 } 1895 1896 if (sync_needed) 1897 dma_sync_single_for_device(dev, edesc->dma_link_tbl, 1898 edesc->dma_len, DMA_BIDIRECTIONAL); 1899 1900 ret = talitos_submit(dev, ctx->ch, desc, callback, areq); 1901 if (ret != -EINPROGRESS) { 1902 common_nonsnoop_hash_unmap(dev, edesc, areq); 1903 kfree(edesc); 1904 } 1905 return ret; 1906 } 1907 1908 static struct talitos_edesc *ahash_edesc_alloc(struct ahash_request *areq, 1909 unsigned int nbytes) 1910 { 1911 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); 1912 struct talitos_ctx *ctx = crypto_ahash_ctx(tfm); 1913 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 1914 struct talitos_private *priv = dev_get_drvdata(ctx->dev); 1915 bool is_sec1 = has_ftr_sec1(priv); 1916 1917 if (is_sec1) 1918 nbytes -= req_ctx->nbuf; 1919 1920 return talitos_edesc_alloc(ctx->dev, req_ctx->psrc, NULL, NULL, 0, 1921 nbytes, 0, 0, 0, areq->base.flags, false); 1922 } 1923 1924 static int ahash_init(struct ahash_request *areq) 1925 { 1926 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); 1927 struct talitos_ctx *ctx = crypto_ahash_ctx(tfm); 1928 struct device *dev = ctx->dev; 1929 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 1930 unsigned int size; 1931 dma_addr_t dma; 1932 1933 /* Initialize the context */ 1934 req_ctx->buf_idx = 0; 1935 req_ctx->nbuf = 0; 1936 req_ctx->first = 1; /* first indicates h/w must init its context */ 1937 req_ctx->swinit = 0; /* assume h/w init of context */ 1938 size = (crypto_ahash_digestsize(tfm) <= SHA256_DIGEST_SIZE) 1939 ? TALITOS_MDEU_CONTEXT_SIZE_MD5_SHA1_SHA256 1940 : TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512; 1941 req_ctx->hw_context_size = size; 1942 1943 dma = dma_map_single(dev, req_ctx->hw_context, req_ctx->hw_context_size, 1944 DMA_TO_DEVICE); 1945 dma_unmap_single(dev, dma, req_ctx->hw_context_size, DMA_TO_DEVICE); 1946 1947 return 0; 1948 } 1949 1950 /* 1951 * on h/w without explicit sha224 support, we initialize h/w context 1952 * manually with sha224 constants, and tell it to run sha256. 1953 */ 1954 static int ahash_init_sha224_swinit(struct ahash_request *areq) 1955 { 1956 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 1957 1958 req_ctx->hw_context[0] = SHA224_H0; 1959 req_ctx->hw_context[1] = SHA224_H1; 1960 req_ctx->hw_context[2] = SHA224_H2; 1961 req_ctx->hw_context[3] = SHA224_H3; 1962 req_ctx->hw_context[4] = SHA224_H4; 1963 req_ctx->hw_context[5] = SHA224_H5; 1964 req_ctx->hw_context[6] = SHA224_H6; 1965 req_ctx->hw_context[7] = SHA224_H7; 1966 1967 /* init 64-bit count */ 1968 req_ctx->hw_context[8] = 0; 1969 req_ctx->hw_context[9] = 0; 1970 1971 ahash_init(areq); 1972 req_ctx->swinit = 1;/* prevent h/w initting context with sha256 values*/ 1973 1974 return 0; 1975 } 1976 1977 static int ahash_process_req(struct ahash_request *areq, unsigned int nbytes) 1978 { 1979 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); 1980 struct talitos_ctx *ctx = crypto_ahash_ctx(tfm); 1981 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 1982 struct talitos_edesc *edesc; 1983 unsigned int blocksize = 1984 crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm)); 1985 unsigned int nbytes_to_hash; 1986 unsigned int to_hash_later; 1987 unsigned int nsg; 1988 int nents; 1989 struct device *dev = ctx->dev; 1990 struct talitos_private *priv = dev_get_drvdata(dev); 1991 bool is_sec1 = has_ftr_sec1(priv); 1992 u8 *ctx_buf = req_ctx->buf[req_ctx->buf_idx]; 1993 1994 if (!req_ctx->last && (nbytes + req_ctx->nbuf <= blocksize)) { 1995 /* Buffer up to one whole block */ 1996 nents = sg_nents_for_len(areq->src, nbytes); 1997 if (nents < 0) { 1998 dev_err(ctx->dev, "Invalid number of src SG.\n"); 1999 return nents; 2000 } 2001 sg_copy_to_buffer(areq->src, nents, 2002 ctx_buf + req_ctx->nbuf, nbytes); 2003 req_ctx->nbuf += nbytes; 2004 return 0; 2005 } 2006 2007 /* At least (blocksize + 1) bytes are available to hash */ 2008 nbytes_to_hash = nbytes + req_ctx->nbuf; 2009 to_hash_later = nbytes_to_hash & (blocksize - 1); 2010 2011 if (req_ctx->last) 2012 to_hash_later = 0; 2013 else if (to_hash_later) 2014 /* There is a partial block. Hash the full block(s) now */ 2015 nbytes_to_hash -= to_hash_later; 2016 else { 2017 /* Keep one block buffered */ 2018 nbytes_to_hash -= blocksize; 2019 to_hash_later = blocksize; 2020 } 2021 2022 /* Chain in any previously buffered data */ 2023 if (!is_sec1 && req_ctx->nbuf) { 2024 nsg = (req_ctx->nbuf < nbytes_to_hash) ? 2 : 1; 2025 sg_init_table(req_ctx->bufsl, nsg); 2026 sg_set_buf(req_ctx->bufsl, ctx_buf, req_ctx->nbuf); 2027 if (nsg > 1) 2028 sg_chain(req_ctx->bufsl, 2, areq->src); 2029 req_ctx->psrc = req_ctx->bufsl; 2030 } else if (is_sec1 && req_ctx->nbuf && req_ctx->nbuf < blocksize) { 2031 int offset; 2032 2033 if (nbytes_to_hash > blocksize) 2034 offset = blocksize - req_ctx->nbuf; 2035 else 2036 offset = nbytes_to_hash - req_ctx->nbuf; 2037 nents = sg_nents_for_len(areq->src, offset); 2038 if (nents < 0) { 2039 dev_err(ctx->dev, "Invalid number of src SG.\n"); 2040 return nents; 2041 } 2042 sg_copy_to_buffer(areq->src, nents, 2043 ctx_buf + req_ctx->nbuf, offset); 2044 req_ctx->nbuf += offset; 2045 req_ctx->psrc = scatterwalk_ffwd(req_ctx->bufsl, areq->src, 2046 offset); 2047 } else 2048 req_ctx->psrc = areq->src; 2049 2050 if (to_hash_later) { 2051 nents = sg_nents_for_len(areq->src, nbytes); 2052 if (nents < 0) { 2053 dev_err(ctx->dev, "Invalid number of src SG.\n"); 2054 return nents; 2055 } 2056 sg_pcopy_to_buffer(areq->src, nents, 2057 req_ctx->buf[(req_ctx->buf_idx + 1) & 1], 2058 to_hash_later, 2059 nbytes - to_hash_later); 2060 } 2061 req_ctx->to_hash_later = to_hash_later; 2062 2063 /* Allocate extended descriptor */ 2064 edesc = ahash_edesc_alloc(areq, nbytes_to_hash); 2065 if (IS_ERR(edesc)) 2066 return PTR_ERR(edesc); 2067 2068 edesc->desc.hdr = ctx->desc_hdr_template; 2069 2070 /* On last one, request SEC to pad; otherwise continue */ 2071 if (req_ctx->last) 2072 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_PAD; 2073 else 2074 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_CONT; 2075 2076 /* request SEC to INIT hash. */ 2077 if (req_ctx->first && !req_ctx->swinit) 2078 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_INIT; 2079 2080 /* When the tfm context has a keylen, it's an HMAC. 2081 * A first or last (ie. not middle) descriptor must request HMAC. 2082 */ 2083 if (ctx->keylen && (req_ctx->first || req_ctx->last)) 2084 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_HMAC; 2085 2086 return common_nonsnoop_hash(edesc, areq, nbytes_to_hash, ahash_done); 2087 } 2088 2089 static int ahash_update(struct ahash_request *areq) 2090 { 2091 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 2092 2093 req_ctx->last = 0; 2094 2095 return ahash_process_req(areq, areq->nbytes); 2096 } 2097 2098 static int ahash_final(struct ahash_request *areq) 2099 { 2100 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 2101 2102 req_ctx->last = 1; 2103 2104 return ahash_process_req(areq, 0); 2105 } 2106 2107 static int ahash_finup(struct ahash_request *areq) 2108 { 2109 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 2110 2111 req_ctx->last = 1; 2112 2113 return ahash_process_req(areq, areq->nbytes); 2114 } 2115 2116 static int ahash_digest(struct ahash_request *areq) 2117 { 2118 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 2119 struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq); 2120 2121 ahash->init(areq); 2122 req_ctx->last = 1; 2123 2124 return ahash_process_req(areq, areq->nbytes); 2125 } 2126 2127 static int ahash_export(struct ahash_request *areq, void *out) 2128 { 2129 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 2130 struct talitos_export_state *export = out; 2131 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); 2132 struct talitos_ctx *ctx = crypto_ahash_ctx(tfm); 2133 struct device *dev = ctx->dev; 2134 dma_addr_t dma; 2135 2136 dma = dma_map_single(dev, req_ctx->hw_context, req_ctx->hw_context_size, 2137 DMA_FROM_DEVICE); 2138 dma_unmap_single(dev, dma, req_ctx->hw_context_size, DMA_FROM_DEVICE); 2139 2140 memcpy(export->hw_context, req_ctx->hw_context, 2141 req_ctx->hw_context_size); 2142 memcpy(export->buf, req_ctx->buf[req_ctx->buf_idx], req_ctx->nbuf); 2143 export->swinit = req_ctx->swinit; 2144 export->first = req_ctx->first; 2145 export->last = req_ctx->last; 2146 export->to_hash_later = req_ctx->to_hash_later; 2147 export->nbuf = req_ctx->nbuf; 2148 2149 return 0; 2150 } 2151 2152 static int ahash_import(struct ahash_request *areq, const void *in) 2153 { 2154 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 2155 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); 2156 struct talitos_ctx *ctx = crypto_ahash_ctx(tfm); 2157 struct device *dev = ctx->dev; 2158 const struct talitos_export_state *export = in; 2159 unsigned int size; 2160 dma_addr_t dma; 2161 2162 memset(req_ctx, 0, sizeof(*req_ctx)); 2163 size = (crypto_ahash_digestsize(tfm) <= SHA256_DIGEST_SIZE) 2164 ? TALITOS_MDEU_CONTEXT_SIZE_MD5_SHA1_SHA256 2165 : TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512; 2166 req_ctx->hw_context_size = size; 2167 memcpy(req_ctx->hw_context, export->hw_context, size); 2168 memcpy(req_ctx->buf[0], export->buf, export->nbuf); 2169 req_ctx->swinit = export->swinit; 2170 req_ctx->first = export->first; 2171 req_ctx->last = export->last; 2172 req_ctx->to_hash_later = export->to_hash_later; 2173 req_ctx->nbuf = export->nbuf; 2174 2175 dma = dma_map_single(dev, req_ctx->hw_context, req_ctx->hw_context_size, 2176 DMA_TO_DEVICE); 2177 dma_unmap_single(dev, dma, req_ctx->hw_context_size, DMA_TO_DEVICE); 2178 2179 return 0; 2180 } 2181 2182 static int keyhash(struct crypto_ahash *tfm, const u8 *key, unsigned int keylen, 2183 u8 *hash) 2184 { 2185 struct talitos_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm)); 2186 2187 struct scatterlist sg[1]; 2188 struct ahash_request *req; 2189 struct crypto_wait wait; 2190 int ret; 2191 2192 crypto_init_wait(&wait); 2193 2194 req = ahash_request_alloc(tfm, GFP_KERNEL); 2195 if (!req) 2196 return -ENOMEM; 2197 2198 /* Keep tfm keylen == 0 during hash of the long key */ 2199 ctx->keylen = 0; 2200 ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, 2201 crypto_req_done, &wait); 2202 2203 sg_init_one(&sg[0], key, keylen); 2204 2205 ahash_request_set_crypt(req, sg, hash, keylen); 2206 ret = crypto_wait_req(crypto_ahash_digest(req), &wait); 2207 2208 ahash_request_free(req); 2209 2210 return ret; 2211 } 2212 2213 static int ahash_setkey(struct crypto_ahash *tfm, const u8 *key, 2214 unsigned int keylen) 2215 { 2216 struct talitos_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm)); 2217 struct device *dev = ctx->dev; 2218 unsigned int blocksize = 2219 crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm)); 2220 unsigned int digestsize = crypto_ahash_digestsize(tfm); 2221 unsigned int keysize = keylen; 2222 u8 hash[SHA512_DIGEST_SIZE]; 2223 int ret; 2224 2225 if (keylen <= blocksize) 2226 memcpy(ctx->key, key, keysize); 2227 else { 2228 /* Must get the hash of the long key */ 2229 ret = keyhash(tfm, key, keylen, hash); 2230 2231 if (ret) 2232 return -EINVAL; 2233 2234 keysize = digestsize; 2235 memcpy(ctx->key, hash, digestsize); 2236 } 2237 2238 if (ctx->keylen) 2239 dma_unmap_single(dev, ctx->dma_key, ctx->keylen, DMA_TO_DEVICE); 2240 2241 ctx->keylen = keysize; 2242 ctx->dma_key = dma_map_single(dev, ctx->key, keysize, DMA_TO_DEVICE); 2243 2244 return 0; 2245 } 2246 2247 2248 struct talitos_alg_template { 2249 u32 type; 2250 u32 priority; 2251 union { 2252 struct skcipher_alg skcipher; 2253 struct ahash_alg hash; 2254 struct aead_alg aead; 2255 } alg; 2256 __be32 desc_hdr_template; 2257 }; 2258 2259 static struct talitos_alg_template driver_algs[] = { 2260 /* AEAD algorithms. These use a single-pass ipsec_esp descriptor */ 2261 { .type = CRYPTO_ALG_TYPE_AEAD, 2262 .alg.aead = { 2263 .base = { 2264 .cra_name = "authenc(hmac(sha1),cbc(aes))", 2265 .cra_driver_name = "authenc-hmac-sha1-" 2266 "cbc-aes-talitos", 2267 .cra_blocksize = AES_BLOCK_SIZE, 2268 .cra_flags = CRYPTO_ALG_ASYNC | 2269 CRYPTO_ALG_ALLOCATES_MEMORY, 2270 }, 2271 .ivsize = AES_BLOCK_SIZE, 2272 .maxauthsize = SHA1_DIGEST_SIZE, 2273 }, 2274 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | 2275 DESC_HDR_SEL0_AESU | 2276 DESC_HDR_MODE0_AESU_CBC | 2277 DESC_HDR_SEL1_MDEUA | 2278 DESC_HDR_MODE1_MDEU_INIT | 2279 DESC_HDR_MODE1_MDEU_PAD | 2280 DESC_HDR_MODE1_MDEU_SHA1_HMAC, 2281 }, 2282 { .type = CRYPTO_ALG_TYPE_AEAD, 2283 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA, 2284 .alg.aead = { 2285 .base = { 2286 .cra_name = "authenc(hmac(sha1),cbc(aes))", 2287 .cra_driver_name = "authenc-hmac-sha1-" 2288 "cbc-aes-talitos-hsna", 2289 .cra_blocksize = AES_BLOCK_SIZE, 2290 .cra_flags = CRYPTO_ALG_ASYNC | 2291 CRYPTO_ALG_ALLOCATES_MEMORY, 2292 }, 2293 .ivsize = AES_BLOCK_SIZE, 2294 .maxauthsize = SHA1_DIGEST_SIZE, 2295 }, 2296 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU | 2297 DESC_HDR_SEL0_AESU | 2298 DESC_HDR_MODE0_AESU_CBC | 2299 DESC_HDR_SEL1_MDEUA | 2300 DESC_HDR_MODE1_MDEU_INIT | 2301 DESC_HDR_MODE1_MDEU_PAD | 2302 DESC_HDR_MODE1_MDEU_SHA1_HMAC, 2303 }, 2304 { .type = CRYPTO_ALG_TYPE_AEAD, 2305 .alg.aead = { 2306 .base = { 2307 .cra_name = "authenc(hmac(sha1)," 2308 "cbc(des3_ede))", 2309 .cra_driver_name = "authenc-hmac-sha1-" 2310 "cbc-3des-talitos", 2311 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2312 .cra_flags = CRYPTO_ALG_ASYNC | 2313 CRYPTO_ALG_ALLOCATES_MEMORY, 2314 }, 2315 .ivsize = DES3_EDE_BLOCK_SIZE, 2316 .maxauthsize = SHA1_DIGEST_SIZE, 2317 .setkey = aead_des3_setkey, 2318 }, 2319 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | 2320 DESC_HDR_SEL0_DEU | 2321 DESC_HDR_MODE0_DEU_CBC | 2322 DESC_HDR_MODE0_DEU_3DES | 2323 DESC_HDR_SEL1_MDEUA | 2324 DESC_HDR_MODE1_MDEU_INIT | 2325 DESC_HDR_MODE1_MDEU_PAD | 2326 DESC_HDR_MODE1_MDEU_SHA1_HMAC, 2327 }, 2328 { .type = CRYPTO_ALG_TYPE_AEAD, 2329 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA, 2330 .alg.aead = { 2331 .base = { 2332 .cra_name = "authenc(hmac(sha1)," 2333 "cbc(des3_ede))", 2334 .cra_driver_name = "authenc-hmac-sha1-" 2335 "cbc-3des-talitos-hsna", 2336 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2337 .cra_flags = CRYPTO_ALG_ASYNC | 2338 CRYPTO_ALG_ALLOCATES_MEMORY, 2339 }, 2340 .ivsize = DES3_EDE_BLOCK_SIZE, 2341 .maxauthsize = SHA1_DIGEST_SIZE, 2342 .setkey = aead_des3_setkey, 2343 }, 2344 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU | 2345 DESC_HDR_SEL0_DEU | 2346 DESC_HDR_MODE0_DEU_CBC | 2347 DESC_HDR_MODE0_DEU_3DES | 2348 DESC_HDR_SEL1_MDEUA | 2349 DESC_HDR_MODE1_MDEU_INIT | 2350 DESC_HDR_MODE1_MDEU_PAD | 2351 DESC_HDR_MODE1_MDEU_SHA1_HMAC, 2352 }, 2353 { .type = CRYPTO_ALG_TYPE_AEAD, 2354 .alg.aead = { 2355 .base = { 2356 .cra_name = "authenc(hmac(sha224),cbc(aes))", 2357 .cra_driver_name = "authenc-hmac-sha224-" 2358 "cbc-aes-talitos", 2359 .cra_blocksize = AES_BLOCK_SIZE, 2360 .cra_flags = CRYPTO_ALG_ASYNC | 2361 CRYPTO_ALG_ALLOCATES_MEMORY, 2362 }, 2363 .ivsize = AES_BLOCK_SIZE, 2364 .maxauthsize = SHA224_DIGEST_SIZE, 2365 }, 2366 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | 2367 DESC_HDR_SEL0_AESU | 2368 DESC_HDR_MODE0_AESU_CBC | 2369 DESC_HDR_SEL1_MDEUA | 2370 DESC_HDR_MODE1_MDEU_INIT | 2371 DESC_HDR_MODE1_MDEU_PAD | 2372 DESC_HDR_MODE1_MDEU_SHA224_HMAC, 2373 }, 2374 { .type = CRYPTO_ALG_TYPE_AEAD, 2375 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA, 2376 .alg.aead = { 2377 .base = { 2378 .cra_name = "authenc(hmac(sha224),cbc(aes))", 2379 .cra_driver_name = "authenc-hmac-sha224-" 2380 "cbc-aes-talitos-hsna", 2381 .cra_blocksize = AES_BLOCK_SIZE, 2382 .cra_flags = CRYPTO_ALG_ASYNC | 2383 CRYPTO_ALG_ALLOCATES_MEMORY, 2384 }, 2385 .ivsize = AES_BLOCK_SIZE, 2386 .maxauthsize = SHA224_DIGEST_SIZE, 2387 }, 2388 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU | 2389 DESC_HDR_SEL0_AESU | 2390 DESC_HDR_MODE0_AESU_CBC | 2391 DESC_HDR_SEL1_MDEUA | 2392 DESC_HDR_MODE1_MDEU_INIT | 2393 DESC_HDR_MODE1_MDEU_PAD | 2394 DESC_HDR_MODE1_MDEU_SHA224_HMAC, 2395 }, 2396 { .type = CRYPTO_ALG_TYPE_AEAD, 2397 .alg.aead = { 2398 .base = { 2399 .cra_name = "authenc(hmac(sha224)," 2400 "cbc(des3_ede))", 2401 .cra_driver_name = "authenc-hmac-sha224-" 2402 "cbc-3des-talitos", 2403 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2404 .cra_flags = CRYPTO_ALG_ASYNC | 2405 CRYPTO_ALG_ALLOCATES_MEMORY, 2406 }, 2407 .ivsize = DES3_EDE_BLOCK_SIZE, 2408 .maxauthsize = SHA224_DIGEST_SIZE, 2409 .setkey = aead_des3_setkey, 2410 }, 2411 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | 2412 DESC_HDR_SEL0_DEU | 2413 DESC_HDR_MODE0_DEU_CBC | 2414 DESC_HDR_MODE0_DEU_3DES | 2415 DESC_HDR_SEL1_MDEUA | 2416 DESC_HDR_MODE1_MDEU_INIT | 2417 DESC_HDR_MODE1_MDEU_PAD | 2418 DESC_HDR_MODE1_MDEU_SHA224_HMAC, 2419 }, 2420 { .type = CRYPTO_ALG_TYPE_AEAD, 2421 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA, 2422 .alg.aead = { 2423 .base = { 2424 .cra_name = "authenc(hmac(sha224)," 2425 "cbc(des3_ede))", 2426 .cra_driver_name = "authenc-hmac-sha224-" 2427 "cbc-3des-talitos-hsna", 2428 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2429 .cra_flags = CRYPTO_ALG_ASYNC | 2430 CRYPTO_ALG_ALLOCATES_MEMORY, 2431 }, 2432 .ivsize = DES3_EDE_BLOCK_SIZE, 2433 .maxauthsize = SHA224_DIGEST_SIZE, 2434 .setkey = aead_des3_setkey, 2435 }, 2436 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU | 2437 DESC_HDR_SEL0_DEU | 2438 DESC_HDR_MODE0_DEU_CBC | 2439 DESC_HDR_MODE0_DEU_3DES | 2440 DESC_HDR_SEL1_MDEUA | 2441 DESC_HDR_MODE1_MDEU_INIT | 2442 DESC_HDR_MODE1_MDEU_PAD | 2443 DESC_HDR_MODE1_MDEU_SHA224_HMAC, 2444 }, 2445 { .type = CRYPTO_ALG_TYPE_AEAD, 2446 .alg.aead = { 2447 .base = { 2448 .cra_name = "authenc(hmac(sha256),cbc(aes))", 2449 .cra_driver_name = "authenc-hmac-sha256-" 2450 "cbc-aes-talitos", 2451 .cra_blocksize = AES_BLOCK_SIZE, 2452 .cra_flags = CRYPTO_ALG_ASYNC | 2453 CRYPTO_ALG_ALLOCATES_MEMORY, 2454 }, 2455 .ivsize = AES_BLOCK_SIZE, 2456 .maxauthsize = SHA256_DIGEST_SIZE, 2457 }, 2458 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | 2459 DESC_HDR_SEL0_AESU | 2460 DESC_HDR_MODE0_AESU_CBC | 2461 DESC_HDR_SEL1_MDEUA | 2462 DESC_HDR_MODE1_MDEU_INIT | 2463 DESC_HDR_MODE1_MDEU_PAD | 2464 DESC_HDR_MODE1_MDEU_SHA256_HMAC, 2465 }, 2466 { .type = CRYPTO_ALG_TYPE_AEAD, 2467 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA, 2468 .alg.aead = { 2469 .base = { 2470 .cra_name = "authenc(hmac(sha256),cbc(aes))", 2471 .cra_driver_name = "authenc-hmac-sha256-" 2472 "cbc-aes-talitos-hsna", 2473 .cra_blocksize = AES_BLOCK_SIZE, 2474 .cra_flags = CRYPTO_ALG_ASYNC | 2475 CRYPTO_ALG_ALLOCATES_MEMORY, 2476 }, 2477 .ivsize = AES_BLOCK_SIZE, 2478 .maxauthsize = SHA256_DIGEST_SIZE, 2479 }, 2480 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU | 2481 DESC_HDR_SEL0_AESU | 2482 DESC_HDR_MODE0_AESU_CBC | 2483 DESC_HDR_SEL1_MDEUA | 2484 DESC_HDR_MODE1_MDEU_INIT | 2485 DESC_HDR_MODE1_MDEU_PAD | 2486 DESC_HDR_MODE1_MDEU_SHA256_HMAC, 2487 }, 2488 { .type = CRYPTO_ALG_TYPE_AEAD, 2489 .alg.aead = { 2490 .base = { 2491 .cra_name = "authenc(hmac(sha256)," 2492 "cbc(des3_ede))", 2493 .cra_driver_name = "authenc-hmac-sha256-" 2494 "cbc-3des-talitos", 2495 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2496 .cra_flags = CRYPTO_ALG_ASYNC | 2497 CRYPTO_ALG_ALLOCATES_MEMORY, 2498 }, 2499 .ivsize = DES3_EDE_BLOCK_SIZE, 2500 .maxauthsize = SHA256_DIGEST_SIZE, 2501 .setkey = aead_des3_setkey, 2502 }, 2503 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | 2504 DESC_HDR_SEL0_DEU | 2505 DESC_HDR_MODE0_DEU_CBC | 2506 DESC_HDR_MODE0_DEU_3DES | 2507 DESC_HDR_SEL1_MDEUA | 2508 DESC_HDR_MODE1_MDEU_INIT | 2509 DESC_HDR_MODE1_MDEU_PAD | 2510 DESC_HDR_MODE1_MDEU_SHA256_HMAC, 2511 }, 2512 { .type = CRYPTO_ALG_TYPE_AEAD, 2513 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA, 2514 .alg.aead = { 2515 .base = { 2516 .cra_name = "authenc(hmac(sha256)," 2517 "cbc(des3_ede))", 2518 .cra_driver_name = "authenc-hmac-sha256-" 2519 "cbc-3des-talitos-hsna", 2520 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2521 .cra_flags = CRYPTO_ALG_ASYNC | 2522 CRYPTO_ALG_ALLOCATES_MEMORY, 2523 }, 2524 .ivsize = DES3_EDE_BLOCK_SIZE, 2525 .maxauthsize = SHA256_DIGEST_SIZE, 2526 .setkey = aead_des3_setkey, 2527 }, 2528 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU | 2529 DESC_HDR_SEL0_DEU | 2530 DESC_HDR_MODE0_DEU_CBC | 2531 DESC_HDR_MODE0_DEU_3DES | 2532 DESC_HDR_SEL1_MDEUA | 2533 DESC_HDR_MODE1_MDEU_INIT | 2534 DESC_HDR_MODE1_MDEU_PAD | 2535 DESC_HDR_MODE1_MDEU_SHA256_HMAC, 2536 }, 2537 { .type = CRYPTO_ALG_TYPE_AEAD, 2538 .alg.aead = { 2539 .base = { 2540 .cra_name = "authenc(hmac(sha384),cbc(aes))", 2541 .cra_driver_name = "authenc-hmac-sha384-" 2542 "cbc-aes-talitos", 2543 .cra_blocksize = AES_BLOCK_SIZE, 2544 .cra_flags = CRYPTO_ALG_ASYNC | 2545 CRYPTO_ALG_ALLOCATES_MEMORY, 2546 }, 2547 .ivsize = AES_BLOCK_SIZE, 2548 .maxauthsize = SHA384_DIGEST_SIZE, 2549 }, 2550 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | 2551 DESC_HDR_SEL0_AESU | 2552 DESC_HDR_MODE0_AESU_CBC | 2553 DESC_HDR_SEL1_MDEUB | 2554 DESC_HDR_MODE1_MDEU_INIT | 2555 DESC_HDR_MODE1_MDEU_PAD | 2556 DESC_HDR_MODE1_MDEUB_SHA384_HMAC, 2557 }, 2558 { .type = CRYPTO_ALG_TYPE_AEAD, 2559 .alg.aead = { 2560 .base = { 2561 .cra_name = "authenc(hmac(sha384)," 2562 "cbc(des3_ede))", 2563 .cra_driver_name = "authenc-hmac-sha384-" 2564 "cbc-3des-talitos", 2565 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2566 .cra_flags = CRYPTO_ALG_ASYNC | 2567 CRYPTO_ALG_ALLOCATES_MEMORY, 2568 }, 2569 .ivsize = DES3_EDE_BLOCK_SIZE, 2570 .maxauthsize = SHA384_DIGEST_SIZE, 2571 .setkey = aead_des3_setkey, 2572 }, 2573 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | 2574 DESC_HDR_SEL0_DEU | 2575 DESC_HDR_MODE0_DEU_CBC | 2576 DESC_HDR_MODE0_DEU_3DES | 2577 DESC_HDR_SEL1_MDEUB | 2578 DESC_HDR_MODE1_MDEU_INIT | 2579 DESC_HDR_MODE1_MDEU_PAD | 2580 DESC_HDR_MODE1_MDEUB_SHA384_HMAC, 2581 }, 2582 { .type = CRYPTO_ALG_TYPE_AEAD, 2583 .alg.aead = { 2584 .base = { 2585 .cra_name = "authenc(hmac(sha512),cbc(aes))", 2586 .cra_driver_name = "authenc-hmac-sha512-" 2587 "cbc-aes-talitos", 2588 .cra_blocksize = AES_BLOCK_SIZE, 2589 .cra_flags = CRYPTO_ALG_ASYNC | 2590 CRYPTO_ALG_ALLOCATES_MEMORY, 2591 }, 2592 .ivsize = AES_BLOCK_SIZE, 2593 .maxauthsize = SHA512_DIGEST_SIZE, 2594 }, 2595 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | 2596 DESC_HDR_SEL0_AESU | 2597 DESC_HDR_MODE0_AESU_CBC | 2598 DESC_HDR_SEL1_MDEUB | 2599 DESC_HDR_MODE1_MDEU_INIT | 2600 DESC_HDR_MODE1_MDEU_PAD | 2601 DESC_HDR_MODE1_MDEUB_SHA512_HMAC, 2602 }, 2603 { .type = CRYPTO_ALG_TYPE_AEAD, 2604 .alg.aead = { 2605 .base = { 2606 .cra_name = "authenc(hmac(sha512)," 2607 "cbc(des3_ede))", 2608 .cra_driver_name = "authenc-hmac-sha512-" 2609 "cbc-3des-talitos", 2610 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2611 .cra_flags = CRYPTO_ALG_ASYNC | 2612 CRYPTO_ALG_ALLOCATES_MEMORY, 2613 }, 2614 .ivsize = DES3_EDE_BLOCK_SIZE, 2615 .maxauthsize = SHA512_DIGEST_SIZE, 2616 .setkey = aead_des3_setkey, 2617 }, 2618 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | 2619 DESC_HDR_SEL0_DEU | 2620 DESC_HDR_MODE0_DEU_CBC | 2621 DESC_HDR_MODE0_DEU_3DES | 2622 DESC_HDR_SEL1_MDEUB | 2623 DESC_HDR_MODE1_MDEU_INIT | 2624 DESC_HDR_MODE1_MDEU_PAD | 2625 DESC_HDR_MODE1_MDEUB_SHA512_HMAC, 2626 }, 2627 { .type = CRYPTO_ALG_TYPE_AEAD, 2628 .alg.aead = { 2629 .base = { 2630 .cra_name = "authenc(hmac(md5),cbc(aes))", 2631 .cra_driver_name = "authenc-hmac-md5-" 2632 "cbc-aes-talitos", 2633 .cra_blocksize = AES_BLOCK_SIZE, 2634 .cra_flags = CRYPTO_ALG_ASYNC | 2635 CRYPTO_ALG_ALLOCATES_MEMORY, 2636 }, 2637 .ivsize = AES_BLOCK_SIZE, 2638 .maxauthsize = MD5_DIGEST_SIZE, 2639 }, 2640 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | 2641 DESC_HDR_SEL0_AESU | 2642 DESC_HDR_MODE0_AESU_CBC | 2643 DESC_HDR_SEL1_MDEUA | 2644 DESC_HDR_MODE1_MDEU_INIT | 2645 DESC_HDR_MODE1_MDEU_PAD | 2646 DESC_HDR_MODE1_MDEU_MD5_HMAC, 2647 }, 2648 { .type = CRYPTO_ALG_TYPE_AEAD, 2649 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA, 2650 .alg.aead = { 2651 .base = { 2652 .cra_name = "authenc(hmac(md5),cbc(aes))", 2653 .cra_driver_name = "authenc-hmac-md5-" 2654 "cbc-aes-talitos-hsna", 2655 .cra_blocksize = AES_BLOCK_SIZE, 2656 .cra_flags = CRYPTO_ALG_ASYNC | 2657 CRYPTO_ALG_ALLOCATES_MEMORY, 2658 }, 2659 .ivsize = AES_BLOCK_SIZE, 2660 .maxauthsize = MD5_DIGEST_SIZE, 2661 }, 2662 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU | 2663 DESC_HDR_SEL0_AESU | 2664 DESC_HDR_MODE0_AESU_CBC | 2665 DESC_HDR_SEL1_MDEUA | 2666 DESC_HDR_MODE1_MDEU_INIT | 2667 DESC_HDR_MODE1_MDEU_PAD | 2668 DESC_HDR_MODE1_MDEU_MD5_HMAC, 2669 }, 2670 { .type = CRYPTO_ALG_TYPE_AEAD, 2671 .alg.aead = { 2672 .base = { 2673 .cra_name = "authenc(hmac(md5),cbc(des3_ede))", 2674 .cra_driver_name = "authenc-hmac-md5-" 2675 "cbc-3des-talitos", 2676 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2677 .cra_flags = CRYPTO_ALG_ASYNC | 2678 CRYPTO_ALG_ALLOCATES_MEMORY, 2679 }, 2680 .ivsize = DES3_EDE_BLOCK_SIZE, 2681 .maxauthsize = MD5_DIGEST_SIZE, 2682 .setkey = aead_des3_setkey, 2683 }, 2684 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | 2685 DESC_HDR_SEL0_DEU | 2686 DESC_HDR_MODE0_DEU_CBC | 2687 DESC_HDR_MODE0_DEU_3DES | 2688 DESC_HDR_SEL1_MDEUA | 2689 DESC_HDR_MODE1_MDEU_INIT | 2690 DESC_HDR_MODE1_MDEU_PAD | 2691 DESC_HDR_MODE1_MDEU_MD5_HMAC, 2692 }, 2693 { .type = CRYPTO_ALG_TYPE_AEAD, 2694 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA, 2695 .alg.aead = { 2696 .base = { 2697 .cra_name = "authenc(hmac(md5),cbc(des3_ede))", 2698 .cra_driver_name = "authenc-hmac-md5-" 2699 "cbc-3des-talitos-hsna", 2700 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2701 .cra_flags = CRYPTO_ALG_ASYNC | 2702 CRYPTO_ALG_ALLOCATES_MEMORY, 2703 }, 2704 .ivsize = DES3_EDE_BLOCK_SIZE, 2705 .maxauthsize = MD5_DIGEST_SIZE, 2706 .setkey = aead_des3_setkey, 2707 }, 2708 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU | 2709 DESC_HDR_SEL0_DEU | 2710 DESC_HDR_MODE0_DEU_CBC | 2711 DESC_HDR_MODE0_DEU_3DES | 2712 DESC_HDR_SEL1_MDEUA | 2713 DESC_HDR_MODE1_MDEU_INIT | 2714 DESC_HDR_MODE1_MDEU_PAD | 2715 DESC_HDR_MODE1_MDEU_MD5_HMAC, 2716 }, 2717 /* SKCIPHER algorithms. */ 2718 { .type = CRYPTO_ALG_TYPE_SKCIPHER, 2719 .alg.skcipher = { 2720 .base.cra_name = "ecb(aes)", 2721 .base.cra_driver_name = "ecb-aes-talitos", 2722 .base.cra_blocksize = AES_BLOCK_SIZE, 2723 .base.cra_flags = CRYPTO_ALG_ASYNC | 2724 CRYPTO_ALG_ALLOCATES_MEMORY, 2725 .min_keysize = AES_MIN_KEY_SIZE, 2726 .max_keysize = AES_MAX_KEY_SIZE, 2727 .setkey = skcipher_aes_setkey, 2728 }, 2729 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 2730 DESC_HDR_SEL0_AESU, 2731 }, 2732 { .type = CRYPTO_ALG_TYPE_SKCIPHER, 2733 .alg.skcipher = { 2734 .base.cra_name = "cbc(aes)", 2735 .base.cra_driver_name = "cbc-aes-talitos", 2736 .base.cra_blocksize = AES_BLOCK_SIZE, 2737 .base.cra_flags = CRYPTO_ALG_ASYNC | 2738 CRYPTO_ALG_ALLOCATES_MEMORY, 2739 .min_keysize = AES_MIN_KEY_SIZE, 2740 .max_keysize = AES_MAX_KEY_SIZE, 2741 .ivsize = AES_BLOCK_SIZE, 2742 .setkey = skcipher_aes_setkey, 2743 }, 2744 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 2745 DESC_HDR_SEL0_AESU | 2746 DESC_HDR_MODE0_AESU_CBC, 2747 }, 2748 { .type = CRYPTO_ALG_TYPE_SKCIPHER, 2749 .alg.skcipher = { 2750 .base.cra_name = "ctr(aes)", 2751 .base.cra_driver_name = "ctr-aes-talitos", 2752 .base.cra_blocksize = 1, 2753 .base.cra_flags = CRYPTO_ALG_ASYNC | 2754 CRYPTO_ALG_ALLOCATES_MEMORY, 2755 .min_keysize = AES_MIN_KEY_SIZE, 2756 .max_keysize = AES_MAX_KEY_SIZE, 2757 .ivsize = AES_BLOCK_SIZE, 2758 .setkey = skcipher_aes_setkey, 2759 }, 2760 .desc_hdr_template = DESC_HDR_TYPE_AESU_CTR_NONSNOOP | 2761 DESC_HDR_SEL0_AESU | 2762 DESC_HDR_MODE0_AESU_CTR, 2763 }, 2764 { .type = CRYPTO_ALG_TYPE_SKCIPHER, 2765 .alg.skcipher = { 2766 .base.cra_name = "ecb(des)", 2767 .base.cra_driver_name = "ecb-des-talitos", 2768 .base.cra_blocksize = DES_BLOCK_SIZE, 2769 .base.cra_flags = CRYPTO_ALG_ASYNC | 2770 CRYPTO_ALG_ALLOCATES_MEMORY, 2771 .min_keysize = DES_KEY_SIZE, 2772 .max_keysize = DES_KEY_SIZE, 2773 .setkey = skcipher_des_setkey, 2774 }, 2775 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 2776 DESC_HDR_SEL0_DEU, 2777 }, 2778 { .type = CRYPTO_ALG_TYPE_SKCIPHER, 2779 .alg.skcipher = { 2780 .base.cra_name = "cbc(des)", 2781 .base.cra_driver_name = "cbc-des-talitos", 2782 .base.cra_blocksize = DES_BLOCK_SIZE, 2783 .base.cra_flags = CRYPTO_ALG_ASYNC | 2784 CRYPTO_ALG_ALLOCATES_MEMORY, 2785 .min_keysize = DES_KEY_SIZE, 2786 .max_keysize = DES_KEY_SIZE, 2787 .ivsize = DES_BLOCK_SIZE, 2788 .setkey = skcipher_des_setkey, 2789 }, 2790 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 2791 DESC_HDR_SEL0_DEU | 2792 DESC_HDR_MODE0_DEU_CBC, 2793 }, 2794 { .type = CRYPTO_ALG_TYPE_SKCIPHER, 2795 .alg.skcipher = { 2796 .base.cra_name = "ecb(des3_ede)", 2797 .base.cra_driver_name = "ecb-3des-talitos", 2798 .base.cra_blocksize = DES3_EDE_BLOCK_SIZE, 2799 .base.cra_flags = CRYPTO_ALG_ASYNC | 2800 CRYPTO_ALG_ALLOCATES_MEMORY, 2801 .min_keysize = DES3_EDE_KEY_SIZE, 2802 .max_keysize = DES3_EDE_KEY_SIZE, 2803 .setkey = skcipher_des3_setkey, 2804 }, 2805 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 2806 DESC_HDR_SEL0_DEU | 2807 DESC_HDR_MODE0_DEU_3DES, 2808 }, 2809 { .type = CRYPTO_ALG_TYPE_SKCIPHER, 2810 .alg.skcipher = { 2811 .base.cra_name = "cbc(des3_ede)", 2812 .base.cra_driver_name = "cbc-3des-talitos", 2813 .base.cra_blocksize = DES3_EDE_BLOCK_SIZE, 2814 .base.cra_flags = CRYPTO_ALG_ASYNC | 2815 CRYPTO_ALG_ALLOCATES_MEMORY, 2816 .min_keysize = DES3_EDE_KEY_SIZE, 2817 .max_keysize = DES3_EDE_KEY_SIZE, 2818 .ivsize = DES3_EDE_BLOCK_SIZE, 2819 .setkey = skcipher_des3_setkey, 2820 }, 2821 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 2822 DESC_HDR_SEL0_DEU | 2823 DESC_HDR_MODE0_DEU_CBC | 2824 DESC_HDR_MODE0_DEU_3DES, 2825 }, 2826 /* AHASH algorithms. */ 2827 { .type = CRYPTO_ALG_TYPE_AHASH, 2828 .alg.hash = { 2829 .halg.digestsize = MD5_DIGEST_SIZE, 2830 .halg.statesize = sizeof(struct talitos_export_state), 2831 .halg.base = { 2832 .cra_name = "md5", 2833 .cra_driver_name = "md5-talitos", 2834 .cra_blocksize = MD5_HMAC_BLOCK_SIZE, 2835 .cra_flags = CRYPTO_ALG_ASYNC | 2836 CRYPTO_ALG_ALLOCATES_MEMORY, 2837 } 2838 }, 2839 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 2840 DESC_HDR_SEL0_MDEUA | 2841 DESC_HDR_MODE0_MDEU_MD5, 2842 }, 2843 { .type = CRYPTO_ALG_TYPE_AHASH, 2844 .alg.hash = { 2845 .halg.digestsize = SHA1_DIGEST_SIZE, 2846 .halg.statesize = sizeof(struct talitos_export_state), 2847 .halg.base = { 2848 .cra_name = "sha1", 2849 .cra_driver_name = "sha1-talitos", 2850 .cra_blocksize = SHA1_BLOCK_SIZE, 2851 .cra_flags = CRYPTO_ALG_ASYNC | 2852 CRYPTO_ALG_ALLOCATES_MEMORY, 2853 } 2854 }, 2855 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 2856 DESC_HDR_SEL0_MDEUA | 2857 DESC_HDR_MODE0_MDEU_SHA1, 2858 }, 2859 { .type = CRYPTO_ALG_TYPE_AHASH, 2860 .alg.hash = { 2861 .halg.digestsize = SHA224_DIGEST_SIZE, 2862 .halg.statesize = sizeof(struct talitos_export_state), 2863 .halg.base = { 2864 .cra_name = "sha224", 2865 .cra_driver_name = "sha224-talitos", 2866 .cra_blocksize = SHA224_BLOCK_SIZE, 2867 .cra_flags = CRYPTO_ALG_ASYNC | 2868 CRYPTO_ALG_ALLOCATES_MEMORY, 2869 } 2870 }, 2871 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 2872 DESC_HDR_SEL0_MDEUA | 2873 DESC_HDR_MODE0_MDEU_SHA224, 2874 }, 2875 { .type = CRYPTO_ALG_TYPE_AHASH, 2876 .alg.hash = { 2877 .halg.digestsize = SHA256_DIGEST_SIZE, 2878 .halg.statesize = sizeof(struct talitos_export_state), 2879 .halg.base = { 2880 .cra_name = "sha256", 2881 .cra_driver_name = "sha256-talitos", 2882 .cra_blocksize = SHA256_BLOCK_SIZE, 2883 .cra_flags = CRYPTO_ALG_ASYNC | 2884 CRYPTO_ALG_ALLOCATES_MEMORY, 2885 } 2886 }, 2887 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 2888 DESC_HDR_SEL0_MDEUA | 2889 DESC_HDR_MODE0_MDEU_SHA256, 2890 }, 2891 { .type = CRYPTO_ALG_TYPE_AHASH, 2892 .alg.hash = { 2893 .halg.digestsize = SHA384_DIGEST_SIZE, 2894 .halg.statesize = sizeof(struct talitos_export_state), 2895 .halg.base = { 2896 .cra_name = "sha384", 2897 .cra_driver_name = "sha384-talitos", 2898 .cra_blocksize = SHA384_BLOCK_SIZE, 2899 .cra_flags = CRYPTO_ALG_ASYNC | 2900 CRYPTO_ALG_ALLOCATES_MEMORY, 2901 } 2902 }, 2903 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 2904 DESC_HDR_SEL0_MDEUB | 2905 DESC_HDR_MODE0_MDEUB_SHA384, 2906 }, 2907 { .type = CRYPTO_ALG_TYPE_AHASH, 2908 .alg.hash = { 2909 .halg.digestsize = SHA512_DIGEST_SIZE, 2910 .halg.statesize = sizeof(struct talitos_export_state), 2911 .halg.base = { 2912 .cra_name = "sha512", 2913 .cra_driver_name = "sha512-talitos", 2914 .cra_blocksize = SHA512_BLOCK_SIZE, 2915 .cra_flags = CRYPTO_ALG_ASYNC | 2916 CRYPTO_ALG_ALLOCATES_MEMORY, 2917 } 2918 }, 2919 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 2920 DESC_HDR_SEL0_MDEUB | 2921 DESC_HDR_MODE0_MDEUB_SHA512, 2922 }, 2923 { .type = CRYPTO_ALG_TYPE_AHASH, 2924 .alg.hash = { 2925 .halg.digestsize = MD5_DIGEST_SIZE, 2926 .halg.statesize = sizeof(struct talitos_export_state), 2927 .halg.base = { 2928 .cra_name = "hmac(md5)", 2929 .cra_driver_name = "hmac-md5-talitos", 2930 .cra_blocksize = MD5_HMAC_BLOCK_SIZE, 2931 .cra_flags = CRYPTO_ALG_ASYNC | 2932 CRYPTO_ALG_ALLOCATES_MEMORY, 2933 } 2934 }, 2935 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 2936 DESC_HDR_SEL0_MDEUA | 2937 DESC_HDR_MODE0_MDEU_MD5, 2938 }, 2939 { .type = CRYPTO_ALG_TYPE_AHASH, 2940 .alg.hash = { 2941 .halg.digestsize = SHA1_DIGEST_SIZE, 2942 .halg.statesize = sizeof(struct talitos_export_state), 2943 .halg.base = { 2944 .cra_name = "hmac(sha1)", 2945 .cra_driver_name = "hmac-sha1-talitos", 2946 .cra_blocksize = SHA1_BLOCK_SIZE, 2947 .cra_flags = CRYPTO_ALG_ASYNC | 2948 CRYPTO_ALG_ALLOCATES_MEMORY, 2949 } 2950 }, 2951 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 2952 DESC_HDR_SEL0_MDEUA | 2953 DESC_HDR_MODE0_MDEU_SHA1, 2954 }, 2955 { .type = CRYPTO_ALG_TYPE_AHASH, 2956 .alg.hash = { 2957 .halg.digestsize = SHA224_DIGEST_SIZE, 2958 .halg.statesize = sizeof(struct talitos_export_state), 2959 .halg.base = { 2960 .cra_name = "hmac(sha224)", 2961 .cra_driver_name = "hmac-sha224-talitos", 2962 .cra_blocksize = SHA224_BLOCK_SIZE, 2963 .cra_flags = CRYPTO_ALG_ASYNC | 2964 CRYPTO_ALG_ALLOCATES_MEMORY, 2965 } 2966 }, 2967 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 2968 DESC_HDR_SEL0_MDEUA | 2969 DESC_HDR_MODE0_MDEU_SHA224, 2970 }, 2971 { .type = CRYPTO_ALG_TYPE_AHASH, 2972 .alg.hash = { 2973 .halg.digestsize = SHA256_DIGEST_SIZE, 2974 .halg.statesize = sizeof(struct talitos_export_state), 2975 .halg.base = { 2976 .cra_name = "hmac(sha256)", 2977 .cra_driver_name = "hmac-sha256-talitos", 2978 .cra_blocksize = SHA256_BLOCK_SIZE, 2979 .cra_flags = CRYPTO_ALG_ASYNC | 2980 CRYPTO_ALG_ALLOCATES_MEMORY, 2981 } 2982 }, 2983 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 2984 DESC_HDR_SEL0_MDEUA | 2985 DESC_HDR_MODE0_MDEU_SHA256, 2986 }, 2987 { .type = CRYPTO_ALG_TYPE_AHASH, 2988 .alg.hash = { 2989 .halg.digestsize = SHA384_DIGEST_SIZE, 2990 .halg.statesize = sizeof(struct talitos_export_state), 2991 .halg.base = { 2992 .cra_name = "hmac(sha384)", 2993 .cra_driver_name = "hmac-sha384-talitos", 2994 .cra_blocksize = SHA384_BLOCK_SIZE, 2995 .cra_flags = CRYPTO_ALG_ASYNC | 2996 CRYPTO_ALG_ALLOCATES_MEMORY, 2997 } 2998 }, 2999 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 3000 DESC_HDR_SEL0_MDEUB | 3001 DESC_HDR_MODE0_MDEUB_SHA384, 3002 }, 3003 { .type = CRYPTO_ALG_TYPE_AHASH, 3004 .alg.hash = { 3005 .halg.digestsize = SHA512_DIGEST_SIZE, 3006 .halg.statesize = sizeof(struct talitos_export_state), 3007 .halg.base = { 3008 .cra_name = "hmac(sha512)", 3009 .cra_driver_name = "hmac-sha512-talitos", 3010 .cra_blocksize = SHA512_BLOCK_SIZE, 3011 .cra_flags = CRYPTO_ALG_ASYNC | 3012 CRYPTO_ALG_ALLOCATES_MEMORY, 3013 } 3014 }, 3015 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 3016 DESC_HDR_SEL0_MDEUB | 3017 DESC_HDR_MODE0_MDEUB_SHA512, 3018 } 3019 }; 3020 3021 struct talitos_crypto_alg { 3022 struct list_head entry; 3023 struct device *dev; 3024 struct talitos_alg_template algt; 3025 }; 3026 3027 static int talitos_init_common(struct talitos_ctx *ctx, 3028 struct talitos_crypto_alg *talitos_alg) 3029 { 3030 struct talitos_private *priv; 3031 3032 /* update context with ptr to dev */ 3033 ctx->dev = talitos_alg->dev; 3034 3035 /* assign SEC channel to tfm in round-robin fashion */ 3036 priv = dev_get_drvdata(ctx->dev); 3037 ctx->ch = atomic_inc_return(&priv->last_chan) & 3038 (priv->num_channels - 1); 3039 3040 /* copy descriptor header template value */ 3041 ctx->desc_hdr_template = talitos_alg->algt.desc_hdr_template; 3042 3043 /* select done notification */ 3044 ctx->desc_hdr_template |= DESC_HDR_DONE_NOTIFY; 3045 3046 return 0; 3047 } 3048 3049 static int talitos_cra_init_aead(struct crypto_aead *tfm) 3050 { 3051 struct aead_alg *alg = crypto_aead_alg(tfm); 3052 struct talitos_crypto_alg *talitos_alg; 3053 struct talitos_ctx *ctx = crypto_aead_ctx(tfm); 3054 3055 talitos_alg = container_of(alg, struct talitos_crypto_alg, 3056 algt.alg.aead); 3057 3058 return talitos_init_common(ctx, talitos_alg); 3059 } 3060 3061 static int talitos_cra_init_skcipher(struct crypto_skcipher *tfm) 3062 { 3063 struct skcipher_alg *alg = crypto_skcipher_alg(tfm); 3064 struct talitos_crypto_alg *talitos_alg; 3065 struct talitos_ctx *ctx = crypto_skcipher_ctx(tfm); 3066 3067 talitos_alg = container_of(alg, struct talitos_crypto_alg, 3068 algt.alg.skcipher); 3069 3070 return talitos_init_common(ctx, talitos_alg); 3071 } 3072 3073 static int talitos_cra_init_ahash(struct crypto_tfm *tfm) 3074 { 3075 struct crypto_alg *alg = tfm->__crt_alg; 3076 struct talitos_crypto_alg *talitos_alg; 3077 struct talitos_ctx *ctx = crypto_tfm_ctx(tfm); 3078 3079 talitos_alg = container_of(__crypto_ahash_alg(alg), 3080 struct talitos_crypto_alg, 3081 algt.alg.hash); 3082 3083 ctx->keylen = 0; 3084 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), 3085 sizeof(struct talitos_ahash_req_ctx)); 3086 3087 return talitos_init_common(ctx, talitos_alg); 3088 } 3089 3090 static void talitos_cra_exit(struct crypto_tfm *tfm) 3091 { 3092 struct talitos_ctx *ctx = crypto_tfm_ctx(tfm); 3093 struct device *dev = ctx->dev; 3094 3095 if (ctx->keylen) 3096 dma_unmap_single(dev, ctx->dma_key, ctx->keylen, DMA_TO_DEVICE); 3097 } 3098 3099 /* 3100 * given the alg's descriptor header template, determine whether descriptor 3101 * type and primary/secondary execution units required match the hw 3102 * capabilities description provided in the device tree node. 3103 */ 3104 static int hw_supports(struct device *dev, __be32 desc_hdr_template) 3105 { 3106 struct talitos_private *priv = dev_get_drvdata(dev); 3107 int ret; 3108 3109 ret = (1 << DESC_TYPE(desc_hdr_template) & priv->desc_types) && 3110 (1 << PRIMARY_EU(desc_hdr_template) & priv->exec_units); 3111 3112 if (SECONDARY_EU(desc_hdr_template)) 3113 ret = ret && (1 << SECONDARY_EU(desc_hdr_template) 3114 & priv->exec_units); 3115 3116 return ret; 3117 } 3118 3119 static int talitos_remove(struct platform_device *ofdev) 3120 { 3121 struct device *dev = &ofdev->dev; 3122 struct talitos_private *priv = dev_get_drvdata(dev); 3123 struct talitos_crypto_alg *t_alg, *n; 3124 int i; 3125 3126 list_for_each_entry_safe(t_alg, n, &priv->alg_list, entry) { 3127 switch (t_alg->algt.type) { 3128 case CRYPTO_ALG_TYPE_SKCIPHER: 3129 crypto_unregister_skcipher(&t_alg->algt.alg.skcipher); 3130 break; 3131 case CRYPTO_ALG_TYPE_AEAD: 3132 crypto_unregister_aead(&t_alg->algt.alg.aead); 3133 break; 3134 case CRYPTO_ALG_TYPE_AHASH: 3135 crypto_unregister_ahash(&t_alg->algt.alg.hash); 3136 break; 3137 } 3138 list_del(&t_alg->entry); 3139 } 3140 3141 if (hw_supports(dev, DESC_HDR_SEL0_RNG)) 3142 talitos_unregister_rng(dev); 3143 3144 for (i = 0; i < 2; i++) 3145 if (priv->irq[i]) { 3146 free_irq(priv->irq[i], dev); 3147 irq_dispose_mapping(priv->irq[i]); 3148 } 3149 3150 tasklet_kill(&priv->done_task[0]); 3151 if (priv->irq[1]) 3152 tasklet_kill(&priv->done_task[1]); 3153 3154 return 0; 3155 } 3156 3157 static struct talitos_crypto_alg *talitos_alg_alloc(struct device *dev, 3158 struct talitos_alg_template 3159 *template) 3160 { 3161 struct talitos_private *priv = dev_get_drvdata(dev); 3162 struct talitos_crypto_alg *t_alg; 3163 struct crypto_alg *alg; 3164 3165 t_alg = devm_kzalloc(dev, sizeof(struct talitos_crypto_alg), 3166 GFP_KERNEL); 3167 if (!t_alg) 3168 return ERR_PTR(-ENOMEM); 3169 3170 t_alg->algt = *template; 3171 3172 switch (t_alg->algt.type) { 3173 case CRYPTO_ALG_TYPE_SKCIPHER: 3174 alg = &t_alg->algt.alg.skcipher.base; 3175 alg->cra_exit = talitos_cra_exit; 3176 t_alg->algt.alg.skcipher.init = talitos_cra_init_skcipher; 3177 t_alg->algt.alg.skcipher.setkey = 3178 t_alg->algt.alg.skcipher.setkey ?: skcipher_setkey; 3179 t_alg->algt.alg.skcipher.encrypt = skcipher_encrypt; 3180 t_alg->algt.alg.skcipher.decrypt = skcipher_decrypt; 3181 break; 3182 case CRYPTO_ALG_TYPE_AEAD: 3183 alg = &t_alg->algt.alg.aead.base; 3184 alg->cra_exit = talitos_cra_exit; 3185 t_alg->algt.alg.aead.init = talitos_cra_init_aead; 3186 t_alg->algt.alg.aead.setkey = t_alg->algt.alg.aead.setkey ?: 3187 aead_setkey; 3188 t_alg->algt.alg.aead.encrypt = aead_encrypt; 3189 t_alg->algt.alg.aead.decrypt = aead_decrypt; 3190 if (!(priv->features & TALITOS_FTR_SHA224_HWINIT) && 3191 !strncmp(alg->cra_name, "authenc(hmac(sha224)", 20)) { 3192 devm_kfree(dev, t_alg); 3193 return ERR_PTR(-ENOTSUPP); 3194 } 3195 break; 3196 case CRYPTO_ALG_TYPE_AHASH: 3197 alg = &t_alg->algt.alg.hash.halg.base; 3198 alg->cra_init = talitos_cra_init_ahash; 3199 alg->cra_exit = talitos_cra_exit; 3200 t_alg->algt.alg.hash.init = ahash_init; 3201 t_alg->algt.alg.hash.update = ahash_update; 3202 t_alg->algt.alg.hash.final = ahash_final; 3203 t_alg->algt.alg.hash.finup = ahash_finup; 3204 t_alg->algt.alg.hash.digest = ahash_digest; 3205 if (!strncmp(alg->cra_name, "hmac", 4)) 3206 t_alg->algt.alg.hash.setkey = ahash_setkey; 3207 t_alg->algt.alg.hash.import = ahash_import; 3208 t_alg->algt.alg.hash.export = ahash_export; 3209 3210 if (!(priv->features & TALITOS_FTR_HMAC_OK) && 3211 !strncmp(alg->cra_name, "hmac", 4)) { 3212 devm_kfree(dev, t_alg); 3213 return ERR_PTR(-ENOTSUPP); 3214 } 3215 if (!(priv->features & TALITOS_FTR_SHA224_HWINIT) && 3216 (!strcmp(alg->cra_name, "sha224") || 3217 !strcmp(alg->cra_name, "hmac(sha224)"))) { 3218 t_alg->algt.alg.hash.init = ahash_init_sha224_swinit; 3219 t_alg->algt.desc_hdr_template = 3220 DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 3221 DESC_HDR_SEL0_MDEUA | 3222 DESC_HDR_MODE0_MDEU_SHA256; 3223 } 3224 break; 3225 default: 3226 dev_err(dev, "unknown algorithm type %d\n", t_alg->algt.type); 3227 devm_kfree(dev, t_alg); 3228 return ERR_PTR(-EINVAL); 3229 } 3230 3231 alg->cra_module = THIS_MODULE; 3232 if (t_alg->algt.priority) 3233 alg->cra_priority = t_alg->algt.priority; 3234 else 3235 alg->cra_priority = TALITOS_CRA_PRIORITY; 3236 if (has_ftr_sec1(priv)) 3237 alg->cra_alignmask = 3; 3238 else 3239 alg->cra_alignmask = 0; 3240 alg->cra_ctxsize = sizeof(struct talitos_ctx); 3241 alg->cra_flags |= CRYPTO_ALG_KERN_DRIVER_ONLY; 3242 3243 t_alg->dev = dev; 3244 3245 return t_alg; 3246 } 3247 3248 static int talitos_probe_irq(struct platform_device *ofdev) 3249 { 3250 struct device *dev = &ofdev->dev; 3251 struct device_node *np = ofdev->dev.of_node; 3252 struct talitos_private *priv = dev_get_drvdata(dev); 3253 int err; 3254 bool is_sec1 = has_ftr_sec1(priv); 3255 3256 priv->irq[0] = irq_of_parse_and_map(np, 0); 3257 if (!priv->irq[0]) { 3258 dev_err(dev, "failed to map irq\n"); 3259 return -EINVAL; 3260 } 3261 if (is_sec1) { 3262 err = request_irq(priv->irq[0], talitos1_interrupt_4ch, 0, 3263 dev_driver_string(dev), dev); 3264 goto primary_out; 3265 } 3266 3267 priv->irq[1] = irq_of_parse_and_map(np, 1); 3268 3269 /* get the primary irq line */ 3270 if (!priv->irq[1]) { 3271 err = request_irq(priv->irq[0], talitos2_interrupt_4ch, 0, 3272 dev_driver_string(dev), dev); 3273 goto primary_out; 3274 } 3275 3276 err = request_irq(priv->irq[0], talitos2_interrupt_ch0_2, 0, 3277 dev_driver_string(dev), dev); 3278 if (err) 3279 goto primary_out; 3280 3281 /* get the secondary irq line */ 3282 err = request_irq(priv->irq[1], talitos2_interrupt_ch1_3, 0, 3283 dev_driver_string(dev), dev); 3284 if (err) { 3285 dev_err(dev, "failed to request secondary irq\n"); 3286 irq_dispose_mapping(priv->irq[1]); 3287 priv->irq[1] = 0; 3288 } 3289 3290 return err; 3291 3292 primary_out: 3293 if (err) { 3294 dev_err(dev, "failed to request primary irq\n"); 3295 irq_dispose_mapping(priv->irq[0]); 3296 priv->irq[0] = 0; 3297 } 3298 3299 return err; 3300 } 3301 3302 static int talitos_probe(struct platform_device *ofdev) 3303 { 3304 struct device *dev = &ofdev->dev; 3305 struct device_node *np = ofdev->dev.of_node; 3306 struct talitos_private *priv; 3307 int i, err; 3308 int stride; 3309 struct resource *res; 3310 3311 priv = devm_kzalloc(dev, sizeof(struct talitos_private), GFP_KERNEL); 3312 if (!priv) 3313 return -ENOMEM; 3314 3315 INIT_LIST_HEAD(&priv->alg_list); 3316 3317 dev_set_drvdata(dev, priv); 3318 3319 priv->ofdev = ofdev; 3320 3321 spin_lock_init(&priv->reg_lock); 3322 3323 res = platform_get_resource(ofdev, IORESOURCE_MEM, 0); 3324 if (!res) 3325 return -ENXIO; 3326 priv->reg = devm_ioremap(dev, res->start, resource_size(res)); 3327 if (!priv->reg) { 3328 dev_err(dev, "failed to of_iomap\n"); 3329 err = -ENOMEM; 3330 goto err_out; 3331 } 3332 3333 /* get SEC version capabilities from device tree */ 3334 of_property_read_u32(np, "fsl,num-channels", &priv->num_channels); 3335 of_property_read_u32(np, "fsl,channel-fifo-len", &priv->chfifo_len); 3336 of_property_read_u32(np, "fsl,exec-units-mask", &priv->exec_units); 3337 of_property_read_u32(np, "fsl,descriptor-types-mask", 3338 &priv->desc_types); 3339 3340 if (!is_power_of_2(priv->num_channels) || !priv->chfifo_len || 3341 !priv->exec_units || !priv->desc_types) { 3342 dev_err(dev, "invalid property data in device tree node\n"); 3343 err = -EINVAL; 3344 goto err_out; 3345 } 3346 3347 if (of_device_is_compatible(np, "fsl,sec3.0")) 3348 priv->features |= TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT; 3349 3350 if (of_device_is_compatible(np, "fsl,sec2.1")) 3351 priv->features |= TALITOS_FTR_HW_AUTH_CHECK | 3352 TALITOS_FTR_SHA224_HWINIT | 3353 TALITOS_FTR_HMAC_OK; 3354 3355 if (of_device_is_compatible(np, "fsl,sec1.0")) 3356 priv->features |= TALITOS_FTR_SEC1; 3357 3358 if (of_device_is_compatible(np, "fsl,sec1.2")) { 3359 priv->reg_deu = priv->reg + TALITOS12_DEU; 3360 priv->reg_aesu = priv->reg + TALITOS12_AESU; 3361 priv->reg_mdeu = priv->reg + TALITOS12_MDEU; 3362 stride = TALITOS1_CH_STRIDE; 3363 } else if (of_device_is_compatible(np, "fsl,sec1.0")) { 3364 priv->reg_deu = priv->reg + TALITOS10_DEU; 3365 priv->reg_aesu = priv->reg + TALITOS10_AESU; 3366 priv->reg_mdeu = priv->reg + TALITOS10_MDEU; 3367 priv->reg_afeu = priv->reg + TALITOS10_AFEU; 3368 priv->reg_rngu = priv->reg + TALITOS10_RNGU; 3369 priv->reg_pkeu = priv->reg + TALITOS10_PKEU; 3370 stride = TALITOS1_CH_STRIDE; 3371 } else { 3372 priv->reg_deu = priv->reg + TALITOS2_DEU; 3373 priv->reg_aesu = priv->reg + TALITOS2_AESU; 3374 priv->reg_mdeu = priv->reg + TALITOS2_MDEU; 3375 priv->reg_afeu = priv->reg + TALITOS2_AFEU; 3376 priv->reg_rngu = priv->reg + TALITOS2_RNGU; 3377 priv->reg_pkeu = priv->reg + TALITOS2_PKEU; 3378 priv->reg_keu = priv->reg + TALITOS2_KEU; 3379 priv->reg_crcu = priv->reg + TALITOS2_CRCU; 3380 stride = TALITOS2_CH_STRIDE; 3381 } 3382 3383 err = talitos_probe_irq(ofdev); 3384 if (err) 3385 goto err_out; 3386 3387 if (has_ftr_sec1(priv)) { 3388 if (priv->num_channels == 1) 3389 tasklet_init(&priv->done_task[0], talitos1_done_ch0, 3390 (unsigned long)dev); 3391 else 3392 tasklet_init(&priv->done_task[0], talitos1_done_4ch, 3393 (unsigned long)dev); 3394 } else { 3395 if (priv->irq[1]) { 3396 tasklet_init(&priv->done_task[0], talitos2_done_ch0_2, 3397 (unsigned long)dev); 3398 tasklet_init(&priv->done_task[1], talitos2_done_ch1_3, 3399 (unsigned long)dev); 3400 } else if (priv->num_channels == 1) { 3401 tasklet_init(&priv->done_task[0], talitos2_done_ch0, 3402 (unsigned long)dev); 3403 } else { 3404 tasklet_init(&priv->done_task[0], talitos2_done_4ch, 3405 (unsigned long)dev); 3406 } 3407 } 3408 3409 priv->chan = devm_kcalloc(dev, 3410 priv->num_channels, 3411 sizeof(struct talitos_channel), 3412 GFP_KERNEL); 3413 if (!priv->chan) { 3414 dev_err(dev, "failed to allocate channel management space\n"); 3415 err = -ENOMEM; 3416 goto err_out; 3417 } 3418 3419 priv->fifo_len = roundup_pow_of_two(priv->chfifo_len); 3420 3421 for (i = 0; i < priv->num_channels; i++) { 3422 priv->chan[i].reg = priv->reg + stride * (i + 1); 3423 if (!priv->irq[1] || !(i & 1)) 3424 priv->chan[i].reg += TALITOS_CH_BASE_OFFSET; 3425 3426 spin_lock_init(&priv->chan[i].head_lock); 3427 spin_lock_init(&priv->chan[i].tail_lock); 3428 3429 priv->chan[i].fifo = devm_kcalloc(dev, 3430 priv->fifo_len, 3431 sizeof(struct talitos_request), 3432 GFP_KERNEL); 3433 if (!priv->chan[i].fifo) { 3434 dev_err(dev, "failed to allocate request fifo %d\n", i); 3435 err = -ENOMEM; 3436 goto err_out; 3437 } 3438 3439 atomic_set(&priv->chan[i].submit_count, 3440 -(priv->chfifo_len - 1)); 3441 } 3442 3443 dma_set_mask(dev, DMA_BIT_MASK(36)); 3444 3445 /* reset and initialize the h/w */ 3446 err = init_device(dev); 3447 if (err) { 3448 dev_err(dev, "failed to initialize device\n"); 3449 goto err_out; 3450 } 3451 3452 /* register the RNG, if available */ 3453 if (hw_supports(dev, DESC_HDR_SEL0_RNG)) { 3454 err = talitos_register_rng(dev); 3455 if (err) { 3456 dev_err(dev, "failed to register hwrng: %d\n", err); 3457 goto err_out; 3458 } else 3459 dev_info(dev, "hwrng\n"); 3460 } 3461 3462 /* register crypto algorithms the device supports */ 3463 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) { 3464 if (hw_supports(dev, driver_algs[i].desc_hdr_template)) { 3465 struct talitos_crypto_alg *t_alg; 3466 struct crypto_alg *alg = NULL; 3467 3468 t_alg = talitos_alg_alloc(dev, &driver_algs[i]); 3469 if (IS_ERR(t_alg)) { 3470 err = PTR_ERR(t_alg); 3471 if (err == -ENOTSUPP) 3472 continue; 3473 goto err_out; 3474 } 3475 3476 switch (t_alg->algt.type) { 3477 case CRYPTO_ALG_TYPE_SKCIPHER: 3478 err = crypto_register_skcipher( 3479 &t_alg->algt.alg.skcipher); 3480 alg = &t_alg->algt.alg.skcipher.base; 3481 break; 3482 3483 case CRYPTO_ALG_TYPE_AEAD: 3484 err = crypto_register_aead( 3485 &t_alg->algt.alg.aead); 3486 alg = &t_alg->algt.alg.aead.base; 3487 break; 3488 3489 case CRYPTO_ALG_TYPE_AHASH: 3490 err = crypto_register_ahash( 3491 &t_alg->algt.alg.hash); 3492 alg = &t_alg->algt.alg.hash.halg.base; 3493 break; 3494 } 3495 if (err) { 3496 dev_err(dev, "%s alg registration failed\n", 3497 alg->cra_driver_name); 3498 devm_kfree(dev, t_alg); 3499 } else 3500 list_add_tail(&t_alg->entry, &priv->alg_list); 3501 } 3502 } 3503 if (!list_empty(&priv->alg_list)) 3504 dev_info(dev, "%s algorithms registered in /proc/crypto\n", 3505 (char *)of_get_property(np, "compatible", NULL)); 3506 3507 return 0; 3508 3509 err_out: 3510 talitos_remove(ofdev); 3511 3512 return err; 3513 } 3514 3515 static const struct of_device_id talitos_match[] = { 3516 #ifdef CONFIG_CRYPTO_DEV_TALITOS1 3517 { 3518 .compatible = "fsl,sec1.0", 3519 }, 3520 #endif 3521 #ifdef CONFIG_CRYPTO_DEV_TALITOS2 3522 { 3523 .compatible = "fsl,sec2.0", 3524 }, 3525 #endif 3526 {}, 3527 }; 3528 MODULE_DEVICE_TABLE(of, talitos_match); 3529 3530 static struct platform_driver talitos_driver = { 3531 .driver = { 3532 .name = "talitos", 3533 .of_match_table = talitos_match, 3534 }, 3535 .probe = talitos_probe, 3536 .remove = talitos_remove, 3537 }; 3538 3539 module_platform_driver(talitos_driver); 3540 3541 MODULE_LICENSE("GPL"); 3542 MODULE_AUTHOR("Kim Phillips <kim.phillips@freescale.com>"); 3543 MODULE_DESCRIPTION("Freescale integrated security engine (SEC) driver"); 3544