1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * talitos - Freescale Integrated Security Engine (SEC) device driver 4 * 5 * Copyright (c) 2008-2011 Freescale Semiconductor, Inc. 6 * 7 * Scatterlist Crypto API glue code copied from files with the following: 8 * Copyright (c) 2006-2007 Herbert Xu <herbert@gondor.apana.org.au> 9 * 10 * Crypto algorithm registration code copied from hifn driver: 11 * 2007+ Copyright (c) Evgeniy Polyakov <johnpol@2ka.mipt.ru> 12 * All rights reserved. 13 */ 14 15 #include <linux/kernel.h> 16 #include <linux/module.h> 17 #include <linux/mod_devicetable.h> 18 #include <linux/device.h> 19 #include <linux/interrupt.h> 20 #include <linux/crypto.h> 21 #include <linux/hw_random.h> 22 #include <linux/of_address.h> 23 #include <linux/of_irq.h> 24 #include <linux/of_platform.h> 25 #include <linux/dma-mapping.h> 26 #include <linux/io.h> 27 #include <linux/spinlock.h> 28 #include <linux/rtnetlink.h> 29 #include <linux/slab.h> 30 31 #include <crypto/algapi.h> 32 #include <crypto/aes.h> 33 #include <crypto/internal/des.h> 34 #include <crypto/sha.h> 35 #include <crypto/md5.h> 36 #include <crypto/internal/aead.h> 37 #include <crypto/authenc.h> 38 #include <crypto/internal/skcipher.h> 39 #include <crypto/hash.h> 40 #include <crypto/internal/hash.h> 41 #include <crypto/scatterwalk.h> 42 43 #include "talitos.h" 44 45 static void to_talitos_ptr(struct talitos_ptr *ptr, dma_addr_t dma_addr, 46 unsigned int len, bool is_sec1) 47 { 48 ptr->ptr = cpu_to_be32(lower_32_bits(dma_addr)); 49 if (is_sec1) { 50 ptr->len1 = cpu_to_be16(len); 51 } else { 52 ptr->len = cpu_to_be16(len); 53 ptr->eptr = upper_32_bits(dma_addr); 54 } 55 } 56 57 static void copy_talitos_ptr(struct talitos_ptr *dst_ptr, 58 struct talitos_ptr *src_ptr, bool is_sec1) 59 { 60 dst_ptr->ptr = src_ptr->ptr; 61 if (is_sec1) { 62 dst_ptr->len1 = src_ptr->len1; 63 } else { 64 dst_ptr->len = src_ptr->len; 65 dst_ptr->eptr = src_ptr->eptr; 66 } 67 } 68 69 static unsigned short from_talitos_ptr_len(struct talitos_ptr *ptr, 70 bool is_sec1) 71 { 72 if (is_sec1) 73 return be16_to_cpu(ptr->len1); 74 else 75 return be16_to_cpu(ptr->len); 76 } 77 78 static void to_talitos_ptr_ext_set(struct talitos_ptr *ptr, u8 val, 79 bool is_sec1) 80 { 81 if (!is_sec1) 82 ptr->j_extent = val; 83 } 84 85 static void to_talitos_ptr_ext_or(struct talitos_ptr *ptr, u8 val, bool is_sec1) 86 { 87 if (!is_sec1) 88 ptr->j_extent |= val; 89 } 90 91 /* 92 * map virtual single (contiguous) pointer to h/w descriptor pointer 93 */ 94 static void __map_single_talitos_ptr(struct device *dev, 95 struct talitos_ptr *ptr, 96 unsigned int len, void *data, 97 enum dma_data_direction dir, 98 unsigned long attrs) 99 { 100 dma_addr_t dma_addr = dma_map_single_attrs(dev, data, len, dir, attrs); 101 struct talitos_private *priv = dev_get_drvdata(dev); 102 bool is_sec1 = has_ftr_sec1(priv); 103 104 to_talitos_ptr(ptr, dma_addr, len, is_sec1); 105 } 106 107 static void map_single_talitos_ptr(struct device *dev, 108 struct talitos_ptr *ptr, 109 unsigned int len, void *data, 110 enum dma_data_direction dir) 111 { 112 __map_single_talitos_ptr(dev, ptr, len, data, dir, 0); 113 } 114 115 static void map_single_talitos_ptr_nosync(struct device *dev, 116 struct talitos_ptr *ptr, 117 unsigned int len, void *data, 118 enum dma_data_direction dir) 119 { 120 __map_single_talitos_ptr(dev, ptr, len, data, dir, 121 DMA_ATTR_SKIP_CPU_SYNC); 122 } 123 124 /* 125 * unmap bus single (contiguous) h/w descriptor pointer 126 */ 127 static void unmap_single_talitos_ptr(struct device *dev, 128 struct talitos_ptr *ptr, 129 enum dma_data_direction dir) 130 { 131 struct talitos_private *priv = dev_get_drvdata(dev); 132 bool is_sec1 = has_ftr_sec1(priv); 133 134 dma_unmap_single(dev, be32_to_cpu(ptr->ptr), 135 from_talitos_ptr_len(ptr, is_sec1), dir); 136 } 137 138 static int reset_channel(struct device *dev, int ch) 139 { 140 struct talitos_private *priv = dev_get_drvdata(dev); 141 unsigned int timeout = TALITOS_TIMEOUT; 142 bool is_sec1 = has_ftr_sec1(priv); 143 144 if (is_sec1) { 145 setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO, 146 TALITOS1_CCCR_LO_RESET); 147 148 while ((in_be32(priv->chan[ch].reg + TALITOS_CCCR_LO) & 149 TALITOS1_CCCR_LO_RESET) && --timeout) 150 cpu_relax(); 151 } else { 152 setbits32(priv->chan[ch].reg + TALITOS_CCCR, 153 TALITOS2_CCCR_RESET); 154 155 while ((in_be32(priv->chan[ch].reg + TALITOS_CCCR) & 156 TALITOS2_CCCR_RESET) && --timeout) 157 cpu_relax(); 158 } 159 160 if (timeout == 0) { 161 dev_err(dev, "failed to reset channel %d\n", ch); 162 return -EIO; 163 } 164 165 /* set 36-bit addressing, done writeback enable and done IRQ enable */ 166 setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO, TALITOS_CCCR_LO_EAE | 167 TALITOS_CCCR_LO_CDWE | TALITOS_CCCR_LO_CDIE); 168 /* enable chaining descriptors */ 169 if (is_sec1) 170 setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO, 171 TALITOS_CCCR_LO_NE); 172 173 /* and ICCR writeback, if available */ 174 if (priv->features & TALITOS_FTR_HW_AUTH_CHECK) 175 setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO, 176 TALITOS_CCCR_LO_IWSE); 177 178 return 0; 179 } 180 181 static int reset_device(struct device *dev) 182 { 183 struct talitos_private *priv = dev_get_drvdata(dev); 184 unsigned int timeout = TALITOS_TIMEOUT; 185 bool is_sec1 = has_ftr_sec1(priv); 186 u32 mcr = is_sec1 ? TALITOS1_MCR_SWR : TALITOS2_MCR_SWR; 187 188 setbits32(priv->reg + TALITOS_MCR, mcr); 189 190 while ((in_be32(priv->reg + TALITOS_MCR) & mcr) 191 && --timeout) 192 cpu_relax(); 193 194 if (priv->irq[1]) { 195 mcr = TALITOS_MCR_RCA1 | TALITOS_MCR_RCA3; 196 setbits32(priv->reg + TALITOS_MCR, mcr); 197 } 198 199 if (timeout == 0) { 200 dev_err(dev, "failed to reset device\n"); 201 return -EIO; 202 } 203 204 return 0; 205 } 206 207 /* 208 * Reset and initialize the device 209 */ 210 static int init_device(struct device *dev) 211 { 212 struct talitos_private *priv = dev_get_drvdata(dev); 213 int ch, err; 214 bool is_sec1 = has_ftr_sec1(priv); 215 216 /* 217 * Master reset 218 * errata documentation: warning: certain SEC interrupts 219 * are not fully cleared by writing the MCR:SWR bit, 220 * set bit twice to completely reset 221 */ 222 err = reset_device(dev); 223 if (err) 224 return err; 225 226 err = reset_device(dev); 227 if (err) 228 return err; 229 230 /* reset channels */ 231 for (ch = 0; ch < priv->num_channels; ch++) { 232 err = reset_channel(dev, ch); 233 if (err) 234 return err; 235 } 236 237 /* enable channel done and error interrupts */ 238 if (is_sec1) { 239 clrbits32(priv->reg + TALITOS_IMR, TALITOS1_IMR_INIT); 240 clrbits32(priv->reg + TALITOS_IMR_LO, TALITOS1_IMR_LO_INIT); 241 /* disable parity error check in DEU (erroneous? test vect.) */ 242 setbits32(priv->reg_deu + TALITOS_EUICR, TALITOS1_DEUICR_KPE); 243 } else { 244 setbits32(priv->reg + TALITOS_IMR, TALITOS2_IMR_INIT); 245 setbits32(priv->reg + TALITOS_IMR_LO, TALITOS2_IMR_LO_INIT); 246 } 247 248 /* disable integrity check error interrupts (use writeback instead) */ 249 if (priv->features & TALITOS_FTR_HW_AUTH_CHECK) 250 setbits32(priv->reg_mdeu + TALITOS_EUICR_LO, 251 TALITOS_MDEUICR_LO_ICE); 252 253 return 0; 254 } 255 256 /** 257 * talitos_submit - submits a descriptor to the device for processing 258 * @dev: the SEC device to be used 259 * @ch: the SEC device channel to be used 260 * @desc: the descriptor to be processed by the device 261 * @callback: whom to call when processing is complete 262 * @context: a handle for use by caller (optional) 263 * 264 * desc must contain valid dma-mapped (bus physical) address pointers. 265 * callback must check err and feedback in descriptor header 266 * for device processing status. 267 */ 268 static int talitos_submit(struct device *dev, int ch, struct talitos_desc *desc, 269 void (*callback)(struct device *dev, 270 struct talitos_desc *desc, 271 void *context, int error), 272 void *context) 273 { 274 struct talitos_private *priv = dev_get_drvdata(dev); 275 struct talitos_request *request; 276 unsigned long flags; 277 int head; 278 bool is_sec1 = has_ftr_sec1(priv); 279 280 spin_lock_irqsave(&priv->chan[ch].head_lock, flags); 281 282 if (!atomic_inc_not_zero(&priv->chan[ch].submit_count)) { 283 /* h/w fifo is full */ 284 spin_unlock_irqrestore(&priv->chan[ch].head_lock, flags); 285 return -EAGAIN; 286 } 287 288 head = priv->chan[ch].head; 289 request = &priv->chan[ch].fifo[head]; 290 291 /* map descriptor and save caller data */ 292 if (is_sec1) { 293 desc->hdr1 = desc->hdr; 294 request->dma_desc = dma_map_single(dev, &desc->hdr1, 295 TALITOS_DESC_SIZE, 296 DMA_BIDIRECTIONAL); 297 } else { 298 request->dma_desc = dma_map_single(dev, desc, 299 TALITOS_DESC_SIZE, 300 DMA_BIDIRECTIONAL); 301 } 302 request->callback = callback; 303 request->context = context; 304 305 /* increment fifo head */ 306 priv->chan[ch].head = (priv->chan[ch].head + 1) & (priv->fifo_len - 1); 307 308 smp_wmb(); 309 request->desc = desc; 310 311 /* GO! */ 312 wmb(); 313 out_be32(priv->chan[ch].reg + TALITOS_FF, 314 upper_32_bits(request->dma_desc)); 315 out_be32(priv->chan[ch].reg + TALITOS_FF_LO, 316 lower_32_bits(request->dma_desc)); 317 318 spin_unlock_irqrestore(&priv->chan[ch].head_lock, flags); 319 320 return -EINPROGRESS; 321 } 322 323 static __be32 get_request_hdr(struct talitos_request *request, bool is_sec1) 324 { 325 struct talitos_edesc *edesc; 326 327 if (!is_sec1) 328 return request->desc->hdr; 329 330 if (!request->desc->next_desc) 331 return request->desc->hdr1; 332 333 edesc = container_of(request->desc, struct talitos_edesc, desc); 334 335 return ((struct talitos_desc *)(edesc->buf + edesc->dma_len))->hdr1; 336 } 337 338 /* 339 * process what was done, notify callback of error if not 340 */ 341 static void flush_channel(struct device *dev, int ch, int error, int reset_ch) 342 { 343 struct talitos_private *priv = dev_get_drvdata(dev); 344 struct talitos_request *request, saved_req; 345 unsigned long flags; 346 int tail, status; 347 bool is_sec1 = has_ftr_sec1(priv); 348 349 spin_lock_irqsave(&priv->chan[ch].tail_lock, flags); 350 351 tail = priv->chan[ch].tail; 352 while (priv->chan[ch].fifo[tail].desc) { 353 __be32 hdr; 354 355 request = &priv->chan[ch].fifo[tail]; 356 357 /* descriptors with their done bits set don't get the error */ 358 rmb(); 359 hdr = get_request_hdr(request, is_sec1); 360 361 if ((hdr & DESC_HDR_DONE) == DESC_HDR_DONE) 362 status = 0; 363 else 364 if (!error) 365 break; 366 else 367 status = error; 368 369 dma_unmap_single(dev, request->dma_desc, 370 TALITOS_DESC_SIZE, 371 DMA_BIDIRECTIONAL); 372 373 /* copy entries so we can call callback outside lock */ 374 saved_req.desc = request->desc; 375 saved_req.callback = request->callback; 376 saved_req.context = request->context; 377 378 /* release request entry in fifo */ 379 smp_wmb(); 380 request->desc = NULL; 381 382 /* increment fifo tail */ 383 priv->chan[ch].tail = (tail + 1) & (priv->fifo_len - 1); 384 385 spin_unlock_irqrestore(&priv->chan[ch].tail_lock, flags); 386 387 atomic_dec(&priv->chan[ch].submit_count); 388 389 saved_req.callback(dev, saved_req.desc, saved_req.context, 390 status); 391 /* channel may resume processing in single desc error case */ 392 if (error && !reset_ch && status == error) 393 return; 394 spin_lock_irqsave(&priv->chan[ch].tail_lock, flags); 395 tail = priv->chan[ch].tail; 396 } 397 398 spin_unlock_irqrestore(&priv->chan[ch].tail_lock, flags); 399 } 400 401 /* 402 * process completed requests for channels that have done status 403 */ 404 #define DEF_TALITOS1_DONE(name, ch_done_mask) \ 405 static void talitos1_done_##name(unsigned long data) \ 406 { \ 407 struct device *dev = (struct device *)data; \ 408 struct talitos_private *priv = dev_get_drvdata(dev); \ 409 unsigned long flags; \ 410 \ 411 if (ch_done_mask & 0x10000000) \ 412 flush_channel(dev, 0, 0, 0); \ 413 if (ch_done_mask & 0x40000000) \ 414 flush_channel(dev, 1, 0, 0); \ 415 if (ch_done_mask & 0x00010000) \ 416 flush_channel(dev, 2, 0, 0); \ 417 if (ch_done_mask & 0x00040000) \ 418 flush_channel(dev, 3, 0, 0); \ 419 \ 420 /* At this point, all completed channels have been processed */ \ 421 /* Unmask done interrupts for channels completed later on. */ \ 422 spin_lock_irqsave(&priv->reg_lock, flags); \ 423 clrbits32(priv->reg + TALITOS_IMR, ch_done_mask); \ 424 clrbits32(priv->reg + TALITOS_IMR_LO, TALITOS1_IMR_LO_INIT); \ 425 spin_unlock_irqrestore(&priv->reg_lock, flags); \ 426 } 427 428 DEF_TALITOS1_DONE(4ch, TALITOS1_ISR_4CHDONE) 429 DEF_TALITOS1_DONE(ch0, TALITOS1_ISR_CH_0_DONE) 430 431 #define DEF_TALITOS2_DONE(name, ch_done_mask) \ 432 static void talitos2_done_##name(unsigned long data) \ 433 { \ 434 struct device *dev = (struct device *)data; \ 435 struct talitos_private *priv = dev_get_drvdata(dev); \ 436 unsigned long flags; \ 437 \ 438 if (ch_done_mask & 1) \ 439 flush_channel(dev, 0, 0, 0); \ 440 if (ch_done_mask & (1 << 2)) \ 441 flush_channel(dev, 1, 0, 0); \ 442 if (ch_done_mask & (1 << 4)) \ 443 flush_channel(dev, 2, 0, 0); \ 444 if (ch_done_mask & (1 << 6)) \ 445 flush_channel(dev, 3, 0, 0); \ 446 \ 447 /* At this point, all completed channels have been processed */ \ 448 /* Unmask done interrupts for channels completed later on. */ \ 449 spin_lock_irqsave(&priv->reg_lock, flags); \ 450 setbits32(priv->reg + TALITOS_IMR, ch_done_mask); \ 451 setbits32(priv->reg + TALITOS_IMR_LO, TALITOS2_IMR_LO_INIT); \ 452 spin_unlock_irqrestore(&priv->reg_lock, flags); \ 453 } 454 455 DEF_TALITOS2_DONE(4ch, TALITOS2_ISR_4CHDONE) 456 DEF_TALITOS2_DONE(ch0, TALITOS2_ISR_CH_0_DONE) 457 DEF_TALITOS2_DONE(ch0_2, TALITOS2_ISR_CH_0_2_DONE) 458 DEF_TALITOS2_DONE(ch1_3, TALITOS2_ISR_CH_1_3_DONE) 459 460 /* 461 * locate current (offending) descriptor 462 */ 463 static u32 current_desc_hdr(struct device *dev, int ch) 464 { 465 struct talitos_private *priv = dev_get_drvdata(dev); 466 int tail, iter; 467 dma_addr_t cur_desc; 468 469 cur_desc = ((u64)in_be32(priv->chan[ch].reg + TALITOS_CDPR)) << 32; 470 cur_desc |= in_be32(priv->chan[ch].reg + TALITOS_CDPR_LO); 471 472 if (!cur_desc) { 473 dev_err(dev, "CDPR is NULL, giving up search for offending descriptor\n"); 474 return 0; 475 } 476 477 tail = priv->chan[ch].tail; 478 479 iter = tail; 480 while (priv->chan[ch].fifo[iter].dma_desc != cur_desc && 481 priv->chan[ch].fifo[iter].desc->next_desc != cur_desc) { 482 iter = (iter + 1) & (priv->fifo_len - 1); 483 if (iter == tail) { 484 dev_err(dev, "couldn't locate current descriptor\n"); 485 return 0; 486 } 487 } 488 489 if (priv->chan[ch].fifo[iter].desc->next_desc == cur_desc) { 490 struct talitos_edesc *edesc; 491 492 edesc = container_of(priv->chan[ch].fifo[iter].desc, 493 struct talitos_edesc, desc); 494 return ((struct talitos_desc *) 495 (edesc->buf + edesc->dma_len))->hdr; 496 } 497 498 return priv->chan[ch].fifo[iter].desc->hdr; 499 } 500 501 /* 502 * user diagnostics; report root cause of error based on execution unit status 503 */ 504 static void report_eu_error(struct device *dev, int ch, u32 desc_hdr) 505 { 506 struct talitos_private *priv = dev_get_drvdata(dev); 507 int i; 508 509 if (!desc_hdr) 510 desc_hdr = in_be32(priv->chan[ch].reg + TALITOS_DESCBUF); 511 512 switch (desc_hdr & DESC_HDR_SEL0_MASK) { 513 case DESC_HDR_SEL0_AFEU: 514 dev_err(dev, "AFEUISR 0x%08x_%08x\n", 515 in_be32(priv->reg_afeu + TALITOS_EUISR), 516 in_be32(priv->reg_afeu + TALITOS_EUISR_LO)); 517 break; 518 case DESC_HDR_SEL0_DEU: 519 dev_err(dev, "DEUISR 0x%08x_%08x\n", 520 in_be32(priv->reg_deu + TALITOS_EUISR), 521 in_be32(priv->reg_deu + TALITOS_EUISR_LO)); 522 break; 523 case DESC_HDR_SEL0_MDEUA: 524 case DESC_HDR_SEL0_MDEUB: 525 dev_err(dev, "MDEUISR 0x%08x_%08x\n", 526 in_be32(priv->reg_mdeu + TALITOS_EUISR), 527 in_be32(priv->reg_mdeu + TALITOS_EUISR_LO)); 528 break; 529 case DESC_HDR_SEL0_RNG: 530 dev_err(dev, "RNGUISR 0x%08x_%08x\n", 531 in_be32(priv->reg_rngu + TALITOS_ISR), 532 in_be32(priv->reg_rngu + TALITOS_ISR_LO)); 533 break; 534 case DESC_HDR_SEL0_PKEU: 535 dev_err(dev, "PKEUISR 0x%08x_%08x\n", 536 in_be32(priv->reg_pkeu + TALITOS_EUISR), 537 in_be32(priv->reg_pkeu + TALITOS_EUISR_LO)); 538 break; 539 case DESC_HDR_SEL0_AESU: 540 dev_err(dev, "AESUISR 0x%08x_%08x\n", 541 in_be32(priv->reg_aesu + TALITOS_EUISR), 542 in_be32(priv->reg_aesu + TALITOS_EUISR_LO)); 543 break; 544 case DESC_HDR_SEL0_CRCU: 545 dev_err(dev, "CRCUISR 0x%08x_%08x\n", 546 in_be32(priv->reg_crcu + TALITOS_EUISR), 547 in_be32(priv->reg_crcu + TALITOS_EUISR_LO)); 548 break; 549 case DESC_HDR_SEL0_KEU: 550 dev_err(dev, "KEUISR 0x%08x_%08x\n", 551 in_be32(priv->reg_pkeu + TALITOS_EUISR), 552 in_be32(priv->reg_pkeu + TALITOS_EUISR_LO)); 553 break; 554 } 555 556 switch (desc_hdr & DESC_HDR_SEL1_MASK) { 557 case DESC_HDR_SEL1_MDEUA: 558 case DESC_HDR_SEL1_MDEUB: 559 dev_err(dev, "MDEUISR 0x%08x_%08x\n", 560 in_be32(priv->reg_mdeu + TALITOS_EUISR), 561 in_be32(priv->reg_mdeu + TALITOS_EUISR_LO)); 562 break; 563 case DESC_HDR_SEL1_CRCU: 564 dev_err(dev, "CRCUISR 0x%08x_%08x\n", 565 in_be32(priv->reg_crcu + TALITOS_EUISR), 566 in_be32(priv->reg_crcu + TALITOS_EUISR_LO)); 567 break; 568 } 569 570 for (i = 0; i < 8; i++) 571 dev_err(dev, "DESCBUF 0x%08x_%08x\n", 572 in_be32(priv->chan[ch].reg + TALITOS_DESCBUF + 8*i), 573 in_be32(priv->chan[ch].reg + TALITOS_DESCBUF_LO + 8*i)); 574 } 575 576 /* 577 * recover from error interrupts 578 */ 579 static void talitos_error(struct device *dev, u32 isr, u32 isr_lo) 580 { 581 struct talitos_private *priv = dev_get_drvdata(dev); 582 unsigned int timeout = TALITOS_TIMEOUT; 583 int ch, error, reset_dev = 0; 584 u32 v_lo; 585 bool is_sec1 = has_ftr_sec1(priv); 586 int reset_ch = is_sec1 ? 1 : 0; /* only SEC2 supports continuation */ 587 588 for (ch = 0; ch < priv->num_channels; ch++) { 589 /* skip channels without errors */ 590 if (is_sec1) { 591 /* bits 29, 31, 17, 19 */ 592 if (!(isr & (1 << (29 + (ch & 1) * 2 - (ch & 2) * 6)))) 593 continue; 594 } else { 595 if (!(isr & (1 << (ch * 2 + 1)))) 596 continue; 597 } 598 599 error = -EINVAL; 600 601 v_lo = in_be32(priv->chan[ch].reg + TALITOS_CCPSR_LO); 602 603 if (v_lo & TALITOS_CCPSR_LO_DOF) { 604 dev_err(dev, "double fetch fifo overflow error\n"); 605 error = -EAGAIN; 606 reset_ch = 1; 607 } 608 if (v_lo & TALITOS_CCPSR_LO_SOF) { 609 /* h/w dropped descriptor */ 610 dev_err(dev, "single fetch fifo overflow error\n"); 611 error = -EAGAIN; 612 } 613 if (v_lo & TALITOS_CCPSR_LO_MDTE) 614 dev_err(dev, "master data transfer error\n"); 615 if (v_lo & TALITOS_CCPSR_LO_SGDLZ) 616 dev_err(dev, is_sec1 ? "pointer not complete error\n" 617 : "s/g data length zero error\n"); 618 if (v_lo & TALITOS_CCPSR_LO_FPZ) 619 dev_err(dev, is_sec1 ? "parity error\n" 620 : "fetch pointer zero error\n"); 621 if (v_lo & TALITOS_CCPSR_LO_IDH) 622 dev_err(dev, "illegal descriptor header error\n"); 623 if (v_lo & TALITOS_CCPSR_LO_IEU) 624 dev_err(dev, is_sec1 ? "static assignment error\n" 625 : "invalid exec unit error\n"); 626 if (v_lo & TALITOS_CCPSR_LO_EU) 627 report_eu_error(dev, ch, current_desc_hdr(dev, ch)); 628 if (!is_sec1) { 629 if (v_lo & TALITOS_CCPSR_LO_GB) 630 dev_err(dev, "gather boundary error\n"); 631 if (v_lo & TALITOS_CCPSR_LO_GRL) 632 dev_err(dev, "gather return/length error\n"); 633 if (v_lo & TALITOS_CCPSR_LO_SB) 634 dev_err(dev, "scatter boundary error\n"); 635 if (v_lo & TALITOS_CCPSR_LO_SRL) 636 dev_err(dev, "scatter return/length error\n"); 637 } 638 639 flush_channel(dev, ch, error, reset_ch); 640 641 if (reset_ch) { 642 reset_channel(dev, ch); 643 } else { 644 setbits32(priv->chan[ch].reg + TALITOS_CCCR, 645 TALITOS2_CCCR_CONT); 646 setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO, 0); 647 while ((in_be32(priv->chan[ch].reg + TALITOS_CCCR) & 648 TALITOS2_CCCR_CONT) && --timeout) 649 cpu_relax(); 650 if (timeout == 0) { 651 dev_err(dev, "failed to restart channel %d\n", 652 ch); 653 reset_dev = 1; 654 } 655 } 656 } 657 if (reset_dev || (is_sec1 && isr & ~TALITOS1_ISR_4CHERR) || 658 (!is_sec1 && isr & ~TALITOS2_ISR_4CHERR) || isr_lo) { 659 if (is_sec1 && (isr_lo & TALITOS1_ISR_TEA_ERR)) 660 dev_err(dev, "TEA error: ISR 0x%08x_%08x\n", 661 isr, isr_lo); 662 else 663 dev_err(dev, "done overflow, internal time out, or " 664 "rngu error: ISR 0x%08x_%08x\n", isr, isr_lo); 665 666 /* purge request queues */ 667 for (ch = 0; ch < priv->num_channels; ch++) 668 flush_channel(dev, ch, -EIO, 1); 669 670 /* reset and reinitialize the device */ 671 init_device(dev); 672 } 673 } 674 675 #define DEF_TALITOS1_INTERRUPT(name, ch_done_mask, ch_err_mask, tlet) \ 676 static irqreturn_t talitos1_interrupt_##name(int irq, void *data) \ 677 { \ 678 struct device *dev = data; \ 679 struct talitos_private *priv = dev_get_drvdata(dev); \ 680 u32 isr, isr_lo; \ 681 unsigned long flags; \ 682 \ 683 spin_lock_irqsave(&priv->reg_lock, flags); \ 684 isr = in_be32(priv->reg + TALITOS_ISR); \ 685 isr_lo = in_be32(priv->reg + TALITOS_ISR_LO); \ 686 /* Acknowledge interrupt */ \ 687 out_be32(priv->reg + TALITOS_ICR, isr & (ch_done_mask | ch_err_mask)); \ 688 out_be32(priv->reg + TALITOS_ICR_LO, isr_lo); \ 689 \ 690 if (unlikely(isr & ch_err_mask || isr_lo & TALITOS1_IMR_LO_INIT)) { \ 691 spin_unlock_irqrestore(&priv->reg_lock, flags); \ 692 talitos_error(dev, isr & ch_err_mask, isr_lo); \ 693 } \ 694 else { \ 695 if (likely(isr & ch_done_mask)) { \ 696 /* mask further done interrupts. */ \ 697 setbits32(priv->reg + TALITOS_IMR, ch_done_mask); \ 698 /* done_task will unmask done interrupts at exit */ \ 699 tasklet_schedule(&priv->done_task[tlet]); \ 700 } \ 701 spin_unlock_irqrestore(&priv->reg_lock, flags); \ 702 } \ 703 \ 704 return (isr & (ch_done_mask | ch_err_mask) || isr_lo) ? IRQ_HANDLED : \ 705 IRQ_NONE; \ 706 } 707 708 DEF_TALITOS1_INTERRUPT(4ch, TALITOS1_ISR_4CHDONE, TALITOS1_ISR_4CHERR, 0) 709 710 #define DEF_TALITOS2_INTERRUPT(name, ch_done_mask, ch_err_mask, tlet) \ 711 static irqreturn_t talitos2_interrupt_##name(int irq, void *data) \ 712 { \ 713 struct device *dev = data; \ 714 struct talitos_private *priv = dev_get_drvdata(dev); \ 715 u32 isr, isr_lo; \ 716 unsigned long flags; \ 717 \ 718 spin_lock_irqsave(&priv->reg_lock, flags); \ 719 isr = in_be32(priv->reg + TALITOS_ISR); \ 720 isr_lo = in_be32(priv->reg + TALITOS_ISR_LO); \ 721 /* Acknowledge interrupt */ \ 722 out_be32(priv->reg + TALITOS_ICR, isr & (ch_done_mask | ch_err_mask)); \ 723 out_be32(priv->reg + TALITOS_ICR_LO, isr_lo); \ 724 \ 725 if (unlikely(isr & ch_err_mask || isr_lo)) { \ 726 spin_unlock_irqrestore(&priv->reg_lock, flags); \ 727 talitos_error(dev, isr & ch_err_mask, isr_lo); \ 728 } \ 729 else { \ 730 if (likely(isr & ch_done_mask)) { \ 731 /* mask further done interrupts. */ \ 732 clrbits32(priv->reg + TALITOS_IMR, ch_done_mask); \ 733 /* done_task will unmask done interrupts at exit */ \ 734 tasklet_schedule(&priv->done_task[tlet]); \ 735 } \ 736 spin_unlock_irqrestore(&priv->reg_lock, flags); \ 737 } \ 738 \ 739 return (isr & (ch_done_mask | ch_err_mask) || isr_lo) ? IRQ_HANDLED : \ 740 IRQ_NONE; \ 741 } 742 743 DEF_TALITOS2_INTERRUPT(4ch, TALITOS2_ISR_4CHDONE, TALITOS2_ISR_4CHERR, 0) 744 DEF_TALITOS2_INTERRUPT(ch0_2, TALITOS2_ISR_CH_0_2_DONE, TALITOS2_ISR_CH_0_2_ERR, 745 0) 746 DEF_TALITOS2_INTERRUPT(ch1_3, TALITOS2_ISR_CH_1_3_DONE, TALITOS2_ISR_CH_1_3_ERR, 747 1) 748 749 /* 750 * hwrng 751 */ 752 static int talitos_rng_data_present(struct hwrng *rng, int wait) 753 { 754 struct device *dev = (struct device *)rng->priv; 755 struct talitos_private *priv = dev_get_drvdata(dev); 756 u32 ofl; 757 int i; 758 759 for (i = 0; i < 20; i++) { 760 ofl = in_be32(priv->reg_rngu + TALITOS_EUSR_LO) & 761 TALITOS_RNGUSR_LO_OFL; 762 if (ofl || !wait) 763 break; 764 udelay(10); 765 } 766 767 return !!ofl; 768 } 769 770 static int talitos_rng_data_read(struct hwrng *rng, u32 *data) 771 { 772 struct device *dev = (struct device *)rng->priv; 773 struct talitos_private *priv = dev_get_drvdata(dev); 774 775 /* rng fifo requires 64-bit accesses */ 776 *data = in_be32(priv->reg_rngu + TALITOS_EU_FIFO); 777 *data = in_be32(priv->reg_rngu + TALITOS_EU_FIFO_LO); 778 779 return sizeof(u32); 780 } 781 782 static int talitos_rng_init(struct hwrng *rng) 783 { 784 struct device *dev = (struct device *)rng->priv; 785 struct talitos_private *priv = dev_get_drvdata(dev); 786 unsigned int timeout = TALITOS_TIMEOUT; 787 788 setbits32(priv->reg_rngu + TALITOS_EURCR_LO, TALITOS_RNGURCR_LO_SR); 789 while (!(in_be32(priv->reg_rngu + TALITOS_EUSR_LO) 790 & TALITOS_RNGUSR_LO_RD) 791 && --timeout) 792 cpu_relax(); 793 if (timeout == 0) { 794 dev_err(dev, "failed to reset rng hw\n"); 795 return -ENODEV; 796 } 797 798 /* start generating */ 799 setbits32(priv->reg_rngu + TALITOS_EUDSR_LO, 0); 800 801 return 0; 802 } 803 804 static int talitos_register_rng(struct device *dev) 805 { 806 struct talitos_private *priv = dev_get_drvdata(dev); 807 int err; 808 809 priv->rng.name = dev_driver_string(dev), 810 priv->rng.init = talitos_rng_init, 811 priv->rng.data_present = talitos_rng_data_present, 812 priv->rng.data_read = talitos_rng_data_read, 813 priv->rng.priv = (unsigned long)dev; 814 815 err = hwrng_register(&priv->rng); 816 if (!err) 817 priv->rng_registered = true; 818 819 return err; 820 } 821 822 static void talitos_unregister_rng(struct device *dev) 823 { 824 struct talitos_private *priv = dev_get_drvdata(dev); 825 826 if (!priv->rng_registered) 827 return; 828 829 hwrng_unregister(&priv->rng); 830 priv->rng_registered = false; 831 } 832 833 /* 834 * crypto alg 835 */ 836 #define TALITOS_CRA_PRIORITY 3000 837 /* 838 * Defines a priority for doing AEAD with descriptors type 839 * HMAC_SNOOP_NO_AFEA (HSNA) instead of type IPSEC_ESP 840 */ 841 #define TALITOS_CRA_PRIORITY_AEAD_HSNA (TALITOS_CRA_PRIORITY - 1) 842 #ifdef CONFIG_CRYPTO_DEV_TALITOS2 843 #define TALITOS_MAX_KEY_SIZE (AES_MAX_KEY_SIZE + SHA512_BLOCK_SIZE) 844 #else 845 #define TALITOS_MAX_KEY_SIZE (AES_MAX_KEY_SIZE + SHA256_BLOCK_SIZE) 846 #endif 847 #define TALITOS_MAX_IV_LENGTH 16 /* max of AES_BLOCK_SIZE, DES3_EDE_BLOCK_SIZE */ 848 849 struct talitos_ctx { 850 struct device *dev; 851 int ch; 852 __be32 desc_hdr_template; 853 u8 key[TALITOS_MAX_KEY_SIZE]; 854 u8 iv[TALITOS_MAX_IV_LENGTH]; 855 dma_addr_t dma_key; 856 unsigned int keylen; 857 unsigned int enckeylen; 858 unsigned int authkeylen; 859 }; 860 861 #define HASH_MAX_BLOCK_SIZE SHA512_BLOCK_SIZE 862 #define TALITOS_MDEU_MAX_CONTEXT_SIZE TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512 863 864 struct talitos_ahash_req_ctx { 865 u32 hw_context[TALITOS_MDEU_MAX_CONTEXT_SIZE / sizeof(u32)]; 866 unsigned int hw_context_size; 867 u8 buf[2][HASH_MAX_BLOCK_SIZE]; 868 int buf_idx; 869 unsigned int swinit; 870 unsigned int first; 871 unsigned int last; 872 unsigned int to_hash_later; 873 unsigned int nbuf; 874 struct scatterlist bufsl[2]; 875 struct scatterlist *psrc; 876 }; 877 878 struct talitos_export_state { 879 u32 hw_context[TALITOS_MDEU_MAX_CONTEXT_SIZE / sizeof(u32)]; 880 u8 buf[HASH_MAX_BLOCK_SIZE]; 881 unsigned int swinit; 882 unsigned int first; 883 unsigned int last; 884 unsigned int to_hash_later; 885 unsigned int nbuf; 886 }; 887 888 static int aead_setkey(struct crypto_aead *authenc, 889 const u8 *key, unsigned int keylen) 890 { 891 struct talitos_ctx *ctx = crypto_aead_ctx(authenc); 892 struct device *dev = ctx->dev; 893 struct crypto_authenc_keys keys; 894 895 if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) 896 goto badkey; 897 898 if (keys.authkeylen + keys.enckeylen > TALITOS_MAX_KEY_SIZE) 899 goto badkey; 900 901 if (ctx->keylen) 902 dma_unmap_single(dev, ctx->dma_key, ctx->keylen, DMA_TO_DEVICE); 903 904 memcpy(ctx->key, keys.authkey, keys.authkeylen); 905 memcpy(&ctx->key[keys.authkeylen], keys.enckey, keys.enckeylen); 906 907 ctx->keylen = keys.authkeylen + keys.enckeylen; 908 ctx->enckeylen = keys.enckeylen; 909 ctx->authkeylen = keys.authkeylen; 910 ctx->dma_key = dma_map_single(dev, ctx->key, ctx->keylen, 911 DMA_TO_DEVICE); 912 913 memzero_explicit(&keys, sizeof(keys)); 914 return 0; 915 916 badkey: 917 memzero_explicit(&keys, sizeof(keys)); 918 return -EINVAL; 919 } 920 921 static int aead_des3_setkey(struct crypto_aead *authenc, 922 const u8 *key, unsigned int keylen) 923 { 924 struct talitos_ctx *ctx = crypto_aead_ctx(authenc); 925 struct device *dev = ctx->dev; 926 struct crypto_authenc_keys keys; 927 int err; 928 929 err = crypto_authenc_extractkeys(&keys, key, keylen); 930 if (unlikely(err)) 931 goto out; 932 933 err = -EINVAL; 934 if (keys.authkeylen + keys.enckeylen > TALITOS_MAX_KEY_SIZE) 935 goto out; 936 937 err = verify_aead_des3_key(authenc, keys.enckey, keys.enckeylen); 938 if (err) 939 goto out; 940 941 if (ctx->keylen) 942 dma_unmap_single(dev, ctx->dma_key, ctx->keylen, DMA_TO_DEVICE); 943 944 memcpy(ctx->key, keys.authkey, keys.authkeylen); 945 memcpy(&ctx->key[keys.authkeylen], keys.enckey, keys.enckeylen); 946 947 ctx->keylen = keys.authkeylen + keys.enckeylen; 948 ctx->enckeylen = keys.enckeylen; 949 ctx->authkeylen = keys.authkeylen; 950 ctx->dma_key = dma_map_single(dev, ctx->key, ctx->keylen, 951 DMA_TO_DEVICE); 952 953 out: 954 memzero_explicit(&keys, sizeof(keys)); 955 return err; 956 } 957 958 static void talitos_sg_unmap(struct device *dev, 959 struct talitos_edesc *edesc, 960 struct scatterlist *src, 961 struct scatterlist *dst, 962 unsigned int len, unsigned int offset) 963 { 964 struct talitos_private *priv = dev_get_drvdata(dev); 965 bool is_sec1 = has_ftr_sec1(priv); 966 unsigned int src_nents = edesc->src_nents ? : 1; 967 unsigned int dst_nents = edesc->dst_nents ? : 1; 968 969 if (is_sec1 && dst && dst_nents > 1) { 970 dma_sync_single_for_device(dev, edesc->dma_link_tbl + offset, 971 len, DMA_FROM_DEVICE); 972 sg_pcopy_from_buffer(dst, dst_nents, edesc->buf + offset, len, 973 offset); 974 } 975 if (src != dst) { 976 if (src_nents == 1 || !is_sec1) 977 dma_unmap_sg(dev, src, src_nents, DMA_TO_DEVICE); 978 979 if (dst && (dst_nents == 1 || !is_sec1)) 980 dma_unmap_sg(dev, dst, dst_nents, DMA_FROM_DEVICE); 981 } else if (src_nents == 1 || !is_sec1) { 982 dma_unmap_sg(dev, src, src_nents, DMA_BIDIRECTIONAL); 983 } 984 } 985 986 static void ipsec_esp_unmap(struct device *dev, 987 struct talitos_edesc *edesc, 988 struct aead_request *areq, bool encrypt) 989 { 990 struct crypto_aead *aead = crypto_aead_reqtfm(areq); 991 struct talitos_ctx *ctx = crypto_aead_ctx(aead); 992 unsigned int ivsize = crypto_aead_ivsize(aead); 993 unsigned int authsize = crypto_aead_authsize(aead); 994 unsigned int cryptlen = areq->cryptlen - (encrypt ? 0 : authsize); 995 bool is_ipsec_esp = edesc->desc.hdr & DESC_HDR_TYPE_IPSEC_ESP; 996 struct talitos_ptr *civ_ptr = &edesc->desc.ptr[is_ipsec_esp ? 2 : 3]; 997 998 if (is_ipsec_esp) 999 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[6], 1000 DMA_FROM_DEVICE); 1001 unmap_single_talitos_ptr(dev, civ_ptr, DMA_TO_DEVICE); 1002 1003 talitos_sg_unmap(dev, edesc, areq->src, areq->dst, 1004 cryptlen + authsize, areq->assoclen); 1005 1006 if (edesc->dma_len) 1007 dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len, 1008 DMA_BIDIRECTIONAL); 1009 1010 if (!is_ipsec_esp) { 1011 unsigned int dst_nents = edesc->dst_nents ? : 1; 1012 1013 sg_pcopy_to_buffer(areq->dst, dst_nents, ctx->iv, ivsize, 1014 areq->assoclen + cryptlen - ivsize); 1015 } 1016 } 1017 1018 /* 1019 * ipsec_esp descriptor callbacks 1020 */ 1021 static void ipsec_esp_encrypt_done(struct device *dev, 1022 struct talitos_desc *desc, void *context, 1023 int err) 1024 { 1025 struct aead_request *areq = context; 1026 struct crypto_aead *authenc = crypto_aead_reqtfm(areq); 1027 unsigned int ivsize = crypto_aead_ivsize(authenc); 1028 struct talitos_edesc *edesc; 1029 1030 edesc = container_of(desc, struct talitos_edesc, desc); 1031 1032 ipsec_esp_unmap(dev, edesc, areq, true); 1033 1034 dma_unmap_single(dev, edesc->iv_dma, ivsize, DMA_TO_DEVICE); 1035 1036 kfree(edesc); 1037 1038 aead_request_complete(areq, err); 1039 } 1040 1041 static void ipsec_esp_decrypt_swauth_done(struct device *dev, 1042 struct talitos_desc *desc, 1043 void *context, int err) 1044 { 1045 struct aead_request *req = context; 1046 struct crypto_aead *authenc = crypto_aead_reqtfm(req); 1047 unsigned int authsize = crypto_aead_authsize(authenc); 1048 struct talitos_edesc *edesc; 1049 char *oicv, *icv; 1050 1051 edesc = container_of(desc, struct talitos_edesc, desc); 1052 1053 ipsec_esp_unmap(dev, edesc, req, false); 1054 1055 if (!err) { 1056 /* auth check */ 1057 oicv = edesc->buf + edesc->dma_len; 1058 icv = oicv - authsize; 1059 1060 err = crypto_memneq(oicv, icv, authsize) ? -EBADMSG : 0; 1061 } 1062 1063 kfree(edesc); 1064 1065 aead_request_complete(req, err); 1066 } 1067 1068 static void ipsec_esp_decrypt_hwauth_done(struct device *dev, 1069 struct talitos_desc *desc, 1070 void *context, int err) 1071 { 1072 struct aead_request *req = context; 1073 struct talitos_edesc *edesc; 1074 1075 edesc = container_of(desc, struct talitos_edesc, desc); 1076 1077 ipsec_esp_unmap(dev, edesc, req, false); 1078 1079 /* check ICV auth status */ 1080 if (!err && ((desc->hdr_lo & DESC_HDR_LO_ICCR1_MASK) != 1081 DESC_HDR_LO_ICCR1_PASS)) 1082 err = -EBADMSG; 1083 1084 kfree(edesc); 1085 1086 aead_request_complete(req, err); 1087 } 1088 1089 /* 1090 * convert scatterlist to SEC h/w link table format 1091 * stop at cryptlen bytes 1092 */ 1093 static int sg_to_link_tbl_offset(struct scatterlist *sg, int sg_count, 1094 unsigned int offset, int datalen, int elen, 1095 struct talitos_ptr *link_tbl_ptr) 1096 { 1097 int n_sg = elen ? sg_count + 1 : sg_count; 1098 int count = 0; 1099 int cryptlen = datalen + elen; 1100 1101 while (cryptlen && sg && n_sg--) { 1102 unsigned int len = sg_dma_len(sg); 1103 1104 if (offset >= len) { 1105 offset -= len; 1106 goto next; 1107 } 1108 1109 len -= offset; 1110 1111 if (len > cryptlen) 1112 len = cryptlen; 1113 1114 if (datalen > 0 && len > datalen) { 1115 to_talitos_ptr(link_tbl_ptr + count, 1116 sg_dma_address(sg) + offset, datalen, 0); 1117 to_talitos_ptr_ext_set(link_tbl_ptr + count, 0, 0); 1118 count++; 1119 len -= datalen; 1120 offset += datalen; 1121 } 1122 to_talitos_ptr(link_tbl_ptr + count, 1123 sg_dma_address(sg) + offset, len, 0); 1124 to_talitos_ptr_ext_set(link_tbl_ptr + count, 0, 0); 1125 count++; 1126 cryptlen -= len; 1127 datalen -= len; 1128 offset = 0; 1129 1130 next: 1131 sg = sg_next(sg); 1132 } 1133 1134 /* tag end of link table */ 1135 if (count > 0) 1136 to_talitos_ptr_ext_set(link_tbl_ptr + count - 1, 1137 DESC_PTR_LNKTBL_RET, 0); 1138 1139 return count; 1140 } 1141 1142 static int talitos_sg_map_ext(struct device *dev, struct scatterlist *src, 1143 unsigned int len, struct talitos_edesc *edesc, 1144 struct talitos_ptr *ptr, int sg_count, 1145 unsigned int offset, int tbl_off, int elen, 1146 bool force) 1147 { 1148 struct talitos_private *priv = dev_get_drvdata(dev); 1149 bool is_sec1 = has_ftr_sec1(priv); 1150 1151 if (!src) { 1152 to_talitos_ptr(ptr, 0, 0, is_sec1); 1153 return 1; 1154 } 1155 to_talitos_ptr_ext_set(ptr, elen, is_sec1); 1156 if (sg_count == 1 && !force) { 1157 to_talitos_ptr(ptr, sg_dma_address(src) + offset, len, is_sec1); 1158 return sg_count; 1159 } 1160 if (is_sec1) { 1161 to_talitos_ptr(ptr, edesc->dma_link_tbl + offset, len, is_sec1); 1162 return sg_count; 1163 } 1164 sg_count = sg_to_link_tbl_offset(src, sg_count, offset, len, elen, 1165 &edesc->link_tbl[tbl_off]); 1166 if (sg_count == 1 && !force) { 1167 /* Only one segment now, so no link tbl needed*/ 1168 copy_talitos_ptr(ptr, &edesc->link_tbl[tbl_off], is_sec1); 1169 return sg_count; 1170 } 1171 to_talitos_ptr(ptr, edesc->dma_link_tbl + 1172 tbl_off * sizeof(struct talitos_ptr), len, is_sec1); 1173 to_talitos_ptr_ext_or(ptr, DESC_PTR_LNKTBL_JUMP, is_sec1); 1174 1175 return sg_count; 1176 } 1177 1178 static int talitos_sg_map(struct device *dev, struct scatterlist *src, 1179 unsigned int len, struct talitos_edesc *edesc, 1180 struct talitos_ptr *ptr, int sg_count, 1181 unsigned int offset, int tbl_off) 1182 { 1183 return talitos_sg_map_ext(dev, src, len, edesc, ptr, sg_count, offset, 1184 tbl_off, 0, false); 1185 } 1186 1187 /* 1188 * fill in and submit ipsec_esp descriptor 1189 */ 1190 static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq, 1191 bool encrypt, 1192 void (*callback)(struct device *dev, 1193 struct talitos_desc *desc, 1194 void *context, int error)) 1195 { 1196 struct crypto_aead *aead = crypto_aead_reqtfm(areq); 1197 unsigned int authsize = crypto_aead_authsize(aead); 1198 struct talitos_ctx *ctx = crypto_aead_ctx(aead); 1199 struct device *dev = ctx->dev; 1200 struct talitos_desc *desc = &edesc->desc; 1201 unsigned int cryptlen = areq->cryptlen - (encrypt ? 0 : authsize); 1202 unsigned int ivsize = crypto_aead_ivsize(aead); 1203 int tbl_off = 0; 1204 int sg_count, ret; 1205 int elen = 0; 1206 bool sync_needed = false; 1207 struct talitos_private *priv = dev_get_drvdata(dev); 1208 bool is_sec1 = has_ftr_sec1(priv); 1209 bool is_ipsec_esp = desc->hdr & DESC_HDR_TYPE_IPSEC_ESP; 1210 struct talitos_ptr *civ_ptr = &desc->ptr[is_ipsec_esp ? 2 : 3]; 1211 struct talitos_ptr *ckey_ptr = &desc->ptr[is_ipsec_esp ? 3 : 2]; 1212 dma_addr_t dma_icv = edesc->dma_link_tbl + edesc->dma_len - authsize; 1213 1214 /* hmac key */ 1215 to_talitos_ptr(&desc->ptr[0], ctx->dma_key, ctx->authkeylen, is_sec1); 1216 1217 sg_count = edesc->src_nents ?: 1; 1218 if (is_sec1 && sg_count > 1) 1219 sg_copy_to_buffer(areq->src, sg_count, edesc->buf, 1220 areq->assoclen + cryptlen); 1221 else 1222 sg_count = dma_map_sg(dev, areq->src, sg_count, 1223 (areq->src == areq->dst) ? 1224 DMA_BIDIRECTIONAL : DMA_TO_DEVICE); 1225 1226 /* hmac data */ 1227 ret = talitos_sg_map(dev, areq->src, areq->assoclen, edesc, 1228 &desc->ptr[1], sg_count, 0, tbl_off); 1229 1230 if (ret > 1) { 1231 tbl_off += ret; 1232 sync_needed = true; 1233 } 1234 1235 /* cipher iv */ 1236 to_talitos_ptr(civ_ptr, edesc->iv_dma, ivsize, is_sec1); 1237 1238 /* cipher key */ 1239 to_talitos_ptr(ckey_ptr, ctx->dma_key + ctx->authkeylen, 1240 ctx->enckeylen, is_sec1); 1241 1242 /* 1243 * cipher in 1244 * map and adjust cipher len to aead request cryptlen. 1245 * extent is bytes of HMAC postpended to ciphertext, 1246 * typically 12 for ipsec 1247 */ 1248 if (is_ipsec_esp && (desc->hdr & DESC_HDR_MODE1_MDEU_CICV)) 1249 elen = authsize; 1250 1251 ret = talitos_sg_map_ext(dev, areq->src, cryptlen, edesc, &desc->ptr[4], 1252 sg_count, areq->assoclen, tbl_off, elen, 1253 false); 1254 1255 if (ret > 1) { 1256 tbl_off += ret; 1257 sync_needed = true; 1258 } 1259 1260 /* cipher out */ 1261 if (areq->src != areq->dst) { 1262 sg_count = edesc->dst_nents ? : 1; 1263 if (!is_sec1 || sg_count == 1) 1264 dma_map_sg(dev, areq->dst, sg_count, DMA_FROM_DEVICE); 1265 } 1266 1267 if (is_ipsec_esp && encrypt) 1268 elen = authsize; 1269 else 1270 elen = 0; 1271 ret = talitos_sg_map_ext(dev, areq->dst, cryptlen, edesc, &desc->ptr[5], 1272 sg_count, areq->assoclen, tbl_off, elen, 1273 is_ipsec_esp && !encrypt); 1274 tbl_off += ret; 1275 1276 if (!encrypt && is_ipsec_esp) { 1277 struct talitos_ptr *tbl_ptr = &edesc->link_tbl[tbl_off]; 1278 1279 /* Add an entry to the link table for ICV data */ 1280 to_talitos_ptr_ext_set(tbl_ptr - 1, 0, is_sec1); 1281 to_talitos_ptr_ext_set(tbl_ptr, DESC_PTR_LNKTBL_RET, is_sec1); 1282 1283 /* icv data follows link tables */ 1284 to_talitos_ptr(tbl_ptr, dma_icv, authsize, is_sec1); 1285 to_talitos_ptr_ext_or(&desc->ptr[5], authsize, is_sec1); 1286 sync_needed = true; 1287 } else if (!encrypt) { 1288 to_talitos_ptr(&desc->ptr[6], dma_icv, authsize, is_sec1); 1289 sync_needed = true; 1290 } else if (!is_ipsec_esp) { 1291 talitos_sg_map(dev, areq->dst, authsize, edesc, &desc->ptr[6], 1292 sg_count, areq->assoclen + cryptlen, tbl_off); 1293 } 1294 1295 /* iv out */ 1296 if (is_ipsec_esp) 1297 map_single_talitos_ptr(dev, &desc->ptr[6], ivsize, ctx->iv, 1298 DMA_FROM_DEVICE); 1299 1300 if (sync_needed) 1301 dma_sync_single_for_device(dev, edesc->dma_link_tbl, 1302 edesc->dma_len, 1303 DMA_BIDIRECTIONAL); 1304 1305 ret = talitos_submit(dev, ctx->ch, desc, callback, areq); 1306 if (ret != -EINPROGRESS) { 1307 ipsec_esp_unmap(dev, edesc, areq, encrypt); 1308 kfree(edesc); 1309 } 1310 return ret; 1311 } 1312 1313 /* 1314 * allocate and map the extended descriptor 1315 */ 1316 static struct talitos_edesc *talitos_edesc_alloc(struct device *dev, 1317 struct scatterlist *src, 1318 struct scatterlist *dst, 1319 u8 *iv, 1320 unsigned int assoclen, 1321 unsigned int cryptlen, 1322 unsigned int authsize, 1323 unsigned int ivsize, 1324 int icv_stashing, 1325 u32 cryptoflags, 1326 bool encrypt) 1327 { 1328 struct talitos_edesc *edesc; 1329 int src_nents, dst_nents, alloc_len, dma_len, src_len, dst_len; 1330 dma_addr_t iv_dma = 0; 1331 gfp_t flags = cryptoflags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : 1332 GFP_ATOMIC; 1333 struct talitos_private *priv = dev_get_drvdata(dev); 1334 bool is_sec1 = has_ftr_sec1(priv); 1335 int max_len = is_sec1 ? TALITOS1_MAX_DATA_LEN : TALITOS2_MAX_DATA_LEN; 1336 1337 if (cryptlen + authsize > max_len) { 1338 dev_err(dev, "length exceeds h/w max limit\n"); 1339 return ERR_PTR(-EINVAL); 1340 } 1341 1342 if (!dst || dst == src) { 1343 src_len = assoclen + cryptlen + authsize; 1344 src_nents = sg_nents_for_len(src, src_len); 1345 if (src_nents < 0) { 1346 dev_err(dev, "Invalid number of src SG.\n"); 1347 return ERR_PTR(-EINVAL); 1348 } 1349 src_nents = (src_nents == 1) ? 0 : src_nents; 1350 dst_nents = dst ? src_nents : 0; 1351 dst_len = 0; 1352 } else { /* dst && dst != src*/ 1353 src_len = assoclen + cryptlen + (encrypt ? 0 : authsize); 1354 src_nents = sg_nents_for_len(src, src_len); 1355 if (src_nents < 0) { 1356 dev_err(dev, "Invalid number of src SG.\n"); 1357 return ERR_PTR(-EINVAL); 1358 } 1359 src_nents = (src_nents == 1) ? 0 : src_nents; 1360 dst_len = assoclen + cryptlen + (encrypt ? authsize : 0); 1361 dst_nents = sg_nents_for_len(dst, dst_len); 1362 if (dst_nents < 0) { 1363 dev_err(dev, "Invalid number of dst SG.\n"); 1364 return ERR_PTR(-EINVAL); 1365 } 1366 dst_nents = (dst_nents == 1) ? 0 : dst_nents; 1367 } 1368 1369 /* 1370 * allocate space for base edesc plus the link tables, 1371 * allowing for two separate entries for AD and generated ICV (+ 2), 1372 * and space for two sets of ICVs (stashed and generated) 1373 */ 1374 alloc_len = sizeof(struct talitos_edesc); 1375 if (src_nents || dst_nents || !encrypt) { 1376 if (is_sec1) 1377 dma_len = (src_nents ? src_len : 0) + 1378 (dst_nents ? dst_len : 0) + authsize; 1379 else 1380 dma_len = (src_nents + dst_nents + 2) * 1381 sizeof(struct talitos_ptr) + authsize; 1382 alloc_len += dma_len; 1383 } else { 1384 dma_len = 0; 1385 } 1386 alloc_len += icv_stashing ? authsize : 0; 1387 1388 /* if its a ahash, add space for a second desc next to the first one */ 1389 if (is_sec1 && !dst) 1390 alloc_len += sizeof(struct talitos_desc); 1391 alloc_len += ivsize; 1392 1393 edesc = kmalloc(alloc_len, GFP_DMA | flags); 1394 if (!edesc) 1395 return ERR_PTR(-ENOMEM); 1396 if (ivsize) { 1397 iv = memcpy(((u8 *)edesc) + alloc_len - ivsize, iv, ivsize); 1398 iv_dma = dma_map_single(dev, iv, ivsize, DMA_TO_DEVICE); 1399 } 1400 memset(&edesc->desc, 0, sizeof(edesc->desc)); 1401 1402 edesc->src_nents = src_nents; 1403 edesc->dst_nents = dst_nents; 1404 edesc->iv_dma = iv_dma; 1405 edesc->dma_len = dma_len; 1406 if (dma_len) 1407 edesc->dma_link_tbl = dma_map_single(dev, &edesc->link_tbl[0], 1408 edesc->dma_len, 1409 DMA_BIDIRECTIONAL); 1410 1411 return edesc; 1412 } 1413 1414 static struct talitos_edesc *aead_edesc_alloc(struct aead_request *areq, u8 *iv, 1415 int icv_stashing, bool encrypt) 1416 { 1417 struct crypto_aead *authenc = crypto_aead_reqtfm(areq); 1418 unsigned int authsize = crypto_aead_authsize(authenc); 1419 struct talitos_ctx *ctx = crypto_aead_ctx(authenc); 1420 unsigned int ivsize = crypto_aead_ivsize(authenc); 1421 unsigned int cryptlen = areq->cryptlen - (encrypt ? 0 : authsize); 1422 1423 return talitos_edesc_alloc(ctx->dev, areq->src, areq->dst, 1424 iv, areq->assoclen, cryptlen, 1425 authsize, ivsize, icv_stashing, 1426 areq->base.flags, encrypt); 1427 } 1428 1429 static int aead_encrypt(struct aead_request *req) 1430 { 1431 struct crypto_aead *authenc = crypto_aead_reqtfm(req); 1432 struct talitos_ctx *ctx = crypto_aead_ctx(authenc); 1433 struct talitos_edesc *edesc; 1434 1435 /* allocate extended descriptor */ 1436 edesc = aead_edesc_alloc(req, req->iv, 0, true); 1437 if (IS_ERR(edesc)) 1438 return PTR_ERR(edesc); 1439 1440 /* set encrypt */ 1441 edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT; 1442 1443 return ipsec_esp(edesc, req, true, ipsec_esp_encrypt_done); 1444 } 1445 1446 static int aead_decrypt(struct aead_request *req) 1447 { 1448 struct crypto_aead *authenc = crypto_aead_reqtfm(req); 1449 unsigned int authsize = crypto_aead_authsize(authenc); 1450 struct talitos_ctx *ctx = crypto_aead_ctx(authenc); 1451 struct talitos_private *priv = dev_get_drvdata(ctx->dev); 1452 struct talitos_edesc *edesc; 1453 void *icvdata; 1454 1455 /* allocate extended descriptor */ 1456 edesc = aead_edesc_alloc(req, req->iv, 1, false); 1457 if (IS_ERR(edesc)) 1458 return PTR_ERR(edesc); 1459 1460 if ((edesc->desc.hdr & DESC_HDR_TYPE_IPSEC_ESP) && 1461 (priv->features & TALITOS_FTR_HW_AUTH_CHECK) && 1462 ((!edesc->src_nents && !edesc->dst_nents) || 1463 priv->features & TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT)) { 1464 1465 /* decrypt and check the ICV */ 1466 edesc->desc.hdr = ctx->desc_hdr_template | 1467 DESC_HDR_DIR_INBOUND | 1468 DESC_HDR_MODE1_MDEU_CICV; 1469 1470 /* reset integrity check result bits */ 1471 1472 return ipsec_esp(edesc, req, false, 1473 ipsec_esp_decrypt_hwauth_done); 1474 } 1475 1476 /* Have to check the ICV with software */ 1477 edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_DIR_INBOUND; 1478 1479 /* stash incoming ICV for later cmp with ICV generated by the h/w */ 1480 icvdata = edesc->buf + edesc->dma_len; 1481 1482 sg_pcopy_to_buffer(req->src, edesc->src_nents ? : 1, icvdata, authsize, 1483 req->assoclen + req->cryptlen - authsize); 1484 1485 return ipsec_esp(edesc, req, false, ipsec_esp_decrypt_swauth_done); 1486 } 1487 1488 static int skcipher_setkey(struct crypto_skcipher *cipher, 1489 const u8 *key, unsigned int keylen) 1490 { 1491 struct talitos_ctx *ctx = crypto_skcipher_ctx(cipher); 1492 struct device *dev = ctx->dev; 1493 1494 if (ctx->keylen) 1495 dma_unmap_single(dev, ctx->dma_key, ctx->keylen, DMA_TO_DEVICE); 1496 1497 memcpy(&ctx->key, key, keylen); 1498 ctx->keylen = keylen; 1499 1500 ctx->dma_key = dma_map_single(dev, ctx->key, keylen, DMA_TO_DEVICE); 1501 1502 return 0; 1503 } 1504 1505 static int skcipher_des_setkey(struct crypto_skcipher *cipher, 1506 const u8 *key, unsigned int keylen) 1507 { 1508 return verify_skcipher_des_key(cipher, key) ?: 1509 skcipher_setkey(cipher, key, keylen); 1510 } 1511 1512 static int skcipher_des3_setkey(struct crypto_skcipher *cipher, 1513 const u8 *key, unsigned int keylen) 1514 { 1515 return verify_skcipher_des3_key(cipher, key) ?: 1516 skcipher_setkey(cipher, key, keylen); 1517 } 1518 1519 static int skcipher_aes_setkey(struct crypto_skcipher *cipher, 1520 const u8 *key, unsigned int keylen) 1521 { 1522 if (keylen == AES_KEYSIZE_128 || keylen == AES_KEYSIZE_192 || 1523 keylen == AES_KEYSIZE_256) 1524 return skcipher_setkey(cipher, key, keylen); 1525 1526 return -EINVAL; 1527 } 1528 1529 static void common_nonsnoop_unmap(struct device *dev, 1530 struct talitos_edesc *edesc, 1531 struct skcipher_request *areq) 1532 { 1533 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[5], DMA_FROM_DEVICE); 1534 1535 talitos_sg_unmap(dev, edesc, areq->src, areq->dst, areq->cryptlen, 0); 1536 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[1], DMA_TO_DEVICE); 1537 1538 if (edesc->dma_len) 1539 dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len, 1540 DMA_BIDIRECTIONAL); 1541 } 1542 1543 static void skcipher_done(struct device *dev, 1544 struct talitos_desc *desc, void *context, 1545 int err) 1546 { 1547 struct skcipher_request *areq = context; 1548 struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(areq); 1549 struct talitos_ctx *ctx = crypto_skcipher_ctx(cipher); 1550 unsigned int ivsize = crypto_skcipher_ivsize(cipher); 1551 struct talitos_edesc *edesc; 1552 1553 edesc = container_of(desc, struct talitos_edesc, desc); 1554 1555 common_nonsnoop_unmap(dev, edesc, areq); 1556 memcpy(areq->iv, ctx->iv, ivsize); 1557 1558 kfree(edesc); 1559 1560 areq->base.complete(&areq->base, err); 1561 } 1562 1563 static int common_nonsnoop(struct talitos_edesc *edesc, 1564 struct skcipher_request *areq, 1565 void (*callback) (struct device *dev, 1566 struct talitos_desc *desc, 1567 void *context, int error)) 1568 { 1569 struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(areq); 1570 struct talitos_ctx *ctx = crypto_skcipher_ctx(cipher); 1571 struct device *dev = ctx->dev; 1572 struct talitos_desc *desc = &edesc->desc; 1573 unsigned int cryptlen = areq->cryptlen; 1574 unsigned int ivsize = crypto_skcipher_ivsize(cipher); 1575 int sg_count, ret; 1576 bool sync_needed = false; 1577 struct talitos_private *priv = dev_get_drvdata(dev); 1578 bool is_sec1 = has_ftr_sec1(priv); 1579 1580 /* first DWORD empty */ 1581 1582 /* cipher iv */ 1583 to_talitos_ptr(&desc->ptr[1], edesc->iv_dma, ivsize, is_sec1); 1584 1585 /* cipher key */ 1586 to_talitos_ptr(&desc->ptr[2], ctx->dma_key, ctx->keylen, is_sec1); 1587 1588 sg_count = edesc->src_nents ?: 1; 1589 if (is_sec1 && sg_count > 1) 1590 sg_copy_to_buffer(areq->src, sg_count, edesc->buf, 1591 cryptlen); 1592 else 1593 sg_count = dma_map_sg(dev, areq->src, sg_count, 1594 (areq->src == areq->dst) ? 1595 DMA_BIDIRECTIONAL : DMA_TO_DEVICE); 1596 /* 1597 * cipher in 1598 */ 1599 sg_count = talitos_sg_map(dev, areq->src, cryptlen, edesc, 1600 &desc->ptr[3], sg_count, 0, 0); 1601 if (sg_count > 1) 1602 sync_needed = true; 1603 1604 /* cipher out */ 1605 if (areq->src != areq->dst) { 1606 sg_count = edesc->dst_nents ? : 1; 1607 if (!is_sec1 || sg_count == 1) 1608 dma_map_sg(dev, areq->dst, sg_count, DMA_FROM_DEVICE); 1609 } 1610 1611 ret = talitos_sg_map(dev, areq->dst, cryptlen, edesc, &desc->ptr[4], 1612 sg_count, 0, (edesc->src_nents + 1)); 1613 if (ret > 1) 1614 sync_needed = true; 1615 1616 /* iv out */ 1617 map_single_talitos_ptr(dev, &desc->ptr[5], ivsize, ctx->iv, 1618 DMA_FROM_DEVICE); 1619 1620 /* last DWORD empty */ 1621 1622 if (sync_needed) 1623 dma_sync_single_for_device(dev, edesc->dma_link_tbl, 1624 edesc->dma_len, DMA_BIDIRECTIONAL); 1625 1626 ret = talitos_submit(dev, ctx->ch, desc, callback, areq); 1627 if (ret != -EINPROGRESS) { 1628 common_nonsnoop_unmap(dev, edesc, areq); 1629 kfree(edesc); 1630 } 1631 return ret; 1632 } 1633 1634 static struct talitos_edesc *skcipher_edesc_alloc(struct skcipher_request * 1635 areq, bool encrypt) 1636 { 1637 struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(areq); 1638 struct talitos_ctx *ctx = crypto_skcipher_ctx(cipher); 1639 unsigned int ivsize = crypto_skcipher_ivsize(cipher); 1640 1641 return talitos_edesc_alloc(ctx->dev, areq->src, areq->dst, 1642 areq->iv, 0, areq->cryptlen, 0, ivsize, 0, 1643 areq->base.flags, encrypt); 1644 } 1645 1646 static int skcipher_encrypt(struct skcipher_request *areq) 1647 { 1648 struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(areq); 1649 struct talitos_ctx *ctx = crypto_skcipher_ctx(cipher); 1650 struct talitos_edesc *edesc; 1651 unsigned int blocksize = 1652 crypto_tfm_alg_blocksize(crypto_skcipher_tfm(cipher)); 1653 1654 if (!areq->cryptlen) 1655 return 0; 1656 1657 if (areq->cryptlen % blocksize) 1658 return -EINVAL; 1659 1660 /* allocate extended descriptor */ 1661 edesc = skcipher_edesc_alloc(areq, true); 1662 if (IS_ERR(edesc)) 1663 return PTR_ERR(edesc); 1664 1665 /* set encrypt */ 1666 edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT; 1667 1668 return common_nonsnoop(edesc, areq, skcipher_done); 1669 } 1670 1671 static int skcipher_decrypt(struct skcipher_request *areq) 1672 { 1673 struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(areq); 1674 struct talitos_ctx *ctx = crypto_skcipher_ctx(cipher); 1675 struct talitos_edesc *edesc; 1676 unsigned int blocksize = 1677 crypto_tfm_alg_blocksize(crypto_skcipher_tfm(cipher)); 1678 1679 if (!areq->cryptlen) 1680 return 0; 1681 1682 if (areq->cryptlen % blocksize) 1683 return -EINVAL; 1684 1685 /* allocate extended descriptor */ 1686 edesc = skcipher_edesc_alloc(areq, false); 1687 if (IS_ERR(edesc)) 1688 return PTR_ERR(edesc); 1689 1690 edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_DIR_INBOUND; 1691 1692 return common_nonsnoop(edesc, areq, skcipher_done); 1693 } 1694 1695 static void common_nonsnoop_hash_unmap(struct device *dev, 1696 struct talitos_edesc *edesc, 1697 struct ahash_request *areq) 1698 { 1699 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 1700 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); 1701 struct talitos_private *priv = dev_get_drvdata(dev); 1702 bool is_sec1 = has_ftr_sec1(priv); 1703 struct talitos_desc *desc = &edesc->desc; 1704 struct talitos_desc *desc2 = (struct talitos_desc *) 1705 (edesc->buf + edesc->dma_len); 1706 1707 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[5], DMA_FROM_DEVICE); 1708 if (desc->next_desc && 1709 desc->ptr[5].ptr != desc2->ptr[5].ptr) 1710 unmap_single_talitos_ptr(dev, &desc2->ptr[5], DMA_FROM_DEVICE); 1711 if (req_ctx->last) 1712 memcpy(areq->result, req_ctx->hw_context, 1713 crypto_ahash_digestsize(tfm)); 1714 1715 if (req_ctx->psrc) 1716 talitos_sg_unmap(dev, edesc, req_ctx->psrc, NULL, 0, 0); 1717 1718 /* When using hashctx-in, must unmap it. */ 1719 if (from_talitos_ptr_len(&edesc->desc.ptr[1], is_sec1)) 1720 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[1], 1721 DMA_TO_DEVICE); 1722 else if (desc->next_desc) 1723 unmap_single_talitos_ptr(dev, &desc2->ptr[1], 1724 DMA_TO_DEVICE); 1725 1726 if (is_sec1 && req_ctx->nbuf) 1727 unmap_single_talitos_ptr(dev, &desc->ptr[3], 1728 DMA_TO_DEVICE); 1729 1730 if (edesc->dma_len) 1731 dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len, 1732 DMA_BIDIRECTIONAL); 1733 1734 if (edesc->desc.next_desc) 1735 dma_unmap_single(dev, be32_to_cpu(edesc->desc.next_desc), 1736 TALITOS_DESC_SIZE, DMA_BIDIRECTIONAL); 1737 } 1738 1739 static void ahash_done(struct device *dev, 1740 struct talitos_desc *desc, void *context, 1741 int err) 1742 { 1743 struct ahash_request *areq = context; 1744 struct talitos_edesc *edesc = 1745 container_of(desc, struct talitos_edesc, desc); 1746 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 1747 1748 if (!req_ctx->last && req_ctx->to_hash_later) { 1749 /* Position any partial block for next update/final/finup */ 1750 req_ctx->buf_idx = (req_ctx->buf_idx + 1) & 1; 1751 req_ctx->nbuf = req_ctx->to_hash_later; 1752 } 1753 common_nonsnoop_hash_unmap(dev, edesc, areq); 1754 1755 kfree(edesc); 1756 1757 areq->base.complete(&areq->base, err); 1758 } 1759 1760 /* 1761 * SEC1 doesn't like hashing of 0 sized message, so we do the padding 1762 * ourself and submit a padded block 1763 */ 1764 static void talitos_handle_buggy_hash(struct talitos_ctx *ctx, 1765 struct talitos_edesc *edesc, 1766 struct talitos_ptr *ptr) 1767 { 1768 static u8 padded_hash[64] = { 1769 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1770 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1771 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1772 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1773 }; 1774 1775 pr_err_once("Bug in SEC1, padding ourself\n"); 1776 edesc->desc.hdr &= ~DESC_HDR_MODE0_MDEU_PAD; 1777 map_single_talitos_ptr(ctx->dev, ptr, sizeof(padded_hash), 1778 (char *)padded_hash, DMA_TO_DEVICE); 1779 } 1780 1781 static int common_nonsnoop_hash(struct talitos_edesc *edesc, 1782 struct ahash_request *areq, unsigned int length, 1783 void (*callback) (struct device *dev, 1784 struct talitos_desc *desc, 1785 void *context, int error)) 1786 { 1787 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); 1788 struct talitos_ctx *ctx = crypto_ahash_ctx(tfm); 1789 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 1790 struct device *dev = ctx->dev; 1791 struct talitos_desc *desc = &edesc->desc; 1792 int ret; 1793 bool sync_needed = false; 1794 struct talitos_private *priv = dev_get_drvdata(dev); 1795 bool is_sec1 = has_ftr_sec1(priv); 1796 int sg_count; 1797 1798 /* first DWORD empty */ 1799 1800 /* hash context in */ 1801 if (!req_ctx->first || req_ctx->swinit) { 1802 map_single_talitos_ptr_nosync(dev, &desc->ptr[1], 1803 req_ctx->hw_context_size, 1804 req_ctx->hw_context, 1805 DMA_TO_DEVICE); 1806 req_ctx->swinit = 0; 1807 } 1808 /* Indicate next op is not the first. */ 1809 req_ctx->first = 0; 1810 1811 /* HMAC key */ 1812 if (ctx->keylen) 1813 to_talitos_ptr(&desc->ptr[2], ctx->dma_key, ctx->keylen, 1814 is_sec1); 1815 1816 if (is_sec1 && req_ctx->nbuf) 1817 length -= req_ctx->nbuf; 1818 1819 sg_count = edesc->src_nents ?: 1; 1820 if (is_sec1 && sg_count > 1) 1821 sg_copy_to_buffer(req_ctx->psrc, sg_count, edesc->buf, length); 1822 else if (length) 1823 sg_count = dma_map_sg(dev, req_ctx->psrc, sg_count, 1824 DMA_TO_DEVICE); 1825 /* 1826 * data in 1827 */ 1828 if (is_sec1 && req_ctx->nbuf) { 1829 map_single_talitos_ptr(dev, &desc->ptr[3], req_ctx->nbuf, 1830 req_ctx->buf[req_ctx->buf_idx], 1831 DMA_TO_DEVICE); 1832 } else { 1833 sg_count = talitos_sg_map(dev, req_ctx->psrc, length, edesc, 1834 &desc->ptr[3], sg_count, 0, 0); 1835 if (sg_count > 1) 1836 sync_needed = true; 1837 } 1838 1839 /* fifth DWORD empty */ 1840 1841 /* hash/HMAC out -or- hash context out */ 1842 if (req_ctx->last) 1843 map_single_talitos_ptr(dev, &desc->ptr[5], 1844 crypto_ahash_digestsize(tfm), 1845 req_ctx->hw_context, DMA_FROM_DEVICE); 1846 else 1847 map_single_talitos_ptr_nosync(dev, &desc->ptr[5], 1848 req_ctx->hw_context_size, 1849 req_ctx->hw_context, 1850 DMA_FROM_DEVICE); 1851 1852 /* last DWORD empty */ 1853 1854 if (is_sec1 && from_talitos_ptr_len(&desc->ptr[3], true) == 0) 1855 talitos_handle_buggy_hash(ctx, edesc, &desc->ptr[3]); 1856 1857 if (is_sec1 && req_ctx->nbuf && length) { 1858 struct talitos_desc *desc2 = (struct talitos_desc *) 1859 (edesc->buf + edesc->dma_len); 1860 dma_addr_t next_desc; 1861 1862 memset(desc2, 0, sizeof(*desc2)); 1863 desc2->hdr = desc->hdr; 1864 desc2->hdr &= ~DESC_HDR_MODE0_MDEU_INIT; 1865 desc2->hdr1 = desc2->hdr; 1866 desc->hdr &= ~DESC_HDR_MODE0_MDEU_PAD; 1867 desc->hdr |= DESC_HDR_MODE0_MDEU_CONT; 1868 desc->hdr &= ~DESC_HDR_DONE_NOTIFY; 1869 1870 if (desc->ptr[1].ptr) 1871 copy_talitos_ptr(&desc2->ptr[1], &desc->ptr[1], 1872 is_sec1); 1873 else 1874 map_single_talitos_ptr_nosync(dev, &desc2->ptr[1], 1875 req_ctx->hw_context_size, 1876 req_ctx->hw_context, 1877 DMA_TO_DEVICE); 1878 copy_talitos_ptr(&desc2->ptr[2], &desc->ptr[2], is_sec1); 1879 sg_count = talitos_sg_map(dev, req_ctx->psrc, length, edesc, 1880 &desc2->ptr[3], sg_count, 0, 0); 1881 if (sg_count > 1) 1882 sync_needed = true; 1883 copy_talitos_ptr(&desc2->ptr[5], &desc->ptr[5], is_sec1); 1884 if (req_ctx->last) 1885 map_single_talitos_ptr_nosync(dev, &desc->ptr[5], 1886 req_ctx->hw_context_size, 1887 req_ctx->hw_context, 1888 DMA_FROM_DEVICE); 1889 1890 next_desc = dma_map_single(dev, &desc2->hdr1, TALITOS_DESC_SIZE, 1891 DMA_BIDIRECTIONAL); 1892 desc->next_desc = cpu_to_be32(next_desc); 1893 } 1894 1895 if (sync_needed) 1896 dma_sync_single_for_device(dev, edesc->dma_link_tbl, 1897 edesc->dma_len, DMA_BIDIRECTIONAL); 1898 1899 ret = talitos_submit(dev, ctx->ch, desc, callback, areq); 1900 if (ret != -EINPROGRESS) { 1901 common_nonsnoop_hash_unmap(dev, edesc, areq); 1902 kfree(edesc); 1903 } 1904 return ret; 1905 } 1906 1907 static struct talitos_edesc *ahash_edesc_alloc(struct ahash_request *areq, 1908 unsigned int nbytes) 1909 { 1910 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); 1911 struct talitos_ctx *ctx = crypto_ahash_ctx(tfm); 1912 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 1913 struct talitos_private *priv = dev_get_drvdata(ctx->dev); 1914 bool is_sec1 = has_ftr_sec1(priv); 1915 1916 if (is_sec1) 1917 nbytes -= req_ctx->nbuf; 1918 1919 return talitos_edesc_alloc(ctx->dev, req_ctx->psrc, NULL, NULL, 0, 1920 nbytes, 0, 0, 0, areq->base.flags, false); 1921 } 1922 1923 static int ahash_init(struct ahash_request *areq) 1924 { 1925 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); 1926 struct talitos_ctx *ctx = crypto_ahash_ctx(tfm); 1927 struct device *dev = ctx->dev; 1928 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 1929 unsigned int size; 1930 dma_addr_t dma; 1931 1932 /* Initialize the context */ 1933 req_ctx->buf_idx = 0; 1934 req_ctx->nbuf = 0; 1935 req_ctx->first = 1; /* first indicates h/w must init its context */ 1936 req_ctx->swinit = 0; /* assume h/w init of context */ 1937 size = (crypto_ahash_digestsize(tfm) <= SHA256_DIGEST_SIZE) 1938 ? TALITOS_MDEU_CONTEXT_SIZE_MD5_SHA1_SHA256 1939 : TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512; 1940 req_ctx->hw_context_size = size; 1941 1942 dma = dma_map_single(dev, req_ctx->hw_context, req_ctx->hw_context_size, 1943 DMA_TO_DEVICE); 1944 dma_unmap_single(dev, dma, req_ctx->hw_context_size, DMA_TO_DEVICE); 1945 1946 return 0; 1947 } 1948 1949 /* 1950 * on h/w without explicit sha224 support, we initialize h/w context 1951 * manually with sha224 constants, and tell it to run sha256. 1952 */ 1953 static int ahash_init_sha224_swinit(struct ahash_request *areq) 1954 { 1955 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 1956 1957 req_ctx->hw_context[0] = SHA224_H0; 1958 req_ctx->hw_context[1] = SHA224_H1; 1959 req_ctx->hw_context[2] = SHA224_H2; 1960 req_ctx->hw_context[3] = SHA224_H3; 1961 req_ctx->hw_context[4] = SHA224_H4; 1962 req_ctx->hw_context[5] = SHA224_H5; 1963 req_ctx->hw_context[6] = SHA224_H6; 1964 req_ctx->hw_context[7] = SHA224_H7; 1965 1966 /* init 64-bit count */ 1967 req_ctx->hw_context[8] = 0; 1968 req_ctx->hw_context[9] = 0; 1969 1970 ahash_init(areq); 1971 req_ctx->swinit = 1;/* prevent h/w initting context with sha256 values*/ 1972 1973 return 0; 1974 } 1975 1976 static int ahash_process_req(struct ahash_request *areq, unsigned int nbytes) 1977 { 1978 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); 1979 struct talitos_ctx *ctx = crypto_ahash_ctx(tfm); 1980 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 1981 struct talitos_edesc *edesc; 1982 unsigned int blocksize = 1983 crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm)); 1984 unsigned int nbytes_to_hash; 1985 unsigned int to_hash_later; 1986 unsigned int nsg; 1987 int nents; 1988 struct device *dev = ctx->dev; 1989 struct talitos_private *priv = dev_get_drvdata(dev); 1990 bool is_sec1 = has_ftr_sec1(priv); 1991 u8 *ctx_buf = req_ctx->buf[req_ctx->buf_idx]; 1992 1993 if (!req_ctx->last && (nbytes + req_ctx->nbuf <= blocksize)) { 1994 /* Buffer up to one whole block */ 1995 nents = sg_nents_for_len(areq->src, nbytes); 1996 if (nents < 0) { 1997 dev_err(ctx->dev, "Invalid number of src SG.\n"); 1998 return nents; 1999 } 2000 sg_copy_to_buffer(areq->src, nents, 2001 ctx_buf + req_ctx->nbuf, nbytes); 2002 req_ctx->nbuf += nbytes; 2003 return 0; 2004 } 2005 2006 /* At least (blocksize + 1) bytes are available to hash */ 2007 nbytes_to_hash = nbytes + req_ctx->nbuf; 2008 to_hash_later = nbytes_to_hash & (blocksize - 1); 2009 2010 if (req_ctx->last) 2011 to_hash_later = 0; 2012 else if (to_hash_later) 2013 /* There is a partial block. Hash the full block(s) now */ 2014 nbytes_to_hash -= to_hash_later; 2015 else { 2016 /* Keep one block buffered */ 2017 nbytes_to_hash -= blocksize; 2018 to_hash_later = blocksize; 2019 } 2020 2021 /* Chain in any previously buffered data */ 2022 if (!is_sec1 && req_ctx->nbuf) { 2023 nsg = (req_ctx->nbuf < nbytes_to_hash) ? 2 : 1; 2024 sg_init_table(req_ctx->bufsl, nsg); 2025 sg_set_buf(req_ctx->bufsl, ctx_buf, req_ctx->nbuf); 2026 if (nsg > 1) 2027 sg_chain(req_ctx->bufsl, 2, areq->src); 2028 req_ctx->psrc = req_ctx->bufsl; 2029 } else if (is_sec1 && req_ctx->nbuf && req_ctx->nbuf < blocksize) { 2030 int offset; 2031 2032 if (nbytes_to_hash > blocksize) 2033 offset = blocksize - req_ctx->nbuf; 2034 else 2035 offset = nbytes_to_hash - req_ctx->nbuf; 2036 nents = sg_nents_for_len(areq->src, offset); 2037 if (nents < 0) { 2038 dev_err(ctx->dev, "Invalid number of src SG.\n"); 2039 return nents; 2040 } 2041 sg_copy_to_buffer(areq->src, nents, 2042 ctx_buf + req_ctx->nbuf, offset); 2043 req_ctx->nbuf += offset; 2044 req_ctx->psrc = scatterwalk_ffwd(req_ctx->bufsl, areq->src, 2045 offset); 2046 } else 2047 req_ctx->psrc = areq->src; 2048 2049 if (to_hash_later) { 2050 nents = sg_nents_for_len(areq->src, nbytes); 2051 if (nents < 0) { 2052 dev_err(ctx->dev, "Invalid number of src SG.\n"); 2053 return nents; 2054 } 2055 sg_pcopy_to_buffer(areq->src, nents, 2056 req_ctx->buf[(req_ctx->buf_idx + 1) & 1], 2057 to_hash_later, 2058 nbytes - to_hash_later); 2059 } 2060 req_ctx->to_hash_later = to_hash_later; 2061 2062 /* Allocate extended descriptor */ 2063 edesc = ahash_edesc_alloc(areq, nbytes_to_hash); 2064 if (IS_ERR(edesc)) 2065 return PTR_ERR(edesc); 2066 2067 edesc->desc.hdr = ctx->desc_hdr_template; 2068 2069 /* On last one, request SEC to pad; otherwise continue */ 2070 if (req_ctx->last) 2071 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_PAD; 2072 else 2073 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_CONT; 2074 2075 /* request SEC to INIT hash. */ 2076 if (req_ctx->first && !req_ctx->swinit) 2077 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_INIT; 2078 2079 /* When the tfm context has a keylen, it's an HMAC. 2080 * A first or last (ie. not middle) descriptor must request HMAC. 2081 */ 2082 if (ctx->keylen && (req_ctx->first || req_ctx->last)) 2083 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_HMAC; 2084 2085 return common_nonsnoop_hash(edesc, areq, nbytes_to_hash, ahash_done); 2086 } 2087 2088 static int ahash_update(struct ahash_request *areq) 2089 { 2090 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 2091 2092 req_ctx->last = 0; 2093 2094 return ahash_process_req(areq, areq->nbytes); 2095 } 2096 2097 static int ahash_final(struct ahash_request *areq) 2098 { 2099 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 2100 2101 req_ctx->last = 1; 2102 2103 return ahash_process_req(areq, 0); 2104 } 2105 2106 static int ahash_finup(struct ahash_request *areq) 2107 { 2108 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 2109 2110 req_ctx->last = 1; 2111 2112 return ahash_process_req(areq, areq->nbytes); 2113 } 2114 2115 static int ahash_digest(struct ahash_request *areq) 2116 { 2117 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 2118 struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq); 2119 2120 ahash->init(areq); 2121 req_ctx->last = 1; 2122 2123 return ahash_process_req(areq, areq->nbytes); 2124 } 2125 2126 static int ahash_export(struct ahash_request *areq, void *out) 2127 { 2128 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 2129 struct talitos_export_state *export = out; 2130 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); 2131 struct talitos_ctx *ctx = crypto_ahash_ctx(tfm); 2132 struct device *dev = ctx->dev; 2133 dma_addr_t dma; 2134 2135 dma = dma_map_single(dev, req_ctx->hw_context, req_ctx->hw_context_size, 2136 DMA_FROM_DEVICE); 2137 dma_unmap_single(dev, dma, req_ctx->hw_context_size, DMA_FROM_DEVICE); 2138 2139 memcpy(export->hw_context, req_ctx->hw_context, 2140 req_ctx->hw_context_size); 2141 memcpy(export->buf, req_ctx->buf[req_ctx->buf_idx], req_ctx->nbuf); 2142 export->swinit = req_ctx->swinit; 2143 export->first = req_ctx->first; 2144 export->last = req_ctx->last; 2145 export->to_hash_later = req_ctx->to_hash_later; 2146 export->nbuf = req_ctx->nbuf; 2147 2148 return 0; 2149 } 2150 2151 static int ahash_import(struct ahash_request *areq, const void *in) 2152 { 2153 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 2154 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); 2155 struct talitos_ctx *ctx = crypto_ahash_ctx(tfm); 2156 struct device *dev = ctx->dev; 2157 const struct talitos_export_state *export = in; 2158 unsigned int size; 2159 dma_addr_t dma; 2160 2161 memset(req_ctx, 0, sizeof(*req_ctx)); 2162 size = (crypto_ahash_digestsize(tfm) <= SHA256_DIGEST_SIZE) 2163 ? TALITOS_MDEU_CONTEXT_SIZE_MD5_SHA1_SHA256 2164 : TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512; 2165 req_ctx->hw_context_size = size; 2166 memcpy(req_ctx->hw_context, export->hw_context, size); 2167 memcpy(req_ctx->buf[0], export->buf, export->nbuf); 2168 req_ctx->swinit = export->swinit; 2169 req_ctx->first = export->first; 2170 req_ctx->last = export->last; 2171 req_ctx->to_hash_later = export->to_hash_later; 2172 req_ctx->nbuf = export->nbuf; 2173 2174 dma = dma_map_single(dev, req_ctx->hw_context, req_ctx->hw_context_size, 2175 DMA_TO_DEVICE); 2176 dma_unmap_single(dev, dma, req_ctx->hw_context_size, DMA_TO_DEVICE); 2177 2178 return 0; 2179 } 2180 2181 static int keyhash(struct crypto_ahash *tfm, const u8 *key, unsigned int keylen, 2182 u8 *hash) 2183 { 2184 struct talitos_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm)); 2185 2186 struct scatterlist sg[1]; 2187 struct ahash_request *req; 2188 struct crypto_wait wait; 2189 int ret; 2190 2191 crypto_init_wait(&wait); 2192 2193 req = ahash_request_alloc(tfm, GFP_KERNEL); 2194 if (!req) 2195 return -ENOMEM; 2196 2197 /* Keep tfm keylen == 0 during hash of the long key */ 2198 ctx->keylen = 0; 2199 ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, 2200 crypto_req_done, &wait); 2201 2202 sg_init_one(&sg[0], key, keylen); 2203 2204 ahash_request_set_crypt(req, sg, hash, keylen); 2205 ret = crypto_wait_req(crypto_ahash_digest(req), &wait); 2206 2207 ahash_request_free(req); 2208 2209 return ret; 2210 } 2211 2212 static int ahash_setkey(struct crypto_ahash *tfm, const u8 *key, 2213 unsigned int keylen) 2214 { 2215 struct talitos_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm)); 2216 struct device *dev = ctx->dev; 2217 unsigned int blocksize = 2218 crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm)); 2219 unsigned int digestsize = crypto_ahash_digestsize(tfm); 2220 unsigned int keysize = keylen; 2221 u8 hash[SHA512_DIGEST_SIZE]; 2222 int ret; 2223 2224 if (keylen <= blocksize) 2225 memcpy(ctx->key, key, keysize); 2226 else { 2227 /* Must get the hash of the long key */ 2228 ret = keyhash(tfm, key, keylen, hash); 2229 2230 if (ret) 2231 return -EINVAL; 2232 2233 keysize = digestsize; 2234 memcpy(ctx->key, hash, digestsize); 2235 } 2236 2237 if (ctx->keylen) 2238 dma_unmap_single(dev, ctx->dma_key, ctx->keylen, DMA_TO_DEVICE); 2239 2240 ctx->keylen = keysize; 2241 ctx->dma_key = dma_map_single(dev, ctx->key, keysize, DMA_TO_DEVICE); 2242 2243 return 0; 2244 } 2245 2246 2247 struct talitos_alg_template { 2248 u32 type; 2249 u32 priority; 2250 union { 2251 struct skcipher_alg skcipher; 2252 struct ahash_alg hash; 2253 struct aead_alg aead; 2254 } alg; 2255 __be32 desc_hdr_template; 2256 }; 2257 2258 static struct talitos_alg_template driver_algs[] = { 2259 /* AEAD algorithms. These use a single-pass ipsec_esp descriptor */ 2260 { .type = CRYPTO_ALG_TYPE_AEAD, 2261 .alg.aead = { 2262 .base = { 2263 .cra_name = "authenc(hmac(sha1),cbc(aes))", 2264 .cra_driver_name = "authenc-hmac-sha1-" 2265 "cbc-aes-talitos", 2266 .cra_blocksize = AES_BLOCK_SIZE, 2267 .cra_flags = CRYPTO_ALG_ASYNC, 2268 }, 2269 .ivsize = AES_BLOCK_SIZE, 2270 .maxauthsize = SHA1_DIGEST_SIZE, 2271 }, 2272 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | 2273 DESC_HDR_SEL0_AESU | 2274 DESC_HDR_MODE0_AESU_CBC | 2275 DESC_HDR_SEL1_MDEUA | 2276 DESC_HDR_MODE1_MDEU_INIT | 2277 DESC_HDR_MODE1_MDEU_PAD | 2278 DESC_HDR_MODE1_MDEU_SHA1_HMAC, 2279 }, 2280 { .type = CRYPTO_ALG_TYPE_AEAD, 2281 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA, 2282 .alg.aead = { 2283 .base = { 2284 .cra_name = "authenc(hmac(sha1),cbc(aes))", 2285 .cra_driver_name = "authenc-hmac-sha1-" 2286 "cbc-aes-talitos-hsna", 2287 .cra_blocksize = AES_BLOCK_SIZE, 2288 .cra_flags = CRYPTO_ALG_ASYNC, 2289 }, 2290 .ivsize = AES_BLOCK_SIZE, 2291 .maxauthsize = SHA1_DIGEST_SIZE, 2292 }, 2293 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU | 2294 DESC_HDR_SEL0_AESU | 2295 DESC_HDR_MODE0_AESU_CBC | 2296 DESC_HDR_SEL1_MDEUA | 2297 DESC_HDR_MODE1_MDEU_INIT | 2298 DESC_HDR_MODE1_MDEU_PAD | 2299 DESC_HDR_MODE1_MDEU_SHA1_HMAC, 2300 }, 2301 { .type = CRYPTO_ALG_TYPE_AEAD, 2302 .alg.aead = { 2303 .base = { 2304 .cra_name = "authenc(hmac(sha1)," 2305 "cbc(des3_ede))", 2306 .cra_driver_name = "authenc-hmac-sha1-" 2307 "cbc-3des-talitos", 2308 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2309 .cra_flags = CRYPTO_ALG_ASYNC, 2310 }, 2311 .ivsize = DES3_EDE_BLOCK_SIZE, 2312 .maxauthsize = SHA1_DIGEST_SIZE, 2313 .setkey = aead_des3_setkey, 2314 }, 2315 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | 2316 DESC_HDR_SEL0_DEU | 2317 DESC_HDR_MODE0_DEU_CBC | 2318 DESC_HDR_MODE0_DEU_3DES | 2319 DESC_HDR_SEL1_MDEUA | 2320 DESC_HDR_MODE1_MDEU_INIT | 2321 DESC_HDR_MODE1_MDEU_PAD | 2322 DESC_HDR_MODE1_MDEU_SHA1_HMAC, 2323 }, 2324 { .type = CRYPTO_ALG_TYPE_AEAD, 2325 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA, 2326 .alg.aead = { 2327 .base = { 2328 .cra_name = "authenc(hmac(sha1)," 2329 "cbc(des3_ede))", 2330 .cra_driver_name = "authenc-hmac-sha1-" 2331 "cbc-3des-talitos-hsna", 2332 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2333 .cra_flags = CRYPTO_ALG_ASYNC, 2334 }, 2335 .ivsize = DES3_EDE_BLOCK_SIZE, 2336 .maxauthsize = SHA1_DIGEST_SIZE, 2337 .setkey = aead_des3_setkey, 2338 }, 2339 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU | 2340 DESC_HDR_SEL0_DEU | 2341 DESC_HDR_MODE0_DEU_CBC | 2342 DESC_HDR_MODE0_DEU_3DES | 2343 DESC_HDR_SEL1_MDEUA | 2344 DESC_HDR_MODE1_MDEU_INIT | 2345 DESC_HDR_MODE1_MDEU_PAD | 2346 DESC_HDR_MODE1_MDEU_SHA1_HMAC, 2347 }, 2348 { .type = CRYPTO_ALG_TYPE_AEAD, 2349 .alg.aead = { 2350 .base = { 2351 .cra_name = "authenc(hmac(sha224),cbc(aes))", 2352 .cra_driver_name = "authenc-hmac-sha224-" 2353 "cbc-aes-talitos", 2354 .cra_blocksize = AES_BLOCK_SIZE, 2355 .cra_flags = CRYPTO_ALG_ASYNC, 2356 }, 2357 .ivsize = AES_BLOCK_SIZE, 2358 .maxauthsize = SHA224_DIGEST_SIZE, 2359 }, 2360 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | 2361 DESC_HDR_SEL0_AESU | 2362 DESC_HDR_MODE0_AESU_CBC | 2363 DESC_HDR_SEL1_MDEUA | 2364 DESC_HDR_MODE1_MDEU_INIT | 2365 DESC_HDR_MODE1_MDEU_PAD | 2366 DESC_HDR_MODE1_MDEU_SHA224_HMAC, 2367 }, 2368 { .type = CRYPTO_ALG_TYPE_AEAD, 2369 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA, 2370 .alg.aead = { 2371 .base = { 2372 .cra_name = "authenc(hmac(sha224),cbc(aes))", 2373 .cra_driver_name = "authenc-hmac-sha224-" 2374 "cbc-aes-talitos-hsna", 2375 .cra_blocksize = AES_BLOCK_SIZE, 2376 .cra_flags = CRYPTO_ALG_ASYNC, 2377 }, 2378 .ivsize = AES_BLOCK_SIZE, 2379 .maxauthsize = SHA224_DIGEST_SIZE, 2380 }, 2381 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU | 2382 DESC_HDR_SEL0_AESU | 2383 DESC_HDR_MODE0_AESU_CBC | 2384 DESC_HDR_SEL1_MDEUA | 2385 DESC_HDR_MODE1_MDEU_INIT | 2386 DESC_HDR_MODE1_MDEU_PAD | 2387 DESC_HDR_MODE1_MDEU_SHA224_HMAC, 2388 }, 2389 { .type = CRYPTO_ALG_TYPE_AEAD, 2390 .alg.aead = { 2391 .base = { 2392 .cra_name = "authenc(hmac(sha224)," 2393 "cbc(des3_ede))", 2394 .cra_driver_name = "authenc-hmac-sha224-" 2395 "cbc-3des-talitos", 2396 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2397 .cra_flags = CRYPTO_ALG_ASYNC, 2398 }, 2399 .ivsize = DES3_EDE_BLOCK_SIZE, 2400 .maxauthsize = SHA224_DIGEST_SIZE, 2401 .setkey = aead_des3_setkey, 2402 }, 2403 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | 2404 DESC_HDR_SEL0_DEU | 2405 DESC_HDR_MODE0_DEU_CBC | 2406 DESC_HDR_MODE0_DEU_3DES | 2407 DESC_HDR_SEL1_MDEUA | 2408 DESC_HDR_MODE1_MDEU_INIT | 2409 DESC_HDR_MODE1_MDEU_PAD | 2410 DESC_HDR_MODE1_MDEU_SHA224_HMAC, 2411 }, 2412 { .type = CRYPTO_ALG_TYPE_AEAD, 2413 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA, 2414 .alg.aead = { 2415 .base = { 2416 .cra_name = "authenc(hmac(sha224)," 2417 "cbc(des3_ede))", 2418 .cra_driver_name = "authenc-hmac-sha224-" 2419 "cbc-3des-talitos-hsna", 2420 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2421 .cra_flags = CRYPTO_ALG_ASYNC, 2422 }, 2423 .ivsize = DES3_EDE_BLOCK_SIZE, 2424 .maxauthsize = SHA224_DIGEST_SIZE, 2425 .setkey = aead_des3_setkey, 2426 }, 2427 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU | 2428 DESC_HDR_SEL0_DEU | 2429 DESC_HDR_MODE0_DEU_CBC | 2430 DESC_HDR_MODE0_DEU_3DES | 2431 DESC_HDR_SEL1_MDEUA | 2432 DESC_HDR_MODE1_MDEU_INIT | 2433 DESC_HDR_MODE1_MDEU_PAD | 2434 DESC_HDR_MODE1_MDEU_SHA224_HMAC, 2435 }, 2436 { .type = CRYPTO_ALG_TYPE_AEAD, 2437 .alg.aead = { 2438 .base = { 2439 .cra_name = "authenc(hmac(sha256),cbc(aes))", 2440 .cra_driver_name = "authenc-hmac-sha256-" 2441 "cbc-aes-talitos", 2442 .cra_blocksize = AES_BLOCK_SIZE, 2443 .cra_flags = CRYPTO_ALG_ASYNC, 2444 }, 2445 .ivsize = AES_BLOCK_SIZE, 2446 .maxauthsize = SHA256_DIGEST_SIZE, 2447 }, 2448 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | 2449 DESC_HDR_SEL0_AESU | 2450 DESC_HDR_MODE0_AESU_CBC | 2451 DESC_HDR_SEL1_MDEUA | 2452 DESC_HDR_MODE1_MDEU_INIT | 2453 DESC_HDR_MODE1_MDEU_PAD | 2454 DESC_HDR_MODE1_MDEU_SHA256_HMAC, 2455 }, 2456 { .type = CRYPTO_ALG_TYPE_AEAD, 2457 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA, 2458 .alg.aead = { 2459 .base = { 2460 .cra_name = "authenc(hmac(sha256),cbc(aes))", 2461 .cra_driver_name = "authenc-hmac-sha256-" 2462 "cbc-aes-talitos-hsna", 2463 .cra_blocksize = AES_BLOCK_SIZE, 2464 .cra_flags = CRYPTO_ALG_ASYNC, 2465 }, 2466 .ivsize = AES_BLOCK_SIZE, 2467 .maxauthsize = SHA256_DIGEST_SIZE, 2468 }, 2469 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU | 2470 DESC_HDR_SEL0_AESU | 2471 DESC_HDR_MODE0_AESU_CBC | 2472 DESC_HDR_SEL1_MDEUA | 2473 DESC_HDR_MODE1_MDEU_INIT | 2474 DESC_HDR_MODE1_MDEU_PAD | 2475 DESC_HDR_MODE1_MDEU_SHA256_HMAC, 2476 }, 2477 { .type = CRYPTO_ALG_TYPE_AEAD, 2478 .alg.aead = { 2479 .base = { 2480 .cra_name = "authenc(hmac(sha256)," 2481 "cbc(des3_ede))", 2482 .cra_driver_name = "authenc-hmac-sha256-" 2483 "cbc-3des-talitos", 2484 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2485 .cra_flags = CRYPTO_ALG_ASYNC, 2486 }, 2487 .ivsize = DES3_EDE_BLOCK_SIZE, 2488 .maxauthsize = SHA256_DIGEST_SIZE, 2489 .setkey = aead_des3_setkey, 2490 }, 2491 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | 2492 DESC_HDR_SEL0_DEU | 2493 DESC_HDR_MODE0_DEU_CBC | 2494 DESC_HDR_MODE0_DEU_3DES | 2495 DESC_HDR_SEL1_MDEUA | 2496 DESC_HDR_MODE1_MDEU_INIT | 2497 DESC_HDR_MODE1_MDEU_PAD | 2498 DESC_HDR_MODE1_MDEU_SHA256_HMAC, 2499 }, 2500 { .type = CRYPTO_ALG_TYPE_AEAD, 2501 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA, 2502 .alg.aead = { 2503 .base = { 2504 .cra_name = "authenc(hmac(sha256)," 2505 "cbc(des3_ede))", 2506 .cra_driver_name = "authenc-hmac-sha256-" 2507 "cbc-3des-talitos-hsna", 2508 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2509 .cra_flags = CRYPTO_ALG_ASYNC, 2510 }, 2511 .ivsize = DES3_EDE_BLOCK_SIZE, 2512 .maxauthsize = SHA256_DIGEST_SIZE, 2513 .setkey = aead_des3_setkey, 2514 }, 2515 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU | 2516 DESC_HDR_SEL0_DEU | 2517 DESC_HDR_MODE0_DEU_CBC | 2518 DESC_HDR_MODE0_DEU_3DES | 2519 DESC_HDR_SEL1_MDEUA | 2520 DESC_HDR_MODE1_MDEU_INIT | 2521 DESC_HDR_MODE1_MDEU_PAD | 2522 DESC_HDR_MODE1_MDEU_SHA256_HMAC, 2523 }, 2524 { .type = CRYPTO_ALG_TYPE_AEAD, 2525 .alg.aead = { 2526 .base = { 2527 .cra_name = "authenc(hmac(sha384),cbc(aes))", 2528 .cra_driver_name = "authenc-hmac-sha384-" 2529 "cbc-aes-talitos", 2530 .cra_blocksize = AES_BLOCK_SIZE, 2531 .cra_flags = CRYPTO_ALG_ASYNC, 2532 }, 2533 .ivsize = AES_BLOCK_SIZE, 2534 .maxauthsize = SHA384_DIGEST_SIZE, 2535 }, 2536 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | 2537 DESC_HDR_SEL0_AESU | 2538 DESC_HDR_MODE0_AESU_CBC | 2539 DESC_HDR_SEL1_MDEUB | 2540 DESC_HDR_MODE1_MDEU_INIT | 2541 DESC_HDR_MODE1_MDEU_PAD | 2542 DESC_HDR_MODE1_MDEUB_SHA384_HMAC, 2543 }, 2544 { .type = CRYPTO_ALG_TYPE_AEAD, 2545 .alg.aead = { 2546 .base = { 2547 .cra_name = "authenc(hmac(sha384)," 2548 "cbc(des3_ede))", 2549 .cra_driver_name = "authenc-hmac-sha384-" 2550 "cbc-3des-talitos", 2551 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2552 .cra_flags = CRYPTO_ALG_ASYNC, 2553 }, 2554 .ivsize = DES3_EDE_BLOCK_SIZE, 2555 .maxauthsize = SHA384_DIGEST_SIZE, 2556 .setkey = aead_des3_setkey, 2557 }, 2558 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | 2559 DESC_HDR_SEL0_DEU | 2560 DESC_HDR_MODE0_DEU_CBC | 2561 DESC_HDR_MODE0_DEU_3DES | 2562 DESC_HDR_SEL1_MDEUB | 2563 DESC_HDR_MODE1_MDEU_INIT | 2564 DESC_HDR_MODE1_MDEU_PAD | 2565 DESC_HDR_MODE1_MDEUB_SHA384_HMAC, 2566 }, 2567 { .type = CRYPTO_ALG_TYPE_AEAD, 2568 .alg.aead = { 2569 .base = { 2570 .cra_name = "authenc(hmac(sha512),cbc(aes))", 2571 .cra_driver_name = "authenc-hmac-sha512-" 2572 "cbc-aes-talitos", 2573 .cra_blocksize = AES_BLOCK_SIZE, 2574 .cra_flags = CRYPTO_ALG_ASYNC, 2575 }, 2576 .ivsize = AES_BLOCK_SIZE, 2577 .maxauthsize = SHA512_DIGEST_SIZE, 2578 }, 2579 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | 2580 DESC_HDR_SEL0_AESU | 2581 DESC_HDR_MODE0_AESU_CBC | 2582 DESC_HDR_SEL1_MDEUB | 2583 DESC_HDR_MODE1_MDEU_INIT | 2584 DESC_HDR_MODE1_MDEU_PAD | 2585 DESC_HDR_MODE1_MDEUB_SHA512_HMAC, 2586 }, 2587 { .type = CRYPTO_ALG_TYPE_AEAD, 2588 .alg.aead = { 2589 .base = { 2590 .cra_name = "authenc(hmac(sha512)," 2591 "cbc(des3_ede))", 2592 .cra_driver_name = "authenc-hmac-sha512-" 2593 "cbc-3des-talitos", 2594 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2595 .cra_flags = CRYPTO_ALG_ASYNC, 2596 }, 2597 .ivsize = DES3_EDE_BLOCK_SIZE, 2598 .maxauthsize = SHA512_DIGEST_SIZE, 2599 .setkey = aead_des3_setkey, 2600 }, 2601 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | 2602 DESC_HDR_SEL0_DEU | 2603 DESC_HDR_MODE0_DEU_CBC | 2604 DESC_HDR_MODE0_DEU_3DES | 2605 DESC_HDR_SEL1_MDEUB | 2606 DESC_HDR_MODE1_MDEU_INIT | 2607 DESC_HDR_MODE1_MDEU_PAD | 2608 DESC_HDR_MODE1_MDEUB_SHA512_HMAC, 2609 }, 2610 { .type = CRYPTO_ALG_TYPE_AEAD, 2611 .alg.aead = { 2612 .base = { 2613 .cra_name = "authenc(hmac(md5),cbc(aes))", 2614 .cra_driver_name = "authenc-hmac-md5-" 2615 "cbc-aes-talitos", 2616 .cra_blocksize = AES_BLOCK_SIZE, 2617 .cra_flags = CRYPTO_ALG_ASYNC, 2618 }, 2619 .ivsize = AES_BLOCK_SIZE, 2620 .maxauthsize = MD5_DIGEST_SIZE, 2621 }, 2622 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | 2623 DESC_HDR_SEL0_AESU | 2624 DESC_HDR_MODE0_AESU_CBC | 2625 DESC_HDR_SEL1_MDEUA | 2626 DESC_HDR_MODE1_MDEU_INIT | 2627 DESC_HDR_MODE1_MDEU_PAD | 2628 DESC_HDR_MODE1_MDEU_MD5_HMAC, 2629 }, 2630 { .type = CRYPTO_ALG_TYPE_AEAD, 2631 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA, 2632 .alg.aead = { 2633 .base = { 2634 .cra_name = "authenc(hmac(md5),cbc(aes))", 2635 .cra_driver_name = "authenc-hmac-md5-" 2636 "cbc-aes-talitos-hsna", 2637 .cra_blocksize = AES_BLOCK_SIZE, 2638 .cra_flags = CRYPTO_ALG_ASYNC, 2639 }, 2640 .ivsize = AES_BLOCK_SIZE, 2641 .maxauthsize = MD5_DIGEST_SIZE, 2642 }, 2643 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU | 2644 DESC_HDR_SEL0_AESU | 2645 DESC_HDR_MODE0_AESU_CBC | 2646 DESC_HDR_SEL1_MDEUA | 2647 DESC_HDR_MODE1_MDEU_INIT | 2648 DESC_HDR_MODE1_MDEU_PAD | 2649 DESC_HDR_MODE1_MDEU_MD5_HMAC, 2650 }, 2651 { .type = CRYPTO_ALG_TYPE_AEAD, 2652 .alg.aead = { 2653 .base = { 2654 .cra_name = "authenc(hmac(md5),cbc(des3_ede))", 2655 .cra_driver_name = "authenc-hmac-md5-" 2656 "cbc-3des-talitos", 2657 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2658 .cra_flags = CRYPTO_ALG_ASYNC, 2659 }, 2660 .ivsize = DES3_EDE_BLOCK_SIZE, 2661 .maxauthsize = MD5_DIGEST_SIZE, 2662 .setkey = aead_des3_setkey, 2663 }, 2664 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | 2665 DESC_HDR_SEL0_DEU | 2666 DESC_HDR_MODE0_DEU_CBC | 2667 DESC_HDR_MODE0_DEU_3DES | 2668 DESC_HDR_SEL1_MDEUA | 2669 DESC_HDR_MODE1_MDEU_INIT | 2670 DESC_HDR_MODE1_MDEU_PAD | 2671 DESC_HDR_MODE1_MDEU_MD5_HMAC, 2672 }, 2673 { .type = CRYPTO_ALG_TYPE_AEAD, 2674 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA, 2675 .alg.aead = { 2676 .base = { 2677 .cra_name = "authenc(hmac(md5),cbc(des3_ede))", 2678 .cra_driver_name = "authenc-hmac-md5-" 2679 "cbc-3des-talitos-hsna", 2680 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2681 .cra_flags = CRYPTO_ALG_ASYNC, 2682 }, 2683 .ivsize = DES3_EDE_BLOCK_SIZE, 2684 .maxauthsize = MD5_DIGEST_SIZE, 2685 .setkey = aead_des3_setkey, 2686 }, 2687 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU | 2688 DESC_HDR_SEL0_DEU | 2689 DESC_HDR_MODE0_DEU_CBC | 2690 DESC_HDR_MODE0_DEU_3DES | 2691 DESC_HDR_SEL1_MDEUA | 2692 DESC_HDR_MODE1_MDEU_INIT | 2693 DESC_HDR_MODE1_MDEU_PAD | 2694 DESC_HDR_MODE1_MDEU_MD5_HMAC, 2695 }, 2696 /* SKCIPHER algorithms. */ 2697 { .type = CRYPTO_ALG_TYPE_SKCIPHER, 2698 .alg.skcipher = { 2699 .base.cra_name = "ecb(aes)", 2700 .base.cra_driver_name = "ecb-aes-talitos", 2701 .base.cra_blocksize = AES_BLOCK_SIZE, 2702 .base.cra_flags = CRYPTO_ALG_ASYNC, 2703 .min_keysize = AES_MIN_KEY_SIZE, 2704 .max_keysize = AES_MAX_KEY_SIZE, 2705 .setkey = skcipher_aes_setkey, 2706 }, 2707 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 2708 DESC_HDR_SEL0_AESU, 2709 }, 2710 { .type = CRYPTO_ALG_TYPE_SKCIPHER, 2711 .alg.skcipher = { 2712 .base.cra_name = "cbc(aes)", 2713 .base.cra_driver_name = "cbc-aes-talitos", 2714 .base.cra_blocksize = AES_BLOCK_SIZE, 2715 .base.cra_flags = CRYPTO_ALG_ASYNC, 2716 .min_keysize = AES_MIN_KEY_SIZE, 2717 .max_keysize = AES_MAX_KEY_SIZE, 2718 .ivsize = AES_BLOCK_SIZE, 2719 .setkey = skcipher_aes_setkey, 2720 }, 2721 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 2722 DESC_HDR_SEL0_AESU | 2723 DESC_HDR_MODE0_AESU_CBC, 2724 }, 2725 { .type = CRYPTO_ALG_TYPE_SKCIPHER, 2726 .alg.skcipher = { 2727 .base.cra_name = "ctr(aes)", 2728 .base.cra_driver_name = "ctr-aes-talitos", 2729 .base.cra_blocksize = 1, 2730 .base.cra_flags = CRYPTO_ALG_ASYNC, 2731 .min_keysize = AES_MIN_KEY_SIZE, 2732 .max_keysize = AES_MAX_KEY_SIZE, 2733 .ivsize = AES_BLOCK_SIZE, 2734 .setkey = skcipher_aes_setkey, 2735 }, 2736 .desc_hdr_template = DESC_HDR_TYPE_AESU_CTR_NONSNOOP | 2737 DESC_HDR_SEL0_AESU | 2738 DESC_HDR_MODE0_AESU_CTR, 2739 }, 2740 { .type = CRYPTO_ALG_TYPE_SKCIPHER, 2741 .alg.skcipher = { 2742 .base.cra_name = "ecb(des)", 2743 .base.cra_driver_name = "ecb-des-talitos", 2744 .base.cra_blocksize = DES_BLOCK_SIZE, 2745 .base.cra_flags = CRYPTO_ALG_ASYNC, 2746 .min_keysize = DES_KEY_SIZE, 2747 .max_keysize = DES_KEY_SIZE, 2748 .setkey = skcipher_des_setkey, 2749 }, 2750 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 2751 DESC_HDR_SEL0_DEU, 2752 }, 2753 { .type = CRYPTO_ALG_TYPE_SKCIPHER, 2754 .alg.skcipher = { 2755 .base.cra_name = "cbc(des)", 2756 .base.cra_driver_name = "cbc-des-talitos", 2757 .base.cra_blocksize = DES_BLOCK_SIZE, 2758 .base.cra_flags = CRYPTO_ALG_ASYNC, 2759 .min_keysize = DES_KEY_SIZE, 2760 .max_keysize = DES_KEY_SIZE, 2761 .ivsize = DES_BLOCK_SIZE, 2762 .setkey = skcipher_des_setkey, 2763 }, 2764 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 2765 DESC_HDR_SEL0_DEU | 2766 DESC_HDR_MODE0_DEU_CBC, 2767 }, 2768 { .type = CRYPTO_ALG_TYPE_SKCIPHER, 2769 .alg.skcipher = { 2770 .base.cra_name = "ecb(des3_ede)", 2771 .base.cra_driver_name = "ecb-3des-talitos", 2772 .base.cra_blocksize = DES3_EDE_BLOCK_SIZE, 2773 .base.cra_flags = CRYPTO_ALG_ASYNC, 2774 .min_keysize = DES3_EDE_KEY_SIZE, 2775 .max_keysize = DES3_EDE_KEY_SIZE, 2776 .setkey = skcipher_des3_setkey, 2777 }, 2778 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 2779 DESC_HDR_SEL0_DEU | 2780 DESC_HDR_MODE0_DEU_3DES, 2781 }, 2782 { .type = CRYPTO_ALG_TYPE_SKCIPHER, 2783 .alg.skcipher = { 2784 .base.cra_name = "cbc(des3_ede)", 2785 .base.cra_driver_name = "cbc-3des-talitos", 2786 .base.cra_blocksize = DES3_EDE_BLOCK_SIZE, 2787 .base.cra_flags = CRYPTO_ALG_ASYNC, 2788 .min_keysize = DES3_EDE_KEY_SIZE, 2789 .max_keysize = DES3_EDE_KEY_SIZE, 2790 .ivsize = DES3_EDE_BLOCK_SIZE, 2791 .setkey = skcipher_des3_setkey, 2792 }, 2793 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 2794 DESC_HDR_SEL0_DEU | 2795 DESC_HDR_MODE0_DEU_CBC | 2796 DESC_HDR_MODE0_DEU_3DES, 2797 }, 2798 /* AHASH algorithms. */ 2799 { .type = CRYPTO_ALG_TYPE_AHASH, 2800 .alg.hash = { 2801 .halg.digestsize = MD5_DIGEST_SIZE, 2802 .halg.statesize = sizeof(struct talitos_export_state), 2803 .halg.base = { 2804 .cra_name = "md5", 2805 .cra_driver_name = "md5-talitos", 2806 .cra_blocksize = MD5_HMAC_BLOCK_SIZE, 2807 .cra_flags = CRYPTO_ALG_ASYNC, 2808 } 2809 }, 2810 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 2811 DESC_HDR_SEL0_MDEUA | 2812 DESC_HDR_MODE0_MDEU_MD5, 2813 }, 2814 { .type = CRYPTO_ALG_TYPE_AHASH, 2815 .alg.hash = { 2816 .halg.digestsize = SHA1_DIGEST_SIZE, 2817 .halg.statesize = sizeof(struct talitos_export_state), 2818 .halg.base = { 2819 .cra_name = "sha1", 2820 .cra_driver_name = "sha1-talitos", 2821 .cra_blocksize = SHA1_BLOCK_SIZE, 2822 .cra_flags = CRYPTO_ALG_ASYNC, 2823 } 2824 }, 2825 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 2826 DESC_HDR_SEL0_MDEUA | 2827 DESC_HDR_MODE0_MDEU_SHA1, 2828 }, 2829 { .type = CRYPTO_ALG_TYPE_AHASH, 2830 .alg.hash = { 2831 .halg.digestsize = SHA224_DIGEST_SIZE, 2832 .halg.statesize = sizeof(struct talitos_export_state), 2833 .halg.base = { 2834 .cra_name = "sha224", 2835 .cra_driver_name = "sha224-talitos", 2836 .cra_blocksize = SHA224_BLOCK_SIZE, 2837 .cra_flags = CRYPTO_ALG_ASYNC, 2838 } 2839 }, 2840 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 2841 DESC_HDR_SEL0_MDEUA | 2842 DESC_HDR_MODE0_MDEU_SHA224, 2843 }, 2844 { .type = CRYPTO_ALG_TYPE_AHASH, 2845 .alg.hash = { 2846 .halg.digestsize = SHA256_DIGEST_SIZE, 2847 .halg.statesize = sizeof(struct talitos_export_state), 2848 .halg.base = { 2849 .cra_name = "sha256", 2850 .cra_driver_name = "sha256-talitos", 2851 .cra_blocksize = SHA256_BLOCK_SIZE, 2852 .cra_flags = CRYPTO_ALG_ASYNC, 2853 } 2854 }, 2855 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 2856 DESC_HDR_SEL0_MDEUA | 2857 DESC_HDR_MODE0_MDEU_SHA256, 2858 }, 2859 { .type = CRYPTO_ALG_TYPE_AHASH, 2860 .alg.hash = { 2861 .halg.digestsize = SHA384_DIGEST_SIZE, 2862 .halg.statesize = sizeof(struct talitos_export_state), 2863 .halg.base = { 2864 .cra_name = "sha384", 2865 .cra_driver_name = "sha384-talitos", 2866 .cra_blocksize = SHA384_BLOCK_SIZE, 2867 .cra_flags = CRYPTO_ALG_ASYNC, 2868 } 2869 }, 2870 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 2871 DESC_HDR_SEL0_MDEUB | 2872 DESC_HDR_MODE0_MDEUB_SHA384, 2873 }, 2874 { .type = CRYPTO_ALG_TYPE_AHASH, 2875 .alg.hash = { 2876 .halg.digestsize = SHA512_DIGEST_SIZE, 2877 .halg.statesize = sizeof(struct talitos_export_state), 2878 .halg.base = { 2879 .cra_name = "sha512", 2880 .cra_driver_name = "sha512-talitos", 2881 .cra_blocksize = SHA512_BLOCK_SIZE, 2882 .cra_flags = CRYPTO_ALG_ASYNC, 2883 } 2884 }, 2885 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 2886 DESC_HDR_SEL0_MDEUB | 2887 DESC_HDR_MODE0_MDEUB_SHA512, 2888 }, 2889 { .type = CRYPTO_ALG_TYPE_AHASH, 2890 .alg.hash = { 2891 .halg.digestsize = MD5_DIGEST_SIZE, 2892 .halg.statesize = sizeof(struct talitos_export_state), 2893 .halg.base = { 2894 .cra_name = "hmac(md5)", 2895 .cra_driver_name = "hmac-md5-talitos", 2896 .cra_blocksize = MD5_HMAC_BLOCK_SIZE, 2897 .cra_flags = CRYPTO_ALG_ASYNC, 2898 } 2899 }, 2900 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 2901 DESC_HDR_SEL0_MDEUA | 2902 DESC_HDR_MODE0_MDEU_MD5, 2903 }, 2904 { .type = CRYPTO_ALG_TYPE_AHASH, 2905 .alg.hash = { 2906 .halg.digestsize = SHA1_DIGEST_SIZE, 2907 .halg.statesize = sizeof(struct talitos_export_state), 2908 .halg.base = { 2909 .cra_name = "hmac(sha1)", 2910 .cra_driver_name = "hmac-sha1-talitos", 2911 .cra_blocksize = SHA1_BLOCK_SIZE, 2912 .cra_flags = CRYPTO_ALG_ASYNC, 2913 } 2914 }, 2915 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 2916 DESC_HDR_SEL0_MDEUA | 2917 DESC_HDR_MODE0_MDEU_SHA1, 2918 }, 2919 { .type = CRYPTO_ALG_TYPE_AHASH, 2920 .alg.hash = { 2921 .halg.digestsize = SHA224_DIGEST_SIZE, 2922 .halg.statesize = sizeof(struct talitos_export_state), 2923 .halg.base = { 2924 .cra_name = "hmac(sha224)", 2925 .cra_driver_name = "hmac-sha224-talitos", 2926 .cra_blocksize = SHA224_BLOCK_SIZE, 2927 .cra_flags = CRYPTO_ALG_ASYNC, 2928 } 2929 }, 2930 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 2931 DESC_HDR_SEL0_MDEUA | 2932 DESC_HDR_MODE0_MDEU_SHA224, 2933 }, 2934 { .type = CRYPTO_ALG_TYPE_AHASH, 2935 .alg.hash = { 2936 .halg.digestsize = SHA256_DIGEST_SIZE, 2937 .halg.statesize = sizeof(struct talitos_export_state), 2938 .halg.base = { 2939 .cra_name = "hmac(sha256)", 2940 .cra_driver_name = "hmac-sha256-talitos", 2941 .cra_blocksize = SHA256_BLOCK_SIZE, 2942 .cra_flags = CRYPTO_ALG_ASYNC, 2943 } 2944 }, 2945 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 2946 DESC_HDR_SEL0_MDEUA | 2947 DESC_HDR_MODE0_MDEU_SHA256, 2948 }, 2949 { .type = CRYPTO_ALG_TYPE_AHASH, 2950 .alg.hash = { 2951 .halg.digestsize = SHA384_DIGEST_SIZE, 2952 .halg.statesize = sizeof(struct talitos_export_state), 2953 .halg.base = { 2954 .cra_name = "hmac(sha384)", 2955 .cra_driver_name = "hmac-sha384-talitos", 2956 .cra_blocksize = SHA384_BLOCK_SIZE, 2957 .cra_flags = CRYPTO_ALG_ASYNC, 2958 } 2959 }, 2960 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 2961 DESC_HDR_SEL0_MDEUB | 2962 DESC_HDR_MODE0_MDEUB_SHA384, 2963 }, 2964 { .type = CRYPTO_ALG_TYPE_AHASH, 2965 .alg.hash = { 2966 .halg.digestsize = SHA512_DIGEST_SIZE, 2967 .halg.statesize = sizeof(struct talitos_export_state), 2968 .halg.base = { 2969 .cra_name = "hmac(sha512)", 2970 .cra_driver_name = "hmac-sha512-talitos", 2971 .cra_blocksize = SHA512_BLOCK_SIZE, 2972 .cra_flags = CRYPTO_ALG_ASYNC, 2973 } 2974 }, 2975 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 2976 DESC_HDR_SEL0_MDEUB | 2977 DESC_HDR_MODE0_MDEUB_SHA512, 2978 } 2979 }; 2980 2981 struct talitos_crypto_alg { 2982 struct list_head entry; 2983 struct device *dev; 2984 struct talitos_alg_template algt; 2985 }; 2986 2987 static int talitos_init_common(struct talitos_ctx *ctx, 2988 struct talitos_crypto_alg *talitos_alg) 2989 { 2990 struct talitos_private *priv; 2991 2992 /* update context with ptr to dev */ 2993 ctx->dev = talitos_alg->dev; 2994 2995 /* assign SEC channel to tfm in round-robin fashion */ 2996 priv = dev_get_drvdata(ctx->dev); 2997 ctx->ch = atomic_inc_return(&priv->last_chan) & 2998 (priv->num_channels - 1); 2999 3000 /* copy descriptor header template value */ 3001 ctx->desc_hdr_template = talitos_alg->algt.desc_hdr_template; 3002 3003 /* select done notification */ 3004 ctx->desc_hdr_template |= DESC_HDR_DONE_NOTIFY; 3005 3006 return 0; 3007 } 3008 3009 static int talitos_cra_init_aead(struct crypto_aead *tfm) 3010 { 3011 struct aead_alg *alg = crypto_aead_alg(tfm); 3012 struct talitos_crypto_alg *talitos_alg; 3013 struct talitos_ctx *ctx = crypto_aead_ctx(tfm); 3014 3015 talitos_alg = container_of(alg, struct talitos_crypto_alg, 3016 algt.alg.aead); 3017 3018 return talitos_init_common(ctx, talitos_alg); 3019 } 3020 3021 static int talitos_cra_init_skcipher(struct crypto_skcipher *tfm) 3022 { 3023 struct skcipher_alg *alg = crypto_skcipher_alg(tfm); 3024 struct talitos_crypto_alg *talitos_alg; 3025 struct talitos_ctx *ctx = crypto_skcipher_ctx(tfm); 3026 3027 talitos_alg = container_of(alg, struct talitos_crypto_alg, 3028 algt.alg.skcipher); 3029 3030 return talitos_init_common(ctx, talitos_alg); 3031 } 3032 3033 static int talitos_cra_init_ahash(struct crypto_tfm *tfm) 3034 { 3035 struct crypto_alg *alg = tfm->__crt_alg; 3036 struct talitos_crypto_alg *talitos_alg; 3037 struct talitos_ctx *ctx = crypto_tfm_ctx(tfm); 3038 3039 talitos_alg = container_of(__crypto_ahash_alg(alg), 3040 struct talitos_crypto_alg, 3041 algt.alg.hash); 3042 3043 ctx->keylen = 0; 3044 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), 3045 sizeof(struct talitos_ahash_req_ctx)); 3046 3047 return talitos_init_common(ctx, talitos_alg); 3048 } 3049 3050 static void talitos_cra_exit(struct crypto_tfm *tfm) 3051 { 3052 struct talitos_ctx *ctx = crypto_tfm_ctx(tfm); 3053 struct device *dev = ctx->dev; 3054 3055 if (ctx->keylen) 3056 dma_unmap_single(dev, ctx->dma_key, ctx->keylen, DMA_TO_DEVICE); 3057 } 3058 3059 /* 3060 * given the alg's descriptor header template, determine whether descriptor 3061 * type and primary/secondary execution units required match the hw 3062 * capabilities description provided in the device tree node. 3063 */ 3064 static int hw_supports(struct device *dev, __be32 desc_hdr_template) 3065 { 3066 struct talitos_private *priv = dev_get_drvdata(dev); 3067 int ret; 3068 3069 ret = (1 << DESC_TYPE(desc_hdr_template) & priv->desc_types) && 3070 (1 << PRIMARY_EU(desc_hdr_template) & priv->exec_units); 3071 3072 if (SECONDARY_EU(desc_hdr_template)) 3073 ret = ret && (1 << SECONDARY_EU(desc_hdr_template) 3074 & priv->exec_units); 3075 3076 return ret; 3077 } 3078 3079 static int talitos_remove(struct platform_device *ofdev) 3080 { 3081 struct device *dev = &ofdev->dev; 3082 struct talitos_private *priv = dev_get_drvdata(dev); 3083 struct talitos_crypto_alg *t_alg, *n; 3084 int i; 3085 3086 list_for_each_entry_safe(t_alg, n, &priv->alg_list, entry) { 3087 switch (t_alg->algt.type) { 3088 case CRYPTO_ALG_TYPE_SKCIPHER: 3089 crypto_unregister_skcipher(&t_alg->algt.alg.skcipher); 3090 break; 3091 case CRYPTO_ALG_TYPE_AEAD: 3092 crypto_unregister_aead(&t_alg->algt.alg.aead); 3093 break; 3094 case CRYPTO_ALG_TYPE_AHASH: 3095 crypto_unregister_ahash(&t_alg->algt.alg.hash); 3096 break; 3097 } 3098 list_del(&t_alg->entry); 3099 } 3100 3101 if (hw_supports(dev, DESC_HDR_SEL0_RNG)) 3102 talitos_unregister_rng(dev); 3103 3104 for (i = 0; i < 2; i++) 3105 if (priv->irq[i]) { 3106 free_irq(priv->irq[i], dev); 3107 irq_dispose_mapping(priv->irq[i]); 3108 } 3109 3110 tasklet_kill(&priv->done_task[0]); 3111 if (priv->irq[1]) 3112 tasklet_kill(&priv->done_task[1]); 3113 3114 return 0; 3115 } 3116 3117 static struct talitos_crypto_alg *talitos_alg_alloc(struct device *dev, 3118 struct talitos_alg_template 3119 *template) 3120 { 3121 struct talitos_private *priv = dev_get_drvdata(dev); 3122 struct talitos_crypto_alg *t_alg; 3123 struct crypto_alg *alg; 3124 3125 t_alg = devm_kzalloc(dev, sizeof(struct talitos_crypto_alg), 3126 GFP_KERNEL); 3127 if (!t_alg) 3128 return ERR_PTR(-ENOMEM); 3129 3130 t_alg->algt = *template; 3131 3132 switch (t_alg->algt.type) { 3133 case CRYPTO_ALG_TYPE_SKCIPHER: 3134 alg = &t_alg->algt.alg.skcipher.base; 3135 alg->cra_exit = talitos_cra_exit; 3136 t_alg->algt.alg.skcipher.init = talitos_cra_init_skcipher; 3137 t_alg->algt.alg.skcipher.setkey = 3138 t_alg->algt.alg.skcipher.setkey ?: skcipher_setkey; 3139 t_alg->algt.alg.skcipher.encrypt = skcipher_encrypt; 3140 t_alg->algt.alg.skcipher.decrypt = skcipher_decrypt; 3141 break; 3142 case CRYPTO_ALG_TYPE_AEAD: 3143 alg = &t_alg->algt.alg.aead.base; 3144 alg->cra_exit = talitos_cra_exit; 3145 t_alg->algt.alg.aead.init = talitos_cra_init_aead; 3146 t_alg->algt.alg.aead.setkey = t_alg->algt.alg.aead.setkey ?: 3147 aead_setkey; 3148 t_alg->algt.alg.aead.encrypt = aead_encrypt; 3149 t_alg->algt.alg.aead.decrypt = aead_decrypt; 3150 if (!(priv->features & TALITOS_FTR_SHA224_HWINIT) && 3151 !strncmp(alg->cra_name, "authenc(hmac(sha224)", 20)) { 3152 devm_kfree(dev, t_alg); 3153 return ERR_PTR(-ENOTSUPP); 3154 } 3155 break; 3156 case CRYPTO_ALG_TYPE_AHASH: 3157 alg = &t_alg->algt.alg.hash.halg.base; 3158 alg->cra_init = talitos_cra_init_ahash; 3159 alg->cra_exit = talitos_cra_exit; 3160 t_alg->algt.alg.hash.init = ahash_init; 3161 t_alg->algt.alg.hash.update = ahash_update; 3162 t_alg->algt.alg.hash.final = ahash_final; 3163 t_alg->algt.alg.hash.finup = ahash_finup; 3164 t_alg->algt.alg.hash.digest = ahash_digest; 3165 if (!strncmp(alg->cra_name, "hmac", 4)) 3166 t_alg->algt.alg.hash.setkey = ahash_setkey; 3167 t_alg->algt.alg.hash.import = ahash_import; 3168 t_alg->algt.alg.hash.export = ahash_export; 3169 3170 if (!(priv->features & TALITOS_FTR_HMAC_OK) && 3171 !strncmp(alg->cra_name, "hmac", 4)) { 3172 devm_kfree(dev, t_alg); 3173 return ERR_PTR(-ENOTSUPP); 3174 } 3175 if (!(priv->features & TALITOS_FTR_SHA224_HWINIT) && 3176 (!strcmp(alg->cra_name, "sha224") || 3177 !strcmp(alg->cra_name, "hmac(sha224)"))) { 3178 t_alg->algt.alg.hash.init = ahash_init_sha224_swinit; 3179 t_alg->algt.desc_hdr_template = 3180 DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 3181 DESC_HDR_SEL0_MDEUA | 3182 DESC_HDR_MODE0_MDEU_SHA256; 3183 } 3184 break; 3185 default: 3186 dev_err(dev, "unknown algorithm type %d\n", t_alg->algt.type); 3187 devm_kfree(dev, t_alg); 3188 return ERR_PTR(-EINVAL); 3189 } 3190 3191 alg->cra_module = THIS_MODULE; 3192 if (t_alg->algt.priority) 3193 alg->cra_priority = t_alg->algt.priority; 3194 else 3195 alg->cra_priority = TALITOS_CRA_PRIORITY; 3196 if (has_ftr_sec1(priv)) 3197 alg->cra_alignmask = 3; 3198 else 3199 alg->cra_alignmask = 0; 3200 alg->cra_ctxsize = sizeof(struct talitos_ctx); 3201 alg->cra_flags |= CRYPTO_ALG_KERN_DRIVER_ONLY; 3202 3203 t_alg->dev = dev; 3204 3205 return t_alg; 3206 } 3207 3208 static int talitos_probe_irq(struct platform_device *ofdev) 3209 { 3210 struct device *dev = &ofdev->dev; 3211 struct device_node *np = ofdev->dev.of_node; 3212 struct talitos_private *priv = dev_get_drvdata(dev); 3213 int err; 3214 bool is_sec1 = has_ftr_sec1(priv); 3215 3216 priv->irq[0] = irq_of_parse_and_map(np, 0); 3217 if (!priv->irq[0]) { 3218 dev_err(dev, "failed to map irq\n"); 3219 return -EINVAL; 3220 } 3221 if (is_sec1) { 3222 err = request_irq(priv->irq[0], talitos1_interrupt_4ch, 0, 3223 dev_driver_string(dev), dev); 3224 goto primary_out; 3225 } 3226 3227 priv->irq[1] = irq_of_parse_and_map(np, 1); 3228 3229 /* get the primary irq line */ 3230 if (!priv->irq[1]) { 3231 err = request_irq(priv->irq[0], talitos2_interrupt_4ch, 0, 3232 dev_driver_string(dev), dev); 3233 goto primary_out; 3234 } 3235 3236 err = request_irq(priv->irq[0], talitos2_interrupt_ch0_2, 0, 3237 dev_driver_string(dev), dev); 3238 if (err) 3239 goto primary_out; 3240 3241 /* get the secondary irq line */ 3242 err = request_irq(priv->irq[1], talitos2_interrupt_ch1_3, 0, 3243 dev_driver_string(dev), dev); 3244 if (err) { 3245 dev_err(dev, "failed to request secondary irq\n"); 3246 irq_dispose_mapping(priv->irq[1]); 3247 priv->irq[1] = 0; 3248 } 3249 3250 return err; 3251 3252 primary_out: 3253 if (err) { 3254 dev_err(dev, "failed to request primary irq\n"); 3255 irq_dispose_mapping(priv->irq[0]); 3256 priv->irq[0] = 0; 3257 } 3258 3259 return err; 3260 } 3261 3262 static int talitos_probe(struct platform_device *ofdev) 3263 { 3264 struct device *dev = &ofdev->dev; 3265 struct device_node *np = ofdev->dev.of_node; 3266 struct talitos_private *priv; 3267 int i, err; 3268 int stride; 3269 struct resource *res; 3270 3271 priv = devm_kzalloc(dev, sizeof(struct talitos_private), GFP_KERNEL); 3272 if (!priv) 3273 return -ENOMEM; 3274 3275 INIT_LIST_HEAD(&priv->alg_list); 3276 3277 dev_set_drvdata(dev, priv); 3278 3279 priv->ofdev = ofdev; 3280 3281 spin_lock_init(&priv->reg_lock); 3282 3283 res = platform_get_resource(ofdev, IORESOURCE_MEM, 0); 3284 if (!res) 3285 return -ENXIO; 3286 priv->reg = devm_ioremap(dev, res->start, resource_size(res)); 3287 if (!priv->reg) { 3288 dev_err(dev, "failed to of_iomap\n"); 3289 err = -ENOMEM; 3290 goto err_out; 3291 } 3292 3293 /* get SEC version capabilities from device tree */ 3294 of_property_read_u32(np, "fsl,num-channels", &priv->num_channels); 3295 of_property_read_u32(np, "fsl,channel-fifo-len", &priv->chfifo_len); 3296 of_property_read_u32(np, "fsl,exec-units-mask", &priv->exec_units); 3297 of_property_read_u32(np, "fsl,descriptor-types-mask", 3298 &priv->desc_types); 3299 3300 if (!is_power_of_2(priv->num_channels) || !priv->chfifo_len || 3301 !priv->exec_units || !priv->desc_types) { 3302 dev_err(dev, "invalid property data in device tree node\n"); 3303 err = -EINVAL; 3304 goto err_out; 3305 } 3306 3307 if (of_device_is_compatible(np, "fsl,sec3.0")) 3308 priv->features |= TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT; 3309 3310 if (of_device_is_compatible(np, "fsl,sec2.1")) 3311 priv->features |= TALITOS_FTR_HW_AUTH_CHECK | 3312 TALITOS_FTR_SHA224_HWINIT | 3313 TALITOS_FTR_HMAC_OK; 3314 3315 if (of_device_is_compatible(np, "fsl,sec1.0")) 3316 priv->features |= TALITOS_FTR_SEC1; 3317 3318 if (of_device_is_compatible(np, "fsl,sec1.2")) { 3319 priv->reg_deu = priv->reg + TALITOS12_DEU; 3320 priv->reg_aesu = priv->reg + TALITOS12_AESU; 3321 priv->reg_mdeu = priv->reg + TALITOS12_MDEU; 3322 stride = TALITOS1_CH_STRIDE; 3323 } else if (of_device_is_compatible(np, "fsl,sec1.0")) { 3324 priv->reg_deu = priv->reg + TALITOS10_DEU; 3325 priv->reg_aesu = priv->reg + TALITOS10_AESU; 3326 priv->reg_mdeu = priv->reg + TALITOS10_MDEU; 3327 priv->reg_afeu = priv->reg + TALITOS10_AFEU; 3328 priv->reg_rngu = priv->reg + TALITOS10_RNGU; 3329 priv->reg_pkeu = priv->reg + TALITOS10_PKEU; 3330 stride = TALITOS1_CH_STRIDE; 3331 } else { 3332 priv->reg_deu = priv->reg + TALITOS2_DEU; 3333 priv->reg_aesu = priv->reg + TALITOS2_AESU; 3334 priv->reg_mdeu = priv->reg + TALITOS2_MDEU; 3335 priv->reg_afeu = priv->reg + TALITOS2_AFEU; 3336 priv->reg_rngu = priv->reg + TALITOS2_RNGU; 3337 priv->reg_pkeu = priv->reg + TALITOS2_PKEU; 3338 priv->reg_keu = priv->reg + TALITOS2_KEU; 3339 priv->reg_crcu = priv->reg + TALITOS2_CRCU; 3340 stride = TALITOS2_CH_STRIDE; 3341 } 3342 3343 err = talitos_probe_irq(ofdev); 3344 if (err) 3345 goto err_out; 3346 3347 if (has_ftr_sec1(priv)) { 3348 if (priv->num_channels == 1) 3349 tasklet_init(&priv->done_task[0], talitos1_done_ch0, 3350 (unsigned long)dev); 3351 else 3352 tasklet_init(&priv->done_task[0], talitos1_done_4ch, 3353 (unsigned long)dev); 3354 } else { 3355 if (priv->irq[1]) { 3356 tasklet_init(&priv->done_task[0], talitos2_done_ch0_2, 3357 (unsigned long)dev); 3358 tasklet_init(&priv->done_task[1], talitos2_done_ch1_3, 3359 (unsigned long)dev); 3360 } else if (priv->num_channels == 1) { 3361 tasklet_init(&priv->done_task[0], talitos2_done_ch0, 3362 (unsigned long)dev); 3363 } else { 3364 tasklet_init(&priv->done_task[0], talitos2_done_4ch, 3365 (unsigned long)dev); 3366 } 3367 } 3368 3369 priv->chan = devm_kcalloc(dev, 3370 priv->num_channels, 3371 sizeof(struct talitos_channel), 3372 GFP_KERNEL); 3373 if (!priv->chan) { 3374 dev_err(dev, "failed to allocate channel management space\n"); 3375 err = -ENOMEM; 3376 goto err_out; 3377 } 3378 3379 priv->fifo_len = roundup_pow_of_two(priv->chfifo_len); 3380 3381 for (i = 0; i < priv->num_channels; i++) { 3382 priv->chan[i].reg = priv->reg + stride * (i + 1); 3383 if (!priv->irq[1] || !(i & 1)) 3384 priv->chan[i].reg += TALITOS_CH_BASE_OFFSET; 3385 3386 spin_lock_init(&priv->chan[i].head_lock); 3387 spin_lock_init(&priv->chan[i].tail_lock); 3388 3389 priv->chan[i].fifo = devm_kcalloc(dev, 3390 priv->fifo_len, 3391 sizeof(struct talitos_request), 3392 GFP_KERNEL); 3393 if (!priv->chan[i].fifo) { 3394 dev_err(dev, "failed to allocate request fifo %d\n", i); 3395 err = -ENOMEM; 3396 goto err_out; 3397 } 3398 3399 atomic_set(&priv->chan[i].submit_count, 3400 -(priv->chfifo_len - 1)); 3401 } 3402 3403 dma_set_mask(dev, DMA_BIT_MASK(36)); 3404 3405 /* reset and initialize the h/w */ 3406 err = init_device(dev); 3407 if (err) { 3408 dev_err(dev, "failed to initialize device\n"); 3409 goto err_out; 3410 } 3411 3412 /* register the RNG, if available */ 3413 if (hw_supports(dev, DESC_HDR_SEL0_RNG)) { 3414 err = talitos_register_rng(dev); 3415 if (err) { 3416 dev_err(dev, "failed to register hwrng: %d\n", err); 3417 goto err_out; 3418 } else 3419 dev_info(dev, "hwrng\n"); 3420 } 3421 3422 /* register crypto algorithms the device supports */ 3423 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) { 3424 if (hw_supports(dev, driver_algs[i].desc_hdr_template)) { 3425 struct talitos_crypto_alg *t_alg; 3426 struct crypto_alg *alg = NULL; 3427 3428 t_alg = talitos_alg_alloc(dev, &driver_algs[i]); 3429 if (IS_ERR(t_alg)) { 3430 err = PTR_ERR(t_alg); 3431 if (err == -ENOTSUPP) 3432 continue; 3433 goto err_out; 3434 } 3435 3436 switch (t_alg->algt.type) { 3437 case CRYPTO_ALG_TYPE_SKCIPHER: 3438 err = crypto_register_skcipher( 3439 &t_alg->algt.alg.skcipher); 3440 alg = &t_alg->algt.alg.skcipher.base; 3441 break; 3442 3443 case CRYPTO_ALG_TYPE_AEAD: 3444 err = crypto_register_aead( 3445 &t_alg->algt.alg.aead); 3446 alg = &t_alg->algt.alg.aead.base; 3447 break; 3448 3449 case CRYPTO_ALG_TYPE_AHASH: 3450 err = crypto_register_ahash( 3451 &t_alg->algt.alg.hash); 3452 alg = &t_alg->algt.alg.hash.halg.base; 3453 break; 3454 } 3455 if (err) { 3456 dev_err(dev, "%s alg registration failed\n", 3457 alg->cra_driver_name); 3458 devm_kfree(dev, t_alg); 3459 } else 3460 list_add_tail(&t_alg->entry, &priv->alg_list); 3461 } 3462 } 3463 if (!list_empty(&priv->alg_list)) 3464 dev_info(dev, "%s algorithms registered in /proc/crypto\n", 3465 (char *)of_get_property(np, "compatible", NULL)); 3466 3467 return 0; 3468 3469 err_out: 3470 talitos_remove(ofdev); 3471 3472 return err; 3473 } 3474 3475 static const struct of_device_id talitos_match[] = { 3476 #ifdef CONFIG_CRYPTO_DEV_TALITOS1 3477 { 3478 .compatible = "fsl,sec1.0", 3479 }, 3480 #endif 3481 #ifdef CONFIG_CRYPTO_DEV_TALITOS2 3482 { 3483 .compatible = "fsl,sec2.0", 3484 }, 3485 #endif 3486 {}, 3487 }; 3488 MODULE_DEVICE_TABLE(of, talitos_match); 3489 3490 static struct platform_driver talitos_driver = { 3491 .driver = { 3492 .name = "talitos", 3493 .of_match_table = talitos_match, 3494 }, 3495 .probe = talitos_probe, 3496 .remove = talitos_remove, 3497 }; 3498 3499 module_platform_driver(talitos_driver); 3500 3501 MODULE_LICENSE("GPL"); 3502 MODULE_AUTHOR("Kim Phillips <kim.phillips@freescale.com>"); 3503 MODULE_DESCRIPTION("Freescale integrated security engine (SEC) driver"); 3504