1 /* 2 * talitos - Freescale Integrated Security Engine (SEC) device driver 3 * 4 * Copyright (c) 2008-2011 Freescale Semiconductor, Inc. 5 * 6 * Scatterlist Crypto API glue code copied from files with the following: 7 * Copyright (c) 2006-2007 Herbert Xu <herbert@gondor.apana.org.au> 8 * 9 * Crypto algorithm registration code copied from hifn driver: 10 * 2007+ Copyright (c) Evgeniy Polyakov <johnpol@2ka.mipt.ru> 11 * All rights reserved. 12 * 13 * This program is free software; you can redistribute it and/or modify 14 * it under the terms of the GNU General Public License as published by 15 * the Free Software Foundation; either version 2 of the License, or 16 * (at your option) any later version. 17 * 18 * This program is distributed in the hope that it will be useful, 19 * but WITHOUT ANY WARRANTY; without even the implied warranty of 20 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 21 * GNU General Public License for more details. 22 * 23 * You should have received a copy of the GNU General Public License 24 * along with this program; if not, write to the Free Software 25 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 26 */ 27 28 #include <linux/kernel.h> 29 #include <linux/module.h> 30 #include <linux/mod_devicetable.h> 31 #include <linux/device.h> 32 #include <linux/interrupt.h> 33 #include <linux/crypto.h> 34 #include <linux/hw_random.h> 35 #include <linux/of_address.h> 36 #include <linux/of_irq.h> 37 #include <linux/of_platform.h> 38 #include <linux/dma-mapping.h> 39 #include <linux/io.h> 40 #include <linux/spinlock.h> 41 #include <linux/rtnetlink.h> 42 #include <linux/slab.h> 43 44 #include <crypto/algapi.h> 45 #include <crypto/aes.h> 46 #include <crypto/des.h> 47 #include <crypto/sha.h> 48 #include <crypto/md5.h> 49 #include <crypto/internal/aead.h> 50 #include <crypto/authenc.h> 51 #include <crypto/skcipher.h> 52 #include <crypto/hash.h> 53 #include <crypto/internal/hash.h> 54 #include <crypto/scatterwalk.h> 55 56 #include "talitos.h" 57 58 static void to_talitos_ptr(struct talitos_ptr *ptr, dma_addr_t dma_addr, 59 bool is_sec1) 60 { 61 ptr->ptr = cpu_to_be32(lower_32_bits(dma_addr)); 62 if (!is_sec1) 63 ptr->eptr = upper_32_bits(dma_addr); 64 } 65 66 static void copy_talitos_ptr(struct talitos_ptr *dst_ptr, 67 struct talitos_ptr *src_ptr, bool is_sec1) 68 { 69 dst_ptr->ptr = src_ptr->ptr; 70 if (!is_sec1) 71 dst_ptr->eptr = src_ptr->eptr; 72 } 73 74 static void to_talitos_ptr_len(struct talitos_ptr *ptr, unsigned int len, 75 bool is_sec1) 76 { 77 if (is_sec1) { 78 ptr->res = 0; 79 ptr->len1 = cpu_to_be16(len); 80 } else { 81 ptr->len = cpu_to_be16(len); 82 } 83 } 84 85 static unsigned short from_talitos_ptr_len(struct talitos_ptr *ptr, 86 bool is_sec1) 87 { 88 if (is_sec1) 89 return be16_to_cpu(ptr->len1); 90 else 91 return be16_to_cpu(ptr->len); 92 } 93 94 static void to_talitos_ptr_extent_clear(struct talitos_ptr *ptr, bool is_sec1) 95 { 96 if (!is_sec1) 97 ptr->j_extent = 0; 98 } 99 100 /* 101 * map virtual single (contiguous) pointer to h/w descriptor pointer 102 */ 103 static void map_single_talitos_ptr(struct device *dev, 104 struct talitos_ptr *ptr, 105 unsigned int len, void *data, 106 enum dma_data_direction dir) 107 { 108 dma_addr_t dma_addr = dma_map_single(dev, data, len, dir); 109 struct talitos_private *priv = dev_get_drvdata(dev); 110 bool is_sec1 = has_ftr_sec1(priv); 111 112 to_talitos_ptr_len(ptr, len, is_sec1); 113 to_talitos_ptr(ptr, dma_addr, is_sec1); 114 to_talitos_ptr_extent_clear(ptr, is_sec1); 115 } 116 117 /* 118 * unmap bus single (contiguous) h/w descriptor pointer 119 */ 120 static void unmap_single_talitos_ptr(struct device *dev, 121 struct talitos_ptr *ptr, 122 enum dma_data_direction dir) 123 { 124 struct talitos_private *priv = dev_get_drvdata(dev); 125 bool is_sec1 = has_ftr_sec1(priv); 126 127 dma_unmap_single(dev, be32_to_cpu(ptr->ptr), 128 from_talitos_ptr_len(ptr, is_sec1), dir); 129 } 130 131 static int reset_channel(struct device *dev, int ch) 132 { 133 struct talitos_private *priv = dev_get_drvdata(dev); 134 unsigned int timeout = TALITOS_TIMEOUT; 135 bool is_sec1 = has_ftr_sec1(priv); 136 137 if (is_sec1) { 138 setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO, 139 TALITOS1_CCCR_LO_RESET); 140 141 while ((in_be32(priv->chan[ch].reg + TALITOS_CCCR_LO) & 142 TALITOS1_CCCR_LO_RESET) && --timeout) 143 cpu_relax(); 144 } else { 145 setbits32(priv->chan[ch].reg + TALITOS_CCCR, 146 TALITOS2_CCCR_RESET); 147 148 while ((in_be32(priv->chan[ch].reg + TALITOS_CCCR) & 149 TALITOS2_CCCR_RESET) && --timeout) 150 cpu_relax(); 151 } 152 153 if (timeout == 0) { 154 dev_err(dev, "failed to reset channel %d\n", ch); 155 return -EIO; 156 } 157 158 /* set 36-bit addressing, done writeback enable and done IRQ enable */ 159 setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO, TALITOS_CCCR_LO_EAE | 160 TALITOS_CCCR_LO_CDWE | TALITOS_CCCR_LO_CDIE); 161 162 /* and ICCR writeback, if available */ 163 if (priv->features & TALITOS_FTR_HW_AUTH_CHECK) 164 setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO, 165 TALITOS_CCCR_LO_IWSE); 166 167 return 0; 168 } 169 170 static int reset_device(struct device *dev) 171 { 172 struct talitos_private *priv = dev_get_drvdata(dev); 173 unsigned int timeout = TALITOS_TIMEOUT; 174 bool is_sec1 = has_ftr_sec1(priv); 175 u32 mcr = is_sec1 ? TALITOS1_MCR_SWR : TALITOS2_MCR_SWR; 176 177 setbits32(priv->reg + TALITOS_MCR, mcr); 178 179 while ((in_be32(priv->reg + TALITOS_MCR) & mcr) 180 && --timeout) 181 cpu_relax(); 182 183 if (priv->irq[1]) { 184 mcr = TALITOS_MCR_RCA1 | TALITOS_MCR_RCA3; 185 setbits32(priv->reg + TALITOS_MCR, mcr); 186 } 187 188 if (timeout == 0) { 189 dev_err(dev, "failed to reset device\n"); 190 return -EIO; 191 } 192 193 return 0; 194 } 195 196 /* 197 * Reset and initialize the device 198 */ 199 static int init_device(struct device *dev) 200 { 201 struct talitos_private *priv = dev_get_drvdata(dev); 202 int ch, err; 203 bool is_sec1 = has_ftr_sec1(priv); 204 205 /* 206 * Master reset 207 * errata documentation: warning: certain SEC interrupts 208 * are not fully cleared by writing the MCR:SWR bit, 209 * set bit twice to completely reset 210 */ 211 err = reset_device(dev); 212 if (err) 213 return err; 214 215 err = reset_device(dev); 216 if (err) 217 return err; 218 219 /* reset channels */ 220 for (ch = 0; ch < priv->num_channels; ch++) { 221 err = reset_channel(dev, ch); 222 if (err) 223 return err; 224 } 225 226 /* enable channel done and error interrupts */ 227 if (is_sec1) { 228 clrbits32(priv->reg + TALITOS_IMR, TALITOS1_IMR_INIT); 229 clrbits32(priv->reg + TALITOS_IMR_LO, TALITOS1_IMR_LO_INIT); 230 /* disable parity error check in DEU (erroneous? test vect.) */ 231 setbits32(priv->reg_deu + TALITOS_EUICR, TALITOS1_DEUICR_KPE); 232 } else { 233 setbits32(priv->reg + TALITOS_IMR, TALITOS2_IMR_INIT); 234 setbits32(priv->reg + TALITOS_IMR_LO, TALITOS2_IMR_LO_INIT); 235 } 236 237 /* disable integrity check error interrupts (use writeback instead) */ 238 if (priv->features & TALITOS_FTR_HW_AUTH_CHECK) 239 setbits32(priv->reg_mdeu + TALITOS_EUICR_LO, 240 TALITOS_MDEUICR_LO_ICE); 241 242 return 0; 243 } 244 245 /** 246 * talitos_submit - submits a descriptor to the device for processing 247 * @dev: the SEC device to be used 248 * @ch: the SEC device channel to be used 249 * @desc: the descriptor to be processed by the device 250 * @callback: whom to call when processing is complete 251 * @context: a handle for use by caller (optional) 252 * 253 * desc must contain valid dma-mapped (bus physical) address pointers. 254 * callback must check err and feedback in descriptor header 255 * for device processing status. 256 */ 257 int talitos_submit(struct device *dev, int ch, struct talitos_desc *desc, 258 void (*callback)(struct device *dev, 259 struct talitos_desc *desc, 260 void *context, int error), 261 void *context) 262 { 263 struct talitos_private *priv = dev_get_drvdata(dev); 264 struct talitos_request *request; 265 unsigned long flags; 266 int head; 267 bool is_sec1 = has_ftr_sec1(priv); 268 269 spin_lock_irqsave(&priv->chan[ch].head_lock, flags); 270 271 if (!atomic_inc_not_zero(&priv->chan[ch].submit_count)) { 272 /* h/w fifo is full */ 273 spin_unlock_irqrestore(&priv->chan[ch].head_lock, flags); 274 return -EAGAIN; 275 } 276 277 head = priv->chan[ch].head; 278 request = &priv->chan[ch].fifo[head]; 279 280 /* map descriptor and save caller data */ 281 if (is_sec1) { 282 desc->hdr1 = desc->hdr; 283 desc->next_desc = 0; 284 request->dma_desc = dma_map_single(dev, &desc->hdr1, 285 TALITOS_DESC_SIZE, 286 DMA_BIDIRECTIONAL); 287 } else { 288 request->dma_desc = dma_map_single(dev, desc, 289 TALITOS_DESC_SIZE, 290 DMA_BIDIRECTIONAL); 291 } 292 request->callback = callback; 293 request->context = context; 294 295 /* increment fifo head */ 296 priv->chan[ch].head = (priv->chan[ch].head + 1) & (priv->fifo_len - 1); 297 298 smp_wmb(); 299 request->desc = desc; 300 301 /* GO! */ 302 wmb(); 303 out_be32(priv->chan[ch].reg + TALITOS_FF, 304 upper_32_bits(request->dma_desc)); 305 out_be32(priv->chan[ch].reg + TALITOS_FF_LO, 306 lower_32_bits(request->dma_desc)); 307 308 spin_unlock_irqrestore(&priv->chan[ch].head_lock, flags); 309 310 return -EINPROGRESS; 311 } 312 EXPORT_SYMBOL(talitos_submit); 313 314 /* 315 * process what was done, notify callback of error if not 316 */ 317 static void flush_channel(struct device *dev, int ch, int error, int reset_ch) 318 { 319 struct talitos_private *priv = dev_get_drvdata(dev); 320 struct talitos_request *request, saved_req; 321 unsigned long flags; 322 int tail, status; 323 bool is_sec1 = has_ftr_sec1(priv); 324 325 spin_lock_irqsave(&priv->chan[ch].tail_lock, flags); 326 327 tail = priv->chan[ch].tail; 328 while (priv->chan[ch].fifo[tail].desc) { 329 __be32 hdr; 330 331 request = &priv->chan[ch].fifo[tail]; 332 333 /* descriptors with their done bits set don't get the error */ 334 rmb(); 335 hdr = is_sec1 ? request->desc->hdr1 : request->desc->hdr; 336 337 if ((hdr & DESC_HDR_DONE) == DESC_HDR_DONE) 338 status = 0; 339 else 340 if (!error) 341 break; 342 else 343 status = error; 344 345 dma_unmap_single(dev, request->dma_desc, 346 TALITOS_DESC_SIZE, 347 DMA_BIDIRECTIONAL); 348 349 /* copy entries so we can call callback outside lock */ 350 saved_req.desc = request->desc; 351 saved_req.callback = request->callback; 352 saved_req.context = request->context; 353 354 /* release request entry in fifo */ 355 smp_wmb(); 356 request->desc = NULL; 357 358 /* increment fifo tail */ 359 priv->chan[ch].tail = (tail + 1) & (priv->fifo_len - 1); 360 361 spin_unlock_irqrestore(&priv->chan[ch].tail_lock, flags); 362 363 atomic_dec(&priv->chan[ch].submit_count); 364 365 saved_req.callback(dev, saved_req.desc, saved_req.context, 366 status); 367 /* channel may resume processing in single desc error case */ 368 if (error && !reset_ch && status == error) 369 return; 370 spin_lock_irqsave(&priv->chan[ch].tail_lock, flags); 371 tail = priv->chan[ch].tail; 372 } 373 374 spin_unlock_irqrestore(&priv->chan[ch].tail_lock, flags); 375 } 376 377 /* 378 * process completed requests for channels that have done status 379 */ 380 #define DEF_TALITOS1_DONE(name, ch_done_mask) \ 381 static void talitos1_done_##name(unsigned long data) \ 382 { \ 383 struct device *dev = (struct device *)data; \ 384 struct talitos_private *priv = dev_get_drvdata(dev); \ 385 unsigned long flags; \ 386 \ 387 if (ch_done_mask & 0x10000000) \ 388 flush_channel(dev, 0, 0, 0); \ 389 if (priv->num_channels == 1) \ 390 goto out; \ 391 if (ch_done_mask & 0x40000000) \ 392 flush_channel(dev, 1, 0, 0); \ 393 if (ch_done_mask & 0x00010000) \ 394 flush_channel(dev, 2, 0, 0); \ 395 if (ch_done_mask & 0x00040000) \ 396 flush_channel(dev, 3, 0, 0); \ 397 \ 398 out: \ 399 /* At this point, all completed channels have been processed */ \ 400 /* Unmask done interrupts for channels completed later on. */ \ 401 spin_lock_irqsave(&priv->reg_lock, flags); \ 402 clrbits32(priv->reg + TALITOS_IMR, ch_done_mask); \ 403 clrbits32(priv->reg + TALITOS_IMR_LO, TALITOS1_IMR_LO_INIT); \ 404 spin_unlock_irqrestore(&priv->reg_lock, flags); \ 405 } 406 407 DEF_TALITOS1_DONE(4ch, TALITOS1_ISR_4CHDONE) 408 409 #define DEF_TALITOS2_DONE(name, ch_done_mask) \ 410 static void talitos2_done_##name(unsigned long data) \ 411 { \ 412 struct device *dev = (struct device *)data; \ 413 struct talitos_private *priv = dev_get_drvdata(dev); \ 414 unsigned long flags; \ 415 \ 416 if (ch_done_mask & 1) \ 417 flush_channel(dev, 0, 0, 0); \ 418 if (priv->num_channels == 1) \ 419 goto out; \ 420 if (ch_done_mask & (1 << 2)) \ 421 flush_channel(dev, 1, 0, 0); \ 422 if (ch_done_mask & (1 << 4)) \ 423 flush_channel(dev, 2, 0, 0); \ 424 if (ch_done_mask & (1 << 6)) \ 425 flush_channel(dev, 3, 0, 0); \ 426 \ 427 out: \ 428 /* At this point, all completed channels have been processed */ \ 429 /* Unmask done interrupts for channels completed later on. */ \ 430 spin_lock_irqsave(&priv->reg_lock, flags); \ 431 setbits32(priv->reg + TALITOS_IMR, ch_done_mask); \ 432 setbits32(priv->reg + TALITOS_IMR_LO, TALITOS2_IMR_LO_INIT); \ 433 spin_unlock_irqrestore(&priv->reg_lock, flags); \ 434 } 435 436 DEF_TALITOS2_DONE(4ch, TALITOS2_ISR_4CHDONE) 437 DEF_TALITOS2_DONE(ch0_2, TALITOS2_ISR_CH_0_2_DONE) 438 DEF_TALITOS2_DONE(ch1_3, TALITOS2_ISR_CH_1_3_DONE) 439 440 /* 441 * locate current (offending) descriptor 442 */ 443 static u32 current_desc_hdr(struct device *dev, int ch) 444 { 445 struct talitos_private *priv = dev_get_drvdata(dev); 446 int tail, iter; 447 dma_addr_t cur_desc; 448 449 cur_desc = ((u64)in_be32(priv->chan[ch].reg + TALITOS_CDPR)) << 32; 450 cur_desc |= in_be32(priv->chan[ch].reg + TALITOS_CDPR_LO); 451 452 if (!cur_desc) { 453 dev_err(dev, "CDPR is NULL, giving up search for offending descriptor\n"); 454 return 0; 455 } 456 457 tail = priv->chan[ch].tail; 458 459 iter = tail; 460 while (priv->chan[ch].fifo[iter].dma_desc != cur_desc) { 461 iter = (iter + 1) & (priv->fifo_len - 1); 462 if (iter == tail) { 463 dev_err(dev, "couldn't locate current descriptor\n"); 464 return 0; 465 } 466 } 467 468 return priv->chan[ch].fifo[iter].desc->hdr; 469 } 470 471 /* 472 * user diagnostics; report root cause of error based on execution unit status 473 */ 474 static void report_eu_error(struct device *dev, int ch, u32 desc_hdr) 475 { 476 struct talitos_private *priv = dev_get_drvdata(dev); 477 int i; 478 479 if (!desc_hdr) 480 desc_hdr = in_be32(priv->chan[ch].reg + TALITOS_DESCBUF); 481 482 switch (desc_hdr & DESC_HDR_SEL0_MASK) { 483 case DESC_HDR_SEL0_AFEU: 484 dev_err(dev, "AFEUISR 0x%08x_%08x\n", 485 in_be32(priv->reg_afeu + TALITOS_EUISR), 486 in_be32(priv->reg_afeu + TALITOS_EUISR_LO)); 487 break; 488 case DESC_HDR_SEL0_DEU: 489 dev_err(dev, "DEUISR 0x%08x_%08x\n", 490 in_be32(priv->reg_deu + TALITOS_EUISR), 491 in_be32(priv->reg_deu + TALITOS_EUISR_LO)); 492 break; 493 case DESC_HDR_SEL0_MDEUA: 494 case DESC_HDR_SEL0_MDEUB: 495 dev_err(dev, "MDEUISR 0x%08x_%08x\n", 496 in_be32(priv->reg_mdeu + TALITOS_EUISR), 497 in_be32(priv->reg_mdeu + TALITOS_EUISR_LO)); 498 break; 499 case DESC_HDR_SEL0_RNG: 500 dev_err(dev, "RNGUISR 0x%08x_%08x\n", 501 in_be32(priv->reg_rngu + TALITOS_ISR), 502 in_be32(priv->reg_rngu + TALITOS_ISR_LO)); 503 break; 504 case DESC_HDR_SEL0_PKEU: 505 dev_err(dev, "PKEUISR 0x%08x_%08x\n", 506 in_be32(priv->reg_pkeu + TALITOS_EUISR), 507 in_be32(priv->reg_pkeu + TALITOS_EUISR_LO)); 508 break; 509 case DESC_HDR_SEL0_AESU: 510 dev_err(dev, "AESUISR 0x%08x_%08x\n", 511 in_be32(priv->reg_aesu + TALITOS_EUISR), 512 in_be32(priv->reg_aesu + TALITOS_EUISR_LO)); 513 break; 514 case DESC_HDR_SEL0_CRCU: 515 dev_err(dev, "CRCUISR 0x%08x_%08x\n", 516 in_be32(priv->reg_crcu + TALITOS_EUISR), 517 in_be32(priv->reg_crcu + TALITOS_EUISR_LO)); 518 break; 519 case DESC_HDR_SEL0_KEU: 520 dev_err(dev, "KEUISR 0x%08x_%08x\n", 521 in_be32(priv->reg_pkeu + TALITOS_EUISR), 522 in_be32(priv->reg_pkeu + TALITOS_EUISR_LO)); 523 break; 524 } 525 526 switch (desc_hdr & DESC_HDR_SEL1_MASK) { 527 case DESC_HDR_SEL1_MDEUA: 528 case DESC_HDR_SEL1_MDEUB: 529 dev_err(dev, "MDEUISR 0x%08x_%08x\n", 530 in_be32(priv->reg_mdeu + TALITOS_EUISR), 531 in_be32(priv->reg_mdeu + TALITOS_EUISR_LO)); 532 break; 533 case DESC_HDR_SEL1_CRCU: 534 dev_err(dev, "CRCUISR 0x%08x_%08x\n", 535 in_be32(priv->reg_crcu + TALITOS_EUISR), 536 in_be32(priv->reg_crcu + TALITOS_EUISR_LO)); 537 break; 538 } 539 540 for (i = 0; i < 8; i++) 541 dev_err(dev, "DESCBUF 0x%08x_%08x\n", 542 in_be32(priv->chan[ch].reg + TALITOS_DESCBUF + 8*i), 543 in_be32(priv->chan[ch].reg + TALITOS_DESCBUF_LO + 8*i)); 544 } 545 546 /* 547 * recover from error interrupts 548 */ 549 static void talitos_error(struct device *dev, u32 isr, u32 isr_lo) 550 { 551 struct talitos_private *priv = dev_get_drvdata(dev); 552 unsigned int timeout = TALITOS_TIMEOUT; 553 int ch, error, reset_dev = 0; 554 u32 v_lo; 555 bool is_sec1 = has_ftr_sec1(priv); 556 int reset_ch = is_sec1 ? 1 : 0; /* only SEC2 supports continuation */ 557 558 for (ch = 0; ch < priv->num_channels; ch++) { 559 /* skip channels without errors */ 560 if (is_sec1) { 561 /* bits 29, 31, 17, 19 */ 562 if (!(isr & (1 << (29 + (ch & 1) * 2 - (ch & 2) * 6)))) 563 continue; 564 } else { 565 if (!(isr & (1 << (ch * 2 + 1)))) 566 continue; 567 } 568 569 error = -EINVAL; 570 571 v_lo = in_be32(priv->chan[ch].reg + TALITOS_CCPSR_LO); 572 573 if (v_lo & TALITOS_CCPSR_LO_DOF) { 574 dev_err(dev, "double fetch fifo overflow error\n"); 575 error = -EAGAIN; 576 reset_ch = 1; 577 } 578 if (v_lo & TALITOS_CCPSR_LO_SOF) { 579 /* h/w dropped descriptor */ 580 dev_err(dev, "single fetch fifo overflow error\n"); 581 error = -EAGAIN; 582 } 583 if (v_lo & TALITOS_CCPSR_LO_MDTE) 584 dev_err(dev, "master data transfer error\n"); 585 if (v_lo & TALITOS_CCPSR_LO_SGDLZ) 586 dev_err(dev, is_sec1 ? "pointeur not complete error\n" 587 : "s/g data length zero error\n"); 588 if (v_lo & TALITOS_CCPSR_LO_FPZ) 589 dev_err(dev, is_sec1 ? "parity error\n" 590 : "fetch pointer zero error\n"); 591 if (v_lo & TALITOS_CCPSR_LO_IDH) 592 dev_err(dev, "illegal descriptor header error\n"); 593 if (v_lo & TALITOS_CCPSR_LO_IEU) 594 dev_err(dev, is_sec1 ? "static assignment error\n" 595 : "invalid exec unit error\n"); 596 if (v_lo & TALITOS_CCPSR_LO_EU) 597 report_eu_error(dev, ch, current_desc_hdr(dev, ch)); 598 if (!is_sec1) { 599 if (v_lo & TALITOS_CCPSR_LO_GB) 600 dev_err(dev, "gather boundary error\n"); 601 if (v_lo & TALITOS_CCPSR_LO_GRL) 602 dev_err(dev, "gather return/length error\n"); 603 if (v_lo & TALITOS_CCPSR_LO_SB) 604 dev_err(dev, "scatter boundary error\n"); 605 if (v_lo & TALITOS_CCPSR_LO_SRL) 606 dev_err(dev, "scatter return/length error\n"); 607 } 608 609 flush_channel(dev, ch, error, reset_ch); 610 611 if (reset_ch) { 612 reset_channel(dev, ch); 613 } else { 614 setbits32(priv->chan[ch].reg + TALITOS_CCCR, 615 TALITOS2_CCCR_CONT); 616 setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO, 0); 617 while ((in_be32(priv->chan[ch].reg + TALITOS_CCCR) & 618 TALITOS2_CCCR_CONT) && --timeout) 619 cpu_relax(); 620 if (timeout == 0) { 621 dev_err(dev, "failed to restart channel %d\n", 622 ch); 623 reset_dev = 1; 624 } 625 } 626 } 627 if (reset_dev || (is_sec1 && isr & ~TALITOS1_ISR_4CHERR) || 628 (!is_sec1 && isr & ~TALITOS2_ISR_4CHERR) || isr_lo) { 629 if (is_sec1 && (isr_lo & TALITOS1_ISR_TEA_ERR)) 630 dev_err(dev, "TEA error: ISR 0x%08x_%08x\n", 631 isr, isr_lo); 632 else 633 dev_err(dev, "done overflow, internal time out, or " 634 "rngu error: ISR 0x%08x_%08x\n", isr, isr_lo); 635 636 /* purge request queues */ 637 for (ch = 0; ch < priv->num_channels; ch++) 638 flush_channel(dev, ch, -EIO, 1); 639 640 /* reset and reinitialize the device */ 641 init_device(dev); 642 } 643 } 644 645 #define DEF_TALITOS1_INTERRUPT(name, ch_done_mask, ch_err_mask, tlet) \ 646 static irqreturn_t talitos1_interrupt_##name(int irq, void *data) \ 647 { \ 648 struct device *dev = data; \ 649 struct talitos_private *priv = dev_get_drvdata(dev); \ 650 u32 isr, isr_lo; \ 651 unsigned long flags; \ 652 \ 653 spin_lock_irqsave(&priv->reg_lock, flags); \ 654 isr = in_be32(priv->reg + TALITOS_ISR); \ 655 isr_lo = in_be32(priv->reg + TALITOS_ISR_LO); \ 656 /* Acknowledge interrupt */ \ 657 out_be32(priv->reg + TALITOS_ICR, isr & (ch_done_mask | ch_err_mask)); \ 658 out_be32(priv->reg + TALITOS_ICR_LO, isr_lo); \ 659 \ 660 if (unlikely(isr & ch_err_mask || isr_lo & TALITOS1_IMR_LO_INIT)) { \ 661 spin_unlock_irqrestore(&priv->reg_lock, flags); \ 662 talitos_error(dev, isr & ch_err_mask, isr_lo); \ 663 } \ 664 else { \ 665 if (likely(isr & ch_done_mask)) { \ 666 /* mask further done interrupts. */ \ 667 setbits32(priv->reg + TALITOS_IMR, ch_done_mask); \ 668 /* done_task will unmask done interrupts at exit */ \ 669 tasklet_schedule(&priv->done_task[tlet]); \ 670 } \ 671 spin_unlock_irqrestore(&priv->reg_lock, flags); \ 672 } \ 673 \ 674 return (isr & (ch_done_mask | ch_err_mask) || isr_lo) ? IRQ_HANDLED : \ 675 IRQ_NONE; \ 676 } 677 678 DEF_TALITOS1_INTERRUPT(4ch, TALITOS1_ISR_4CHDONE, TALITOS1_ISR_4CHERR, 0) 679 680 #define DEF_TALITOS2_INTERRUPT(name, ch_done_mask, ch_err_mask, tlet) \ 681 static irqreturn_t talitos2_interrupt_##name(int irq, void *data) \ 682 { \ 683 struct device *dev = data; \ 684 struct talitos_private *priv = dev_get_drvdata(dev); \ 685 u32 isr, isr_lo; \ 686 unsigned long flags; \ 687 \ 688 spin_lock_irqsave(&priv->reg_lock, flags); \ 689 isr = in_be32(priv->reg + TALITOS_ISR); \ 690 isr_lo = in_be32(priv->reg + TALITOS_ISR_LO); \ 691 /* Acknowledge interrupt */ \ 692 out_be32(priv->reg + TALITOS_ICR, isr & (ch_done_mask | ch_err_mask)); \ 693 out_be32(priv->reg + TALITOS_ICR_LO, isr_lo); \ 694 \ 695 if (unlikely(isr & ch_err_mask || isr_lo)) { \ 696 spin_unlock_irqrestore(&priv->reg_lock, flags); \ 697 talitos_error(dev, isr & ch_err_mask, isr_lo); \ 698 } \ 699 else { \ 700 if (likely(isr & ch_done_mask)) { \ 701 /* mask further done interrupts. */ \ 702 clrbits32(priv->reg + TALITOS_IMR, ch_done_mask); \ 703 /* done_task will unmask done interrupts at exit */ \ 704 tasklet_schedule(&priv->done_task[tlet]); \ 705 } \ 706 spin_unlock_irqrestore(&priv->reg_lock, flags); \ 707 } \ 708 \ 709 return (isr & (ch_done_mask | ch_err_mask) || isr_lo) ? IRQ_HANDLED : \ 710 IRQ_NONE; \ 711 } 712 713 DEF_TALITOS2_INTERRUPT(4ch, TALITOS2_ISR_4CHDONE, TALITOS2_ISR_4CHERR, 0) 714 DEF_TALITOS2_INTERRUPT(ch0_2, TALITOS2_ISR_CH_0_2_DONE, TALITOS2_ISR_CH_0_2_ERR, 715 0) 716 DEF_TALITOS2_INTERRUPT(ch1_3, TALITOS2_ISR_CH_1_3_DONE, TALITOS2_ISR_CH_1_3_ERR, 717 1) 718 719 /* 720 * hwrng 721 */ 722 static int talitos_rng_data_present(struct hwrng *rng, int wait) 723 { 724 struct device *dev = (struct device *)rng->priv; 725 struct talitos_private *priv = dev_get_drvdata(dev); 726 u32 ofl; 727 int i; 728 729 for (i = 0; i < 20; i++) { 730 ofl = in_be32(priv->reg_rngu + TALITOS_EUSR_LO) & 731 TALITOS_RNGUSR_LO_OFL; 732 if (ofl || !wait) 733 break; 734 udelay(10); 735 } 736 737 return !!ofl; 738 } 739 740 static int talitos_rng_data_read(struct hwrng *rng, u32 *data) 741 { 742 struct device *dev = (struct device *)rng->priv; 743 struct talitos_private *priv = dev_get_drvdata(dev); 744 745 /* rng fifo requires 64-bit accesses */ 746 *data = in_be32(priv->reg_rngu + TALITOS_EU_FIFO); 747 *data = in_be32(priv->reg_rngu + TALITOS_EU_FIFO_LO); 748 749 return sizeof(u32); 750 } 751 752 static int talitos_rng_init(struct hwrng *rng) 753 { 754 struct device *dev = (struct device *)rng->priv; 755 struct talitos_private *priv = dev_get_drvdata(dev); 756 unsigned int timeout = TALITOS_TIMEOUT; 757 758 setbits32(priv->reg_rngu + TALITOS_EURCR_LO, TALITOS_RNGURCR_LO_SR); 759 while (!(in_be32(priv->reg_rngu + TALITOS_EUSR_LO) 760 & TALITOS_RNGUSR_LO_RD) 761 && --timeout) 762 cpu_relax(); 763 if (timeout == 0) { 764 dev_err(dev, "failed to reset rng hw\n"); 765 return -ENODEV; 766 } 767 768 /* start generating */ 769 setbits32(priv->reg_rngu + TALITOS_EUDSR_LO, 0); 770 771 return 0; 772 } 773 774 static int talitos_register_rng(struct device *dev) 775 { 776 struct talitos_private *priv = dev_get_drvdata(dev); 777 int err; 778 779 priv->rng.name = dev_driver_string(dev), 780 priv->rng.init = talitos_rng_init, 781 priv->rng.data_present = talitos_rng_data_present, 782 priv->rng.data_read = talitos_rng_data_read, 783 priv->rng.priv = (unsigned long)dev; 784 785 err = hwrng_register(&priv->rng); 786 if (!err) 787 priv->rng_registered = true; 788 789 return err; 790 } 791 792 static void talitos_unregister_rng(struct device *dev) 793 { 794 struct talitos_private *priv = dev_get_drvdata(dev); 795 796 if (!priv->rng_registered) 797 return; 798 799 hwrng_unregister(&priv->rng); 800 priv->rng_registered = false; 801 } 802 803 /* 804 * crypto alg 805 */ 806 #define TALITOS_CRA_PRIORITY 3000 807 #define TALITOS_MAX_KEY_SIZE 96 808 #define TALITOS_MAX_IV_LENGTH 16 /* max of AES_BLOCK_SIZE, DES3_EDE_BLOCK_SIZE */ 809 810 struct talitos_ctx { 811 struct device *dev; 812 int ch; 813 __be32 desc_hdr_template; 814 u8 key[TALITOS_MAX_KEY_SIZE]; 815 u8 iv[TALITOS_MAX_IV_LENGTH]; 816 unsigned int keylen; 817 unsigned int enckeylen; 818 unsigned int authkeylen; 819 }; 820 821 #define HASH_MAX_BLOCK_SIZE SHA512_BLOCK_SIZE 822 #define TALITOS_MDEU_MAX_CONTEXT_SIZE TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512 823 824 struct talitos_ahash_req_ctx { 825 u32 hw_context[TALITOS_MDEU_MAX_CONTEXT_SIZE / sizeof(u32)]; 826 unsigned int hw_context_size; 827 u8 buf[HASH_MAX_BLOCK_SIZE]; 828 u8 bufnext[HASH_MAX_BLOCK_SIZE]; 829 unsigned int swinit; 830 unsigned int first; 831 unsigned int last; 832 unsigned int to_hash_later; 833 unsigned int nbuf; 834 struct scatterlist bufsl[2]; 835 struct scatterlist *psrc; 836 }; 837 838 static int aead_setkey(struct crypto_aead *authenc, 839 const u8 *key, unsigned int keylen) 840 { 841 struct talitos_ctx *ctx = crypto_aead_ctx(authenc); 842 struct crypto_authenc_keys keys; 843 844 if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) 845 goto badkey; 846 847 if (keys.authkeylen + keys.enckeylen > TALITOS_MAX_KEY_SIZE) 848 goto badkey; 849 850 memcpy(ctx->key, keys.authkey, keys.authkeylen); 851 memcpy(&ctx->key[keys.authkeylen], keys.enckey, keys.enckeylen); 852 853 ctx->keylen = keys.authkeylen + keys.enckeylen; 854 ctx->enckeylen = keys.enckeylen; 855 ctx->authkeylen = keys.authkeylen; 856 857 return 0; 858 859 badkey: 860 crypto_aead_set_flags(authenc, CRYPTO_TFM_RES_BAD_KEY_LEN); 861 return -EINVAL; 862 } 863 864 /* 865 * talitos_edesc - s/w-extended descriptor 866 * @src_nents: number of segments in input scatterlist 867 * @dst_nents: number of segments in output scatterlist 868 * @icv_ool: whether ICV is out-of-line 869 * @iv_dma: dma address of iv for checking continuity and link table 870 * @dma_len: length of dma mapped link_tbl space 871 * @dma_link_tbl: bus physical address of link_tbl/buf 872 * @desc: h/w descriptor 873 * @link_tbl: input and output h/w link tables (if {src,dst}_nents > 1) (SEC2) 874 * @buf: input and output buffeur (if {src,dst}_nents > 1) (SEC1) 875 * 876 * if decrypting (with authcheck), or either one of src_nents or dst_nents 877 * is greater than 1, an integrity check value is concatenated to the end 878 * of link_tbl data 879 */ 880 struct talitos_edesc { 881 int src_nents; 882 int dst_nents; 883 bool icv_ool; 884 dma_addr_t iv_dma; 885 int dma_len; 886 dma_addr_t dma_link_tbl; 887 struct talitos_desc desc; 888 union { 889 struct talitos_ptr link_tbl[0]; 890 u8 buf[0]; 891 }; 892 }; 893 894 static void talitos_sg_unmap(struct device *dev, 895 struct talitos_edesc *edesc, 896 struct scatterlist *src, 897 struct scatterlist *dst) 898 { 899 unsigned int src_nents = edesc->src_nents ? : 1; 900 unsigned int dst_nents = edesc->dst_nents ? : 1; 901 902 if (src != dst) { 903 dma_unmap_sg(dev, src, src_nents, DMA_TO_DEVICE); 904 905 if (dst) { 906 dma_unmap_sg(dev, dst, dst_nents, DMA_FROM_DEVICE); 907 } 908 } else 909 dma_unmap_sg(dev, src, src_nents, DMA_BIDIRECTIONAL); 910 } 911 912 static void ipsec_esp_unmap(struct device *dev, 913 struct talitos_edesc *edesc, 914 struct aead_request *areq) 915 { 916 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[6], DMA_FROM_DEVICE); 917 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[3], DMA_TO_DEVICE); 918 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[2], DMA_TO_DEVICE); 919 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[0], DMA_TO_DEVICE); 920 921 talitos_sg_unmap(dev, edesc, areq->src, areq->dst); 922 923 if (edesc->dma_len) 924 dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len, 925 DMA_BIDIRECTIONAL); 926 } 927 928 /* 929 * ipsec_esp descriptor callbacks 930 */ 931 static void ipsec_esp_encrypt_done(struct device *dev, 932 struct talitos_desc *desc, void *context, 933 int err) 934 { 935 struct aead_request *areq = context; 936 struct crypto_aead *authenc = crypto_aead_reqtfm(areq); 937 unsigned int authsize = crypto_aead_authsize(authenc); 938 struct talitos_edesc *edesc; 939 struct scatterlist *sg; 940 void *icvdata; 941 942 edesc = container_of(desc, struct talitos_edesc, desc); 943 944 ipsec_esp_unmap(dev, edesc, areq); 945 946 /* copy the generated ICV to dst */ 947 if (edesc->icv_ool) { 948 icvdata = &edesc->link_tbl[edesc->src_nents + 949 edesc->dst_nents + 2]; 950 sg = sg_last(areq->dst, edesc->dst_nents); 951 memcpy((char *)sg_virt(sg) + sg->length - authsize, 952 icvdata, authsize); 953 } 954 955 kfree(edesc); 956 957 aead_request_complete(areq, err); 958 } 959 960 static void ipsec_esp_decrypt_swauth_done(struct device *dev, 961 struct talitos_desc *desc, 962 void *context, int err) 963 { 964 struct aead_request *req = context; 965 struct crypto_aead *authenc = crypto_aead_reqtfm(req); 966 unsigned int authsize = crypto_aead_authsize(authenc); 967 struct talitos_edesc *edesc; 968 struct scatterlist *sg; 969 char *oicv, *icv; 970 971 edesc = container_of(desc, struct talitos_edesc, desc); 972 973 ipsec_esp_unmap(dev, edesc, req); 974 975 if (!err) { 976 /* auth check */ 977 sg = sg_last(req->dst, edesc->dst_nents ? : 1); 978 icv = (char *)sg_virt(sg) + sg->length - authsize; 979 980 if (edesc->dma_len) { 981 oicv = (char *)&edesc->link_tbl[edesc->src_nents + 982 edesc->dst_nents + 2]; 983 if (edesc->icv_ool) 984 icv = oicv + authsize; 985 } else 986 oicv = (char *)&edesc->link_tbl[0]; 987 988 err = crypto_memneq(oicv, icv, authsize) ? -EBADMSG : 0; 989 } 990 991 kfree(edesc); 992 993 aead_request_complete(req, err); 994 } 995 996 static void ipsec_esp_decrypt_hwauth_done(struct device *dev, 997 struct talitos_desc *desc, 998 void *context, int err) 999 { 1000 struct aead_request *req = context; 1001 struct talitos_edesc *edesc; 1002 1003 edesc = container_of(desc, struct talitos_edesc, desc); 1004 1005 ipsec_esp_unmap(dev, edesc, req); 1006 1007 /* check ICV auth status */ 1008 if (!err && ((desc->hdr_lo & DESC_HDR_LO_ICCR1_MASK) != 1009 DESC_HDR_LO_ICCR1_PASS)) 1010 err = -EBADMSG; 1011 1012 kfree(edesc); 1013 1014 aead_request_complete(req, err); 1015 } 1016 1017 /* 1018 * convert scatterlist to SEC h/w link table format 1019 * stop at cryptlen bytes 1020 */ 1021 static int sg_to_link_tbl_offset(struct scatterlist *sg, int sg_count, 1022 unsigned int offset, int cryptlen, 1023 struct talitos_ptr *link_tbl_ptr) 1024 { 1025 int n_sg = sg_count; 1026 int count = 0; 1027 1028 while (cryptlen && sg && n_sg--) { 1029 unsigned int len = sg_dma_len(sg); 1030 1031 if (offset >= len) { 1032 offset -= len; 1033 goto next; 1034 } 1035 1036 len -= offset; 1037 1038 if (len > cryptlen) 1039 len = cryptlen; 1040 1041 to_talitos_ptr(link_tbl_ptr + count, 1042 sg_dma_address(sg) + offset, 0); 1043 link_tbl_ptr[count].len = cpu_to_be16(len); 1044 link_tbl_ptr[count].j_extent = 0; 1045 count++; 1046 cryptlen -= len; 1047 offset = 0; 1048 1049 next: 1050 sg = sg_next(sg); 1051 } 1052 1053 /* tag end of link table */ 1054 if (count > 0) 1055 link_tbl_ptr[count - 1].j_extent = DESC_PTR_LNKTBL_RETURN; 1056 1057 return count; 1058 } 1059 1060 static inline int sg_to_link_tbl(struct scatterlist *sg, int sg_count, 1061 int cryptlen, 1062 struct talitos_ptr *link_tbl_ptr) 1063 { 1064 return sg_to_link_tbl_offset(sg, sg_count, 0, cryptlen, 1065 link_tbl_ptr); 1066 } 1067 1068 /* 1069 * fill in and submit ipsec_esp descriptor 1070 */ 1071 static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq, 1072 void (*callback)(struct device *dev, 1073 struct talitos_desc *desc, 1074 void *context, int error)) 1075 { 1076 struct crypto_aead *aead = crypto_aead_reqtfm(areq); 1077 unsigned int authsize = crypto_aead_authsize(aead); 1078 struct talitos_ctx *ctx = crypto_aead_ctx(aead); 1079 struct device *dev = ctx->dev; 1080 struct talitos_desc *desc = &edesc->desc; 1081 unsigned int cryptlen = areq->cryptlen; 1082 unsigned int ivsize = crypto_aead_ivsize(aead); 1083 int tbl_off = 0; 1084 int sg_count, ret; 1085 int sg_link_tbl_len; 1086 1087 /* hmac key */ 1088 map_single_talitos_ptr(dev, &desc->ptr[0], ctx->authkeylen, &ctx->key, 1089 DMA_TO_DEVICE); 1090 1091 sg_count = dma_map_sg(dev, areq->src, edesc->src_nents ?: 1, 1092 (areq->src == areq->dst) ? DMA_BIDIRECTIONAL 1093 : DMA_TO_DEVICE); 1094 /* hmac data */ 1095 desc->ptr[1].len = cpu_to_be16(areq->assoclen); 1096 if (sg_count > 1 && 1097 (ret = sg_to_link_tbl_offset(areq->src, sg_count, 0, 1098 areq->assoclen, 1099 &edesc->link_tbl[tbl_off])) > 1) { 1100 to_talitos_ptr(&desc->ptr[1], edesc->dma_link_tbl + tbl_off * 1101 sizeof(struct talitos_ptr), 0); 1102 desc->ptr[1].j_extent = DESC_PTR_LNKTBL_JUMP; 1103 1104 dma_sync_single_for_device(dev, edesc->dma_link_tbl, 1105 edesc->dma_len, DMA_BIDIRECTIONAL); 1106 1107 tbl_off += ret; 1108 } else { 1109 to_talitos_ptr(&desc->ptr[1], sg_dma_address(areq->src), 0); 1110 desc->ptr[1].j_extent = 0; 1111 } 1112 1113 /* cipher iv */ 1114 to_talitos_ptr(&desc->ptr[2], edesc->iv_dma, 0); 1115 desc->ptr[2].len = cpu_to_be16(ivsize); 1116 desc->ptr[2].j_extent = 0; 1117 1118 /* cipher key */ 1119 map_single_talitos_ptr(dev, &desc->ptr[3], ctx->enckeylen, 1120 (char *)&ctx->key + ctx->authkeylen, 1121 DMA_TO_DEVICE); 1122 1123 /* 1124 * cipher in 1125 * map and adjust cipher len to aead request cryptlen. 1126 * extent is bytes of HMAC postpended to ciphertext, 1127 * typically 12 for ipsec 1128 */ 1129 desc->ptr[4].len = cpu_to_be16(cryptlen); 1130 desc->ptr[4].j_extent = authsize; 1131 1132 sg_link_tbl_len = cryptlen; 1133 if (edesc->desc.hdr & DESC_HDR_MODE1_MDEU_CICV) 1134 sg_link_tbl_len += authsize; 1135 1136 if (sg_count == 1) { 1137 to_talitos_ptr(&desc->ptr[4], sg_dma_address(areq->src) + 1138 areq->assoclen, 0); 1139 } else if ((ret = sg_to_link_tbl_offset(areq->src, sg_count, 1140 areq->assoclen, sg_link_tbl_len, 1141 &edesc->link_tbl[tbl_off])) > 1142 1) { 1143 desc->ptr[4].j_extent |= DESC_PTR_LNKTBL_JUMP; 1144 to_talitos_ptr(&desc->ptr[4], edesc->dma_link_tbl + 1145 tbl_off * 1146 sizeof(struct talitos_ptr), 0); 1147 dma_sync_single_for_device(dev, edesc->dma_link_tbl, 1148 edesc->dma_len, 1149 DMA_BIDIRECTIONAL); 1150 tbl_off += ret; 1151 } else { 1152 copy_talitos_ptr(&desc->ptr[4], &edesc->link_tbl[tbl_off], 0); 1153 } 1154 1155 /* cipher out */ 1156 desc->ptr[5].len = cpu_to_be16(cryptlen); 1157 desc->ptr[5].j_extent = authsize; 1158 1159 if (areq->src != areq->dst) 1160 sg_count = dma_map_sg(dev, areq->dst, edesc->dst_nents ? : 1, 1161 DMA_FROM_DEVICE); 1162 1163 edesc->icv_ool = false; 1164 1165 if (sg_count == 1) { 1166 to_talitos_ptr(&desc->ptr[5], sg_dma_address(areq->dst) + 1167 areq->assoclen, 0); 1168 } else if ((sg_count = 1169 sg_to_link_tbl_offset(areq->dst, sg_count, 1170 areq->assoclen, cryptlen, 1171 &edesc->link_tbl[tbl_off])) > 1) { 1172 struct talitos_ptr *tbl_ptr = &edesc->link_tbl[tbl_off]; 1173 1174 to_talitos_ptr(&desc->ptr[5], edesc->dma_link_tbl + 1175 tbl_off * sizeof(struct talitos_ptr), 0); 1176 1177 /* Add an entry to the link table for ICV data */ 1178 tbl_ptr += sg_count - 1; 1179 tbl_ptr->j_extent = 0; 1180 tbl_ptr++; 1181 tbl_ptr->j_extent = DESC_PTR_LNKTBL_RETURN; 1182 tbl_ptr->len = cpu_to_be16(authsize); 1183 1184 /* icv data follows link tables */ 1185 to_talitos_ptr(tbl_ptr, edesc->dma_link_tbl + 1186 (edesc->src_nents + edesc->dst_nents + 1187 2) * sizeof(struct talitos_ptr) + 1188 authsize, 0); 1189 desc->ptr[5].j_extent |= DESC_PTR_LNKTBL_JUMP; 1190 dma_sync_single_for_device(ctx->dev, edesc->dma_link_tbl, 1191 edesc->dma_len, DMA_BIDIRECTIONAL); 1192 1193 edesc->icv_ool = true; 1194 } else { 1195 copy_talitos_ptr(&desc->ptr[5], &edesc->link_tbl[tbl_off], 0); 1196 } 1197 1198 /* iv out */ 1199 map_single_talitos_ptr(dev, &desc->ptr[6], ivsize, ctx->iv, 1200 DMA_FROM_DEVICE); 1201 1202 ret = talitos_submit(dev, ctx->ch, desc, callback, areq); 1203 if (ret != -EINPROGRESS) { 1204 ipsec_esp_unmap(dev, edesc, areq); 1205 kfree(edesc); 1206 } 1207 return ret; 1208 } 1209 1210 /* 1211 * allocate and map the extended descriptor 1212 */ 1213 static struct talitos_edesc *talitos_edesc_alloc(struct device *dev, 1214 struct scatterlist *src, 1215 struct scatterlist *dst, 1216 u8 *iv, 1217 unsigned int assoclen, 1218 unsigned int cryptlen, 1219 unsigned int authsize, 1220 unsigned int ivsize, 1221 int icv_stashing, 1222 u32 cryptoflags, 1223 bool encrypt) 1224 { 1225 struct talitos_edesc *edesc; 1226 int src_nents, dst_nents, alloc_len, dma_len; 1227 dma_addr_t iv_dma = 0; 1228 gfp_t flags = cryptoflags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : 1229 GFP_ATOMIC; 1230 struct talitos_private *priv = dev_get_drvdata(dev); 1231 bool is_sec1 = has_ftr_sec1(priv); 1232 int max_len = is_sec1 ? TALITOS1_MAX_DATA_LEN : TALITOS2_MAX_DATA_LEN; 1233 void *err; 1234 1235 if (cryptlen + authsize > max_len) { 1236 dev_err(dev, "length exceeds h/w max limit\n"); 1237 return ERR_PTR(-EINVAL); 1238 } 1239 1240 if (ivsize) 1241 iv_dma = dma_map_single(dev, iv, ivsize, DMA_TO_DEVICE); 1242 1243 if (!dst || dst == src) { 1244 src_nents = sg_nents_for_len(src, 1245 assoclen + cryptlen + authsize); 1246 if (src_nents < 0) { 1247 dev_err(dev, "Invalid number of src SG.\n"); 1248 err = ERR_PTR(-EINVAL); 1249 goto error_sg; 1250 } 1251 src_nents = (src_nents == 1) ? 0 : src_nents; 1252 dst_nents = dst ? src_nents : 0; 1253 } else { /* dst && dst != src*/ 1254 src_nents = sg_nents_for_len(src, assoclen + cryptlen + 1255 (encrypt ? 0 : authsize)); 1256 if (src_nents < 0) { 1257 dev_err(dev, "Invalid number of src SG.\n"); 1258 err = ERR_PTR(-EINVAL); 1259 goto error_sg; 1260 } 1261 src_nents = (src_nents == 1) ? 0 : src_nents; 1262 dst_nents = sg_nents_for_len(dst, assoclen + cryptlen + 1263 (encrypt ? authsize : 0)); 1264 if (dst_nents < 0) { 1265 dev_err(dev, "Invalid number of dst SG.\n"); 1266 err = ERR_PTR(-EINVAL); 1267 goto error_sg; 1268 } 1269 dst_nents = (dst_nents == 1) ? 0 : dst_nents; 1270 } 1271 1272 /* 1273 * allocate space for base edesc plus the link tables, 1274 * allowing for two separate entries for AD and generated ICV (+ 2), 1275 * and space for two sets of ICVs (stashed and generated) 1276 */ 1277 alloc_len = sizeof(struct talitos_edesc); 1278 if (src_nents || dst_nents) { 1279 if (is_sec1) 1280 dma_len = (src_nents ? cryptlen : 0) + 1281 (dst_nents ? cryptlen : 0); 1282 else 1283 dma_len = (src_nents + dst_nents + 2) * 1284 sizeof(struct talitos_ptr) + authsize * 2; 1285 alloc_len += dma_len; 1286 } else { 1287 dma_len = 0; 1288 alloc_len += icv_stashing ? authsize : 0; 1289 } 1290 1291 edesc = kmalloc(alloc_len, GFP_DMA | flags); 1292 if (!edesc) { 1293 dev_err(dev, "could not allocate edescriptor\n"); 1294 err = ERR_PTR(-ENOMEM); 1295 goto error_sg; 1296 } 1297 1298 edesc->src_nents = src_nents; 1299 edesc->dst_nents = dst_nents; 1300 edesc->iv_dma = iv_dma; 1301 edesc->dma_len = dma_len; 1302 if (dma_len) 1303 edesc->dma_link_tbl = dma_map_single(dev, &edesc->link_tbl[0], 1304 edesc->dma_len, 1305 DMA_BIDIRECTIONAL); 1306 1307 return edesc; 1308 error_sg: 1309 if (iv_dma) 1310 dma_unmap_single(dev, iv_dma, ivsize, DMA_TO_DEVICE); 1311 return err; 1312 } 1313 1314 static struct talitos_edesc *aead_edesc_alloc(struct aead_request *areq, u8 *iv, 1315 int icv_stashing, bool encrypt) 1316 { 1317 struct crypto_aead *authenc = crypto_aead_reqtfm(areq); 1318 unsigned int authsize = crypto_aead_authsize(authenc); 1319 struct talitos_ctx *ctx = crypto_aead_ctx(authenc); 1320 unsigned int ivsize = crypto_aead_ivsize(authenc); 1321 1322 return talitos_edesc_alloc(ctx->dev, areq->src, areq->dst, 1323 iv, areq->assoclen, areq->cryptlen, 1324 authsize, ivsize, icv_stashing, 1325 areq->base.flags, encrypt); 1326 } 1327 1328 static int aead_encrypt(struct aead_request *req) 1329 { 1330 struct crypto_aead *authenc = crypto_aead_reqtfm(req); 1331 struct talitos_ctx *ctx = crypto_aead_ctx(authenc); 1332 struct talitos_edesc *edesc; 1333 1334 /* allocate extended descriptor */ 1335 edesc = aead_edesc_alloc(req, req->iv, 0, true); 1336 if (IS_ERR(edesc)) 1337 return PTR_ERR(edesc); 1338 1339 /* set encrypt */ 1340 edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT; 1341 1342 return ipsec_esp(edesc, req, ipsec_esp_encrypt_done); 1343 } 1344 1345 static int aead_decrypt(struct aead_request *req) 1346 { 1347 struct crypto_aead *authenc = crypto_aead_reqtfm(req); 1348 unsigned int authsize = crypto_aead_authsize(authenc); 1349 struct talitos_ctx *ctx = crypto_aead_ctx(authenc); 1350 struct talitos_private *priv = dev_get_drvdata(ctx->dev); 1351 struct talitos_edesc *edesc; 1352 struct scatterlist *sg; 1353 void *icvdata; 1354 1355 req->cryptlen -= authsize; 1356 1357 /* allocate extended descriptor */ 1358 edesc = aead_edesc_alloc(req, req->iv, 1, false); 1359 if (IS_ERR(edesc)) 1360 return PTR_ERR(edesc); 1361 1362 if ((priv->features & TALITOS_FTR_HW_AUTH_CHECK) && 1363 ((!edesc->src_nents && !edesc->dst_nents) || 1364 priv->features & TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT)) { 1365 1366 /* decrypt and check the ICV */ 1367 edesc->desc.hdr = ctx->desc_hdr_template | 1368 DESC_HDR_DIR_INBOUND | 1369 DESC_HDR_MODE1_MDEU_CICV; 1370 1371 /* reset integrity check result bits */ 1372 edesc->desc.hdr_lo = 0; 1373 1374 return ipsec_esp(edesc, req, ipsec_esp_decrypt_hwauth_done); 1375 } 1376 1377 /* Have to check the ICV with software */ 1378 edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_DIR_INBOUND; 1379 1380 /* stash incoming ICV for later cmp with ICV generated by the h/w */ 1381 if (edesc->dma_len) 1382 icvdata = (char *)&edesc->link_tbl[edesc->src_nents + 1383 edesc->dst_nents + 2]; 1384 else 1385 icvdata = &edesc->link_tbl[0]; 1386 1387 sg = sg_last(req->src, edesc->src_nents ? : 1); 1388 1389 memcpy(icvdata, (char *)sg_virt(sg) + sg->length - authsize, authsize); 1390 1391 return ipsec_esp(edesc, req, ipsec_esp_decrypt_swauth_done); 1392 } 1393 1394 static int ablkcipher_setkey(struct crypto_ablkcipher *cipher, 1395 const u8 *key, unsigned int keylen) 1396 { 1397 struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher); 1398 1399 memcpy(&ctx->key, key, keylen); 1400 ctx->keylen = keylen; 1401 1402 return 0; 1403 } 1404 1405 static void unmap_sg_talitos_ptr(struct device *dev, struct scatterlist *src, 1406 struct scatterlist *dst, unsigned int len, 1407 struct talitos_edesc *edesc) 1408 { 1409 struct talitos_private *priv = dev_get_drvdata(dev); 1410 bool is_sec1 = has_ftr_sec1(priv); 1411 1412 if (is_sec1) { 1413 if (!edesc->src_nents) { 1414 dma_unmap_sg(dev, src, 1, 1415 dst != src ? DMA_TO_DEVICE 1416 : DMA_BIDIRECTIONAL); 1417 } 1418 if (dst && edesc->dst_nents) { 1419 dma_sync_single_for_device(dev, 1420 edesc->dma_link_tbl + len, 1421 len, DMA_FROM_DEVICE); 1422 sg_copy_from_buffer(dst, edesc->dst_nents ? : 1, 1423 edesc->buf + len, len); 1424 } else if (dst && dst != src) { 1425 dma_unmap_sg(dev, dst, 1, DMA_FROM_DEVICE); 1426 } 1427 } else { 1428 talitos_sg_unmap(dev, edesc, src, dst); 1429 } 1430 } 1431 1432 static void common_nonsnoop_unmap(struct device *dev, 1433 struct talitos_edesc *edesc, 1434 struct ablkcipher_request *areq) 1435 { 1436 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[5], DMA_FROM_DEVICE); 1437 1438 unmap_sg_talitos_ptr(dev, areq->src, areq->dst, areq->nbytes, edesc); 1439 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[2], DMA_TO_DEVICE); 1440 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[1], DMA_TO_DEVICE); 1441 1442 if (edesc->dma_len) 1443 dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len, 1444 DMA_BIDIRECTIONAL); 1445 } 1446 1447 static void ablkcipher_done(struct device *dev, 1448 struct talitos_desc *desc, void *context, 1449 int err) 1450 { 1451 struct ablkcipher_request *areq = context; 1452 struct talitos_edesc *edesc; 1453 1454 edesc = container_of(desc, struct talitos_edesc, desc); 1455 1456 common_nonsnoop_unmap(dev, edesc, areq); 1457 1458 kfree(edesc); 1459 1460 areq->base.complete(&areq->base, err); 1461 } 1462 1463 int map_sg_in_talitos_ptr(struct device *dev, struct scatterlist *src, 1464 unsigned int len, struct talitos_edesc *edesc, 1465 enum dma_data_direction dir, struct talitos_ptr *ptr) 1466 { 1467 int sg_count; 1468 struct talitos_private *priv = dev_get_drvdata(dev); 1469 bool is_sec1 = has_ftr_sec1(priv); 1470 1471 to_talitos_ptr_len(ptr, len, is_sec1); 1472 1473 if (is_sec1) { 1474 sg_count = edesc->src_nents ? : 1; 1475 1476 if (sg_count == 1) { 1477 dma_map_sg(dev, src, 1, dir); 1478 to_talitos_ptr(ptr, sg_dma_address(src), is_sec1); 1479 } else { 1480 sg_copy_to_buffer(src, sg_count, edesc->buf, len); 1481 to_talitos_ptr(ptr, edesc->dma_link_tbl, is_sec1); 1482 dma_sync_single_for_device(dev, edesc->dma_link_tbl, 1483 len, DMA_TO_DEVICE); 1484 } 1485 } else { 1486 to_talitos_ptr_extent_clear(ptr, is_sec1); 1487 1488 sg_count = dma_map_sg(dev, src, edesc->src_nents ? : 1, dir); 1489 1490 if (sg_count == 1) { 1491 to_talitos_ptr(ptr, sg_dma_address(src), is_sec1); 1492 } else { 1493 sg_count = sg_to_link_tbl(src, sg_count, len, 1494 &edesc->link_tbl[0]); 1495 if (sg_count > 1) { 1496 to_talitos_ptr(ptr, edesc->dma_link_tbl, 0); 1497 ptr->j_extent |= DESC_PTR_LNKTBL_JUMP; 1498 dma_sync_single_for_device(dev, 1499 edesc->dma_link_tbl, 1500 edesc->dma_len, 1501 DMA_BIDIRECTIONAL); 1502 } else { 1503 /* Only one segment now, so no link tbl needed*/ 1504 to_talitos_ptr(ptr, sg_dma_address(src), 1505 is_sec1); 1506 } 1507 } 1508 } 1509 return sg_count; 1510 } 1511 1512 void map_sg_out_talitos_ptr(struct device *dev, struct scatterlist *dst, 1513 unsigned int len, struct talitos_edesc *edesc, 1514 enum dma_data_direction dir, 1515 struct talitos_ptr *ptr, int sg_count) 1516 { 1517 struct talitos_private *priv = dev_get_drvdata(dev); 1518 bool is_sec1 = has_ftr_sec1(priv); 1519 1520 if (dir != DMA_NONE) 1521 sg_count = dma_map_sg(dev, dst, edesc->dst_nents ? : 1, dir); 1522 1523 to_talitos_ptr_len(ptr, len, is_sec1); 1524 1525 if (is_sec1) { 1526 if (sg_count == 1) { 1527 if (dir != DMA_NONE) 1528 dma_map_sg(dev, dst, 1, dir); 1529 to_talitos_ptr(ptr, sg_dma_address(dst), is_sec1); 1530 } else { 1531 to_talitos_ptr(ptr, edesc->dma_link_tbl + len, is_sec1); 1532 dma_sync_single_for_device(dev, 1533 edesc->dma_link_tbl + len, 1534 len, DMA_FROM_DEVICE); 1535 } 1536 } else { 1537 to_talitos_ptr_extent_clear(ptr, is_sec1); 1538 1539 if (sg_count == 1) { 1540 to_talitos_ptr(ptr, sg_dma_address(dst), is_sec1); 1541 } else { 1542 struct talitos_ptr *link_tbl_ptr = 1543 &edesc->link_tbl[edesc->src_nents + 1]; 1544 1545 to_talitos_ptr(ptr, edesc->dma_link_tbl + 1546 (edesc->src_nents + 1) * 1547 sizeof(struct talitos_ptr), 0); 1548 ptr->j_extent |= DESC_PTR_LNKTBL_JUMP; 1549 sg_to_link_tbl(dst, sg_count, len, link_tbl_ptr); 1550 dma_sync_single_for_device(dev, edesc->dma_link_tbl, 1551 edesc->dma_len, 1552 DMA_BIDIRECTIONAL); 1553 } 1554 } 1555 } 1556 1557 static int common_nonsnoop(struct talitos_edesc *edesc, 1558 struct ablkcipher_request *areq, 1559 void (*callback) (struct device *dev, 1560 struct talitos_desc *desc, 1561 void *context, int error)) 1562 { 1563 struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq); 1564 struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher); 1565 struct device *dev = ctx->dev; 1566 struct talitos_desc *desc = &edesc->desc; 1567 unsigned int cryptlen = areq->nbytes; 1568 unsigned int ivsize = crypto_ablkcipher_ivsize(cipher); 1569 int sg_count, ret; 1570 struct talitos_private *priv = dev_get_drvdata(dev); 1571 bool is_sec1 = has_ftr_sec1(priv); 1572 1573 /* first DWORD empty */ 1574 desc->ptr[0] = zero_entry; 1575 1576 /* cipher iv */ 1577 to_talitos_ptr(&desc->ptr[1], edesc->iv_dma, is_sec1); 1578 to_talitos_ptr_len(&desc->ptr[1], ivsize, is_sec1); 1579 to_talitos_ptr_extent_clear(&desc->ptr[1], is_sec1); 1580 1581 /* cipher key */ 1582 map_single_talitos_ptr(dev, &desc->ptr[2], ctx->keylen, 1583 (char *)&ctx->key, DMA_TO_DEVICE); 1584 1585 /* 1586 * cipher in 1587 */ 1588 sg_count = map_sg_in_talitos_ptr(dev, areq->src, cryptlen, edesc, 1589 (areq->src == areq->dst) ? 1590 DMA_BIDIRECTIONAL : DMA_TO_DEVICE, 1591 &desc->ptr[3]); 1592 1593 /* cipher out */ 1594 map_sg_out_talitos_ptr(dev, areq->dst, cryptlen, edesc, 1595 (areq->src == areq->dst) ? DMA_NONE 1596 : DMA_FROM_DEVICE, 1597 &desc->ptr[4], sg_count); 1598 1599 /* iv out */ 1600 map_single_talitos_ptr(dev, &desc->ptr[5], ivsize, ctx->iv, 1601 DMA_FROM_DEVICE); 1602 1603 /* last DWORD empty */ 1604 desc->ptr[6] = zero_entry; 1605 1606 ret = talitos_submit(dev, ctx->ch, desc, callback, areq); 1607 if (ret != -EINPROGRESS) { 1608 common_nonsnoop_unmap(dev, edesc, areq); 1609 kfree(edesc); 1610 } 1611 return ret; 1612 } 1613 1614 static struct talitos_edesc *ablkcipher_edesc_alloc(struct ablkcipher_request * 1615 areq, bool encrypt) 1616 { 1617 struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq); 1618 struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher); 1619 unsigned int ivsize = crypto_ablkcipher_ivsize(cipher); 1620 1621 return talitos_edesc_alloc(ctx->dev, areq->src, areq->dst, 1622 areq->info, 0, areq->nbytes, 0, ivsize, 0, 1623 areq->base.flags, encrypt); 1624 } 1625 1626 static int ablkcipher_encrypt(struct ablkcipher_request *areq) 1627 { 1628 struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq); 1629 struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher); 1630 struct talitos_edesc *edesc; 1631 1632 /* allocate extended descriptor */ 1633 edesc = ablkcipher_edesc_alloc(areq, true); 1634 if (IS_ERR(edesc)) 1635 return PTR_ERR(edesc); 1636 1637 /* set encrypt */ 1638 edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT; 1639 1640 return common_nonsnoop(edesc, areq, ablkcipher_done); 1641 } 1642 1643 static int ablkcipher_decrypt(struct ablkcipher_request *areq) 1644 { 1645 struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq); 1646 struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher); 1647 struct talitos_edesc *edesc; 1648 1649 /* allocate extended descriptor */ 1650 edesc = ablkcipher_edesc_alloc(areq, false); 1651 if (IS_ERR(edesc)) 1652 return PTR_ERR(edesc); 1653 1654 edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_DIR_INBOUND; 1655 1656 return common_nonsnoop(edesc, areq, ablkcipher_done); 1657 } 1658 1659 static void common_nonsnoop_hash_unmap(struct device *dev, 1660 struct talitos_edesc *edesc, 1661 struct ahash_request *areq) 1662 { 1663 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 1664 struct talitos_private *priv = dev_get_drvdata(dev); 1665 bool is_sec1 = has_ftr_sec1(priv); 1666 1667 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[5], DMA_FROM_DEVICE); 1668 1669 unmap_sg_talitos_ptr(dev, req_ctx->psrc, NULL, 0, edesc); 1670 1671 /* When using hashctx-in, must unmap it. */ 1672 if (from_talitos_ptr_len(&edesc->desc.ptr[1], is_sec1)) 1673 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[1], 1674 DMA_TO_DEVICE); 1675 1676 if (from_talitos_ptr_len(&edesc->desc.ptr[2], is_sec1)) 1677 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[2], 1678 DMA_TO_DEVICE); 1679 1680 if (edesc->dma_len) 1681 dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len, 1682 DMA_BIDIRECTIONAL); 1683 1684 } 1685 1686 static void ahash_done(struct device *dev, 1687 struct talitos_desc *desc, void *context, 1688 int err) 1689 { 1690 struct ahash_request *areq = context; 1691 struct talitos_edesc *edesc = 1692 container_of(desc, struct talitos_edesc, desc); 1693 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 1694 1695 if (!req_ctx->last && req_ctx->to_hash_later) { 1696 /* Position any partial block for next update/final/finup */ 1697 memcpy(req_ctx->buf, req_ctx->bufnext, req_ctx->to_hash_later); 1698 req_ctx->nbuf = req_ctx->to_hash_later; 1699 } 1700 common_nonsnoop_hash_unmap(dev, edesc, areq); 1701 1702 kfree(edesc); 1703 1704 areq->base.complete(&areq->base, err); 1705 } 1706 1707 /* 1708 * SEC1 doesn't like hashing of 0 sized message, so we do the padding 1709 * ourself and submit a padded block 1710 */ 1711 void talitos_handle_buggy_hash(struct talitos_ctx *ctx, 1712 struct talitos_edesc *edesc, 1713 struct talitos_ptr *ptr) 1714 { 1715 static u8 padded_hash[64] = { 1716 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1717 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1718 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1719 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1720 }; 1721 1722 pr_err_once("Bug in SEC1, padding ourself\n"); 1723 edesc->desc.hdr &= ~DESC_HDR_MODE0_MDEU_PAD; 1724 map_single_talitos_ptr(ctx->dev, ptr, sizeof(padded_hash), 1725 (char *)padded_hash, DMA_TO_DEVICE); 1726 } 1727 1728 static int common_nonsnoop_hash(struct talitos_edesc *edesc, 1729 struct ahash_request *areq, unsigned int length, 1730 void (*callback) (struct device *dev, 1731 struct talitos_desc *desc, 1732 void *context, int error)) 1733 { 1734 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); 1735 struct talitos_ctx *ctx = crypto_ahash_ctx(tfm); 1736 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 1737 struct device *dev = ctx->dev; 1738 struct talitos_desc *desc = &edesc->desc; 1739 int ret; 1740 struct talitos_private *priv = dev_get_drvdata(dev); 1741 bool is_sec1 = has_ftr_sec1(priv); 1742 1743 /* first DWORD empty */ 1744 desc->ptr[0] = zero_entry; 1745 1746 /* hash context in */ 1747 if (!req_ctx->first || req_ctx->swinit) { 1748 map_single_talitos_ptr(dev, &desc->ptr[1], 1749 req_ctx->hw_context_size, 1750 (char *)req_ctx->hw_context, 1751 DMA_TO_DEVICE); 1752 req_ctx->swinit = 0; 1753 } else { 1754 desc->ptr[1] = zero_entry; 1755 /* Indicate next op is not the first. */ 1756 req_ctx->first = 0; 1757 } 1758 1759 /* HMAC key */ 1760 if (ctx->keylen) 1761 map_single_talitos_ptr(dev, &desc->ptr[2], ctx->keylen, 1762 (char *)&ctx->key, DMA_TO_DEVICE); 1763 else 1764 desc->ptr[2] = zero_entry; 1765 1766 /* 1767 * data in 1768 */ 1769 map_sg_in_talitos_ptr(dev, req_ctx->psrc, length, edesc, 1770 DMA_TO_DEVICE, &desc->ptr[3]); 1771 1772 /* fifth DWORD empty */ 1773 desc->ptr[4] = zero_entry; 1774 1775 /* hash/HMAC out -or- hash context out */ 1776 if (req_ctx->last) 1777 map_single_talitos_ptr(dev, &desc->ptr[5], 1778 crypto_ahash_digestsize(tfm), 1779 areq->result, DMA_FROM_DEVICE); 1780 else 1781 map_single_talitos_ptr(dev, &desc->ptr[5], 1782 req_ctx->hw_context_size, 1783 req_ctx->hw_context, DMA_FROM_DEVICE); 1784 1785 /* last DWORD empty */ 1786 desc->ptr[6] = zero_entry; 1787 1788 if (is_sec1 && from_talitos_ptr_len(&desc->ptr[3], true) == 0) 1789 talitos_handle_buggy_hash(ctx, edesc, &desc->ptr[3]); 1790 1791 ret = talitos_submit(dev, ctx->ch, desc, callback, areq); 1792 if (ret != -EINPROGRESS) { 1793 common_nonsnoop_hash_unmap(dev, edesc, areq); 1794 kfree(edesc); 1795 } 1796 return ret; 1797 } 1798 1799 static struct talitos_edesc *ahash_edesc_alloc(struct ahash_request *areq, 1800 unsigned int nbytes) 1801 { 1802 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); 1803 struct talitos_ctx *ctx = crypto_ahash_ctx(tfm); 1804 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 1805 1806 return talitos_edesc_alloc(ctx->dev, req_ctx->psrc, NULL, NULL, 0, 1807 nbytes, 0, 0, 0, areq->base.flags, false); 1808 } 1809 1810 static int ahash_init(struct ahash_request *areq) 1811 { 1812 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); 1813 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 1814 1815 /* Initialize the context */ 1816 req_ctx->nbuf = 0; 1817 req_ctx->first = 1; /* first indicates h/w must init its context */ 1818 req_ctx->swinit = 0; /* assume h/w init of context */ 1819 req_ctx->hw_context_size = 1820 (crypto_ahash_digestsize(tfm) <= SHA256_DIGEST_SIZE) 1821 ? TALITOS_MDEU_CONTEXT_SIZE_MD5_SHA1_SHA256 1822 : TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512; 1823 1824 return 0; 1825 } 1826 1827 /* 1828 * on h/w without explicit sha224 support, we initialize h/w context 1829 * manually with sha224 constants, and tell it to run sha256. 1830 */ 1831 static int ahash_init_sha224_swinit(struct ahash_request *areq) 1832 { 1833 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 1834 1835 ahash_init(areq); 1836 req_ctx->swinit = 1;/* prevent h/w initting context with sha256 values*/ 1837 1838 req_ctx->hw_context[0] = SHA224_H0; 1839 req_ctx->hw_context[1] = SHA224_H1; 1840 req_ctx->hw_context[2] = SHA224_H2; 1841 req_ctx->hw_context[3] = SHA224_H3; 1842 req_ctx->hw_context[4] = SHA224_H4; 1843 req_ctx->hw_context[5] = SHA224_H5; 1844 req_ctx->hw_context[6] = SHA224_H6; 1845 req_ctx->hw_context[7] = SHA224_H7; 1846 1847 /* init 64-bit count */ 1848 req_ctx->hw_context[8] = 0; 1849 req_ctx->hw_context[9] = 0; 1850 1851 return 0; 1852 } 1853 1854 static int ahash_process_req(struct ahash_request *areq, unsigned int nbytes) 1855 { 1856 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); 1857 struct talitos_ctx *ctx = crypto_ahash_ctx(tfm); 1858 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 1859 struct talitos_edesc *edesc; 1860 unsigned int blocksize = 1861 crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm)); 1862 unsigned int nbytes_to_hash; 1863 unsigned int to_hash_later; 1864 unsigned int nsg; 1865 int nents; 1866 1867 if (!req_ctx->last && (nbytes + req_ctx->nbuf <= blocksize)) { 1868 /* Buffer up to one whole block */ 1869 nents = sg_nents_for_len(areq->src, nbytes); 1870 if (nents < 0) { 1871 dev_err(ctx->dev, "Invalid number of src SG.\n"); 1872 return nents; 1873 } 1874 sg_copy_to_buffer(areq->src, nents, 1875 req_ctx->buf + req_ctx->nbuf, nbytes); 1876 req_ctx->nbuf += nbytes; 1877 return 0; 1878 } 1879 1880 /* At least (blocksize + 1) bytes are available to hash */ 1881 nbytes_to_hash = nbytes + req_ctx->nbuf; 1882 to_hash_later = nbytes_to_hash & (blocksize - 1); 1883 1884 if (req_ctx->last) 1885 to_hash_later = 0; 1886 else if (to_hash_later) 1887 /* There is a partial block. Hash the full block(s) now */ 1888 nbytes_to_hash -= to_hash_later; 1889 else { 1890 /* Keep one block buffered */ 1891 nbytes_to_hash -= blocksize; 1892 to_hash_later = blocksize; 1893 } 1894 1895 /* Chain in any previously buffered data */ 1896 if (req_ctx->nbuf) { 1897 nsg = (req_ctx->nbuf < nbytes_to_hash) ? 2 : 1; 1898 sg_init_table(req_ctx->bufsl, nsg); 1899 sg_set_buf(req_ctx->bufsl, req_ctx->buf, req_ctx->nbuf); 1900 if (nsg > 1) 1901 sg_chain(req_ctx->bufsl, 2, areq->src); 1902 req_ctx->psrc = req_ctx->bufsl; 1903 } else 1904 req_ctx->psrc = areq->src; 1905 1906 if (to_hash_later) { 1907 nents = sg_nents_for_len(areq->src, nbytes); 1908 if (nents < 0) { 1909 dev_err(ctx->dev, "Invalid number of src SG.\n"); 1910 return nents; 1911 } 1912 sg_pcopy_to_buffer(areq->src, nents, 1913 req_ctx->bufnext, 1914 to_hash_later, 1915 nbytes - to_hash_later); 1916 } 1917 req_ctx->to_hash_later = to_hash_later; 1918 1919 /* Allocate extended descriptor */ 1920 edesc = ahash_edesc_alloc(areq, nbytes_to_hash); 1921 if (IS_ERR(edesc)) 1922 return PTR_ERR(edesc); 1923 1924 edesc->desc.hdr = ctx->desc_hdr_template; 1925 1926 /* On last one, request SEC to pad; otherwise continue */ 1927 if (req_ctx->last) 1928 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_PAD; 1929 else 1930 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_CONT; 1931 1932 /* request SEC to INIT hash. */ 1933 if (req_ctx->first && !req_ctx->swinit) 1934 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_INIT; 1935 1936 /* When the tfm context has a keylen, it's an HMAC. 1937 * A first or last (ie. not middle) descriptor must request HMAC. 1938 */ 1939 if (ctx->keylen && (req_ctx->first || req_ctx->last)) 1940 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_HMAC; 1941 1942 return common_nonsnoop_hash(edesc, areq, nbytes_to_hash, 1943 ahash_done); 1944 } 1945 1946 static int ahash_update(struct ahash_request *areq) 1947 { 1948 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 1949 1950 req_ctx->last = 0; 1951 1952 return ahash_process_req(areq, areq->nbytes); 1953 } 1954 1955 static int ahash_final(struct ahash_request *areq) 1956 { 1957 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 1958 1959 req_ctx->last = 1; 1960 1961 return ahash_process_req(areq, 0); 1962 } 1963 1964 static int ahash_finup(struct ahash_request *areq) 1965 { 1966 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 1967 1968 req_ctx->last = 1; 1969 1970 return ahash_process_req(areq, areq->nbytes); 1971 } 1972 1973 static int ahash_digest(struct ahash_request *areq) 1974 { 1975 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 1976 struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq); 1977 1978 ahash->init(areq); 1979 req_ctx->last = 1; 1980 1981 return ahash_process_req(areq, areq->nbytes); 1982 } 1983 1984 struct keyhash_result { 1985 struct completion completion; 1986 int err; 1987 }; 1988 1989 static void keyhash_complete(struct crypto_async_request *req, int err) 1990 { 1991 struct keyhash_result *res = req->data; 1992 1993 if (err == -EINPROGRESS) 1994 return; 1995 1996 res->err = err; 1997 complete(&res->completion); 1998 } 1999 2000 static int keyhash(struct crypto_ahash *tfm, const u8 *key, unsigned int keylen, 2001 u8 *hash) 2002 { 2003 struct talitos_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm)); 2004 2005 struct scatterlist sg[1]; 2006 struct ahash_request *req; 2007 struct keyhash_result hresult; 2008 int ret; 2009 2010 init_completion(&hresult.completion); 2011 2012 req = ahash_request_alloc(tfm, GFP_KERNEL); 2013 if (!req) 2014 return -ENOMEM; 2015 2016 /* Keep tfm keylen == 0 during hash of the long key */ 2017 ctx->keylen = 0; 2018 ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, 2019 keyhash_complete, &hresult); 2020 2021 sg_init_one(&sg[0], key, keylen); 2022 2023 ahash_request_set_crypt(req, sg, hash, keylen); 2024 ret = crypto_ahash_digest(req); 2025 switch (ret) { 2026 case 0: 2027 break; 2028 case -EINPROGRESS: 2029 case -EBUSY: 2030 ret = wait_for_completion_interruptible( 2031 &hresult.completion); 2032 if (!ret) 2033 ret = hresult.err; 2034 break; 2035 default: 2036 break; 2037 } 2038 ahash_request_free(req); 2039 2040 return ret; 2041 } 2042 2043 static int ahash_setkey(struct crypto_ahash *tfm, const u8 *key, 2044 unsigned int keylen) 2045 { 2046 struct talitos_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm)); 2047 unsigned int blocksize = 2048 crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm)); 2049 unsigned int digestsize = crypto_ahash_digestsize(tfm); 2050 unsigned int keysize = keylen; 2051 u8 hash[SHA512_DIGEST_SIZE]; 2052 int ret; 2053 2054 if (keylen <= blocksize) 2055 memcpy(ctx->key, key, keysize); 2056 else { 2057 /* Must get the hash of the long key */ 2058 ret = keyhash(tfm, key, keylen, hash); 2059 2060 if (ret) { 2061 crypto_ahash_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); 2062 return -EINVAL; 2063 } 2064 2065 keysize = digestsize; 2066 memcpy(ctx->key, hash, digestsize); 2067 } 2068 2069 ctx->keylen = keysize; 2070 2071 return 0; 2072 } 2073 2074 2075 struct talitos_alg_template { 2076 u32 type; 2077 union { 2078 struct crypto_alg crypto; 2079 struct ahash_alg hash; 2080 struct aead_alg aead; 2081 } alg; 2082 __be32 desc_hdr_template; 2083 }; 2084 2085 static struct talitos_alg_template driver_algs[] = { 2086 /* AEAD algorithms. These use a single-pass ipsec_esp descriptor */ 2087 { .type = CRYPTO_ALG_TYPE_AEAD, 2088 .alg.aead = { 2089 .base = { 2090 .cra_name = "authenc(hmac(sha1),cbc(aes))", 2091 .cra_driver_name = "authenc-hmac-sha1-" 2092 "cbc-aes-talitos", 2093 .cra_blocksize = AES_BLOCK_SIZE, 2094 .cra_flags = CRYPTO_ALG_ASYNC, 2095 }, 2096 .ivsize = AES_BLOCK_SIZE, 2097 .maxauthsize = SHA1_DIGEST_SIZE, 2098 }, 2099 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | 2100 DESC_HDR_SEL0_AESU | 2101 DESC_HDR_MODE0_AESU_CBC | 2102 DESC_HDR_SEL1_MDEUA | 2103 DESC_HDR_MODE1_MDEU_INIT | 2104 DESC_HDR_MODE1_MDEU_PAD | 2105 DESC_HDR_MODE1_MDEU_SHA1_HMAC, 2106 }, 2107 { .type = CRYPTO_ALG_TYPE_AEAD, 2108 .alg.aead = { 2109 .base = { 2110 .cra_name = "authenc(hmac(sha1)," 2111 "cbc(des3_ede))", 2112 .cra_driver_name = "authenc-hmac-sha1-" 2113 "cbc-3des-talitos", 2114 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2115 .cra_flags = CRYPTO_ALG_ASYNC, 2116 }, 2117 .ivsize = DES3_EDE_BLOCK_SIZE, 2118 .maxauthsize = SHA1_DIGEST_SIZE, 2119 }, 2120 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | 2121 DESC_HDR_SEL0_DEU | 2122 DESC_HDR_MODE0_DEU_CBC | 2123 DESC_HDR_MODE0_DEU_3DES | 2124 DESC_HDR_SEL1_MDEUA | 2125 DESC_HDR_MODE1_MDEU_INIT | 2126 DESC_HDR_MODE1_MDEU_PAD | 2127 DESC_HDR_MODE1_MDEU_SHA1_HMAC, 2128 }, 2129 { .type = CRYPTO_ALG_TYPE_AEAD, 2130 .alg.aead = { 2131 .base = { 2132 .cra_name = "authenc(hmac(sha224),cbc(aes))", 2133 .cra_driver_name = "authenc-hmac-sha224-" 2134 "cbc-aes-talitos", 2135 .cra_blocksize = AES_BLOCK_SIZE, 2136 .cra_flags = CRYPTO_ALG_ASYNC, 2137 }, 2138 .ivsize = AES_BLOCK_SIZE, 2139 .maxauthsize = SHA224_DIGEST_SIZE, 2140 }, 2141 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | 2142 DESC_HDR_SEL0_AESU | 2143 DESC_HDR_MODE0_AESU_CBC | 2144 DESC_HDR_SEL1_MDEUA | 2145 DESC_HDR_MODE1_MDEU_INIT | 2146 DESC_HDR_MODE1_MDEU_PAD | 2147 DESC_HDR_MODE1_MDEU_SHA224_HMAC, 2148 }, 2149 { .type = CRYPTO_ALG_TYPE_AEAD, 2150 .alg.aead = { 2151 .base = { 2152 .cra_name = "authenc(hmac(sha224)," 2153 "cbc(des3_ede))", 2154 .cra_driver_name = "authenc-hmac-sha224-" 2155 "cbc-3des-talitos", 2156 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2157 .cra_flags = CRYPTO_ALG_ASYNC, 2158 }, 2159 .ivsize = DES3_EDE_BLOCK_SIZE, 2160 .maxauthsize = SHA224_DIGEST_SIZE, 2161 }, 2162 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | 2163 DESC_HDR_SEL0_DEU | 2164 DESC_HDR_MODE0_DEU_CBC | 2165 DESC_HDR_MODE0_DEU_3DES | 2166 DESC_HDR_SEL1_MDEUA | 2167 DESC_HDR_MODE1_MDEU_INIT | 2168 DESC_HDR_MODE1_MDEU_PAD | 2169 DESC_HDR_MODE1_MDEU_SHA224_HMAC, 2170 }, 2171 { .type = CRYPTO_ALG_TYPE_AEAD, 2172 .alg.aead = { 2173 .base = { 2174 .cra_name = "authenc(hmac(sha256),cbc(aes))", 2175 .cra_driver_name = "authenc-hmac-sha256-" 2176 "cbc-aes-talitos", 2177 .cra_blocksize = AES_BLOCK_SIZE, 2178 .cra_flags = CRYPTO_ALG_ASYNC, 2179 }, 2180 .ivsize = AES_BLOCK_SIZE, 2181 .maxauthsize = SHA256_DIGEST_SIZE, 2182 }, 2183 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | 2184 DESC_HDR_SEL0_AESU | 2185 DESC_HDR_MODE0_AESU_CBC | 2186 DESC_HDR_SEL1_MDEUA | 2187 DESC_HDR_MODE1_MDEU_INIT | 2188 DESC_HDR_MODE1_MDEU_PAD | 2189 DESC_HDR_MODE1_MDEU_SHA256_HMAC, 2190 }, 2191 { .type = CRYPTO_ALG_TYPE_AEAD, 2192 .alg.aead = { 2193 .base = { 2194 .cra_name = "authenc(hmac(sha256)," 2195 "cbc(des3_ede))", 2196 .cra_driver_name = "authenc-hmac-sha256-" 2197 "cbc-3des-talitos", 2198 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2199 .cra_flags = CRYPTO_ALG_ASYNC, 2200 }, 2201 .ivsize = DES3_EDE_BLOCK_SIZE, 2202 .maxauthsize = SHA256_DIGEST_SIZE, 2203 }, 2204 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | 2205 DESC_HDR_SEL0_DEU | 2206 DESC_HDR_MODE0_DEU_CBC | 2207 DESC_HDR_MODE0_DEU_3DES | 2208 DESC_HDR_SEL1_MDEUA | 2209 DESC_HDR_MODE1_MDEU_INIT | 2210 DESC_HDR_MODE1_MDEU_PAD | 2211 DESC_HDR_MODE1_MDEU_SHA256_HMAC, 2212 }, 2213 { .type = CRYPTO_ALG_TYPE_AEAD, 2214 .alg.aead = { 2215 .base = { 2216 .cra_name = "authenc(hmac(sha384),cbc(aes))", 2217 .cra_driver_name = "authenc-hmac-sha384-" 2218 "cbc-aes-talitos", 2219 .cra_blocksize = AES_BLOCK_SIZE, 2220 .cra_flags = CRYPTO_ALG_ASYNC, 2221 }, 2222 .ivsize = AES_BLOCK_SIZE, 2223 .maxauthsize = SHA384_DIGEST_SIZE, 2224 }, 2225 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | 2226 DESC_HDR_SEL0_AESU | 2227 DESC_HDR_MODE0_AESU_CBC | 2228 DESC_HDR_SEL1_MDEUB | 2229 DESC_HDR_MODE1_MDEU_INIT | 2230 DESC_HDR_MODE1_MDEU_PAD | 2231 DESC_HDR_MODE1_MDEUB_SHA384_HMAC, 2232 }, 2233 { .type = CRYPTO_ALG_TYPE_AEAD, 2234 .alg.aead = { 2235 .base = { 2236 .cra_name = "authenc(hmac(sha384)," 2237 "cbc(des3_ede))", 2238 .cra_driver_name = "authenc-hmac-sha384-" 2239 "cbc-3des-talitos", 2240 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2241 .cra_flags = CRYPTO_ALG_ASYNC, 2242 }, 2243 .ivsize = DES3_EDE_BLOCK_SIZE, 2244 .maxauthsize = SHA384_DIGEST_SIZE, 2245 }, 2246 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | 2247 DESC_HDR_SEL0_DEU | 2248 DESC_HDR_MODE0_DEU_CBC | 2249 DESC_HDR_MODE0_DEU_3DES | 2250 DESC_HDR_SEL1_MDEUB | 2251 DESC_HDR_MODE1_MDEU_INIT | 2252 DESC_HDR_MODE1_MDEU_PAD | 2253 DESC_HDR_MODE1_MDEUB_SHA384_HMAC, 2254 }, 2255 { .type = CRYPTO_ALG_TYPE_AEAD, 2256 .alg.aead = { 2257 .base = { 2258 .cra_name = "authenc(hmac(sha512),cbc(aes))", 2259 .cra_driver_name = "authenc-hmac-sha512-" 2260 "cbc-aes-talitos", 2261 .cra_blocksize = AES_BLOCK_SIZE, 2262 .cra_flags = CRYPTO_ALG_ASYNC, 2263 }, 2264 .ivsize = AES_BLOCK_SIZE, 2265 .maxauthsize = SHA512_DIGEST_SIZE, 2266 }, 2267 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | 2268 DESC_HDR_SEL0_AESU | 2269 DESC_HDR_MODE0_AESU_CBC | 2270 DESC_HDR_SEL1_MDEUB | 2271 DESC_HDR_MODE1_MDEU_INIT | 2272 DESC_HDR_MODE1_MDEU_PAD | 2273 DESC_HDR_MODE1_MDEUB_SHA512_HMAC, 2274 }, 2275 { .type = CRYPTO_ALG_TYPE_AEAD, 2276 .alg.aead = { 2277 .base = { 2278 .cra_name = "authenc(hmac(sha512)," 2279 "cbc(des3_ede))", 2280 .cra_driver_name = "authenc-hmac-sha512-" 2281 "cbc-3des-talitos", 2282 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2283 .cra_flags = CRYPTO_ALG_ASYNC, 2284 }, 2285 .ivsize = DES3_EDE_BLOCK_SIZE, 2286 .maxauthsize = SHA512_DIGEST_SIZE, 2287 }, 2288 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | 2289 DESC_HDR_SEL0_DEU | 2290 DESC_HDR_MODE0_DEU_CBC | 2291 DESC_HDR_MODE0_DEU_3DES | 2292 DESC_HDR_SEL1_MDEUB | 2293 DESC_HDR_MODE1_MDEU_INIT | 2294 DESC_HDR_MODE1_MDEU_PAD | 2295 DESC_HDR_MODE1_MDEUB_SHA512_HMAC, 2296 }, 2297 { .type = CRYPTO_ALG_TYPE_AEAD, 2298 .alg.aead = { 2299 .base = { 2300 .cra_name = "authenc(hmac(md5),cbc(aes))", 2301 .cra_driver_name = "authenc-hmac-md5-" 2302 "cbc-aes-talitos", 2303 .cra_blocksize = AES_BLOCK_SIZE, 2304 .cra_flags = CRYPTO_ALG_ASYNC, 2305 }, 2306 .ivsize = AES_BLOCK_SIZE, 2307 .maxauthsize = MD5_DIGEST_SIZE, 2308 }, 2309 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | 2310 DESC_HDR_SEL0_AESU | 2311 DESC_HDR_MODE0_AESU_CBC | 2312 DESC_HDR_SEL1_MDEUA | 2313 DESC_HDR_MODE1_MDEU_INIT | 2314 DESC_HDR_MODE1_MDEU_PAD | 2315 DESC_HDR_MODE1_MDEU_MD5_HMAC, 2316 }, 2317 { .type = CRYPTO_ALG_TYPE_AEAD, 2318 .alg.aead = { 2319 .base = { 2320 .cra_name = "authenc(hmac(md5),cbc(des3_ede))", 2321 .cra_driver_name = "authenc-hmac-md5-" 2322 "cbc-3des-talitos", 2323 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2324 .cra_flags = CRYPTO_ALG_ASYNC, 2325 }, 2326 .ivsize = DES3_EDE_BLOCK_SIZE, 2327 .maxauthsize = MD5_DIGEST_SIZE, 2328 }, 2329 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | 2330 DESC_HDR_SEL0_DEU | 2331 DESC_HDR_MODE0_DEU_CBC | 2332 DESC_HDR_MODE0_DEU_3DES | 2333 DESC_HDR_SEL1_MDEUA | 2334 DESC_HDR_MODE1_MDEU_INIT | 2335 DESC_HDR_MODE1_MDEU_PAD | 2336 DESC_HDR_MODE1_MDEU_MD5_HMAC, 2337 }, 2338 /* ABLKCIPHER algorithms. */ 2339 { .type = CRYPTO_ALG_TYPE_ABLKCIPHER, 2340 .alg.crypto = { 2341 .cra_name = "ecb(aes)", 2342 .cra_driver_name = "ecb-aes-talitos", 2343 .cra_blocksize = AES_BLOCK_SIZE, 2344 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | 2345 CRYPTO_ALG_ASYNC, 2346 .cra_ablkcipher = { 2347 .min_keysize = AES_MIN_KEY_SIZE, 2348 .max_keysize = AES_MAX_KEY_SIZE, 2349 .ivsize = AES_BLOCK_SIZE, 2350 } 2351 }, 2352 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 2353 DESC_HDR_SEL0_AESU, 2354 }, 2355 { .type = CRYPTO_ALG_TYPE_ABLKCIPHER, 2356 .alg.crypto = { 2357 .cra_name = "cbc(aes)", 2358 .cra_driver_name = "cbc-aes-talitos", 2359 .cra_blocksize = AES_BLOCK_SIZE, 2360 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | 2361 CRYPTO_ALG_ASYNC, 2362 .cra_ablkcipher = { 2363 .min_keysize = AES_MIN_KEY_SIZE, 2364 .max_keysize = AES_MAX_KEY_SIZE, 2365 .ivsize = AES_BLOCK_SIZE, 2366 } 2367 }, 2368 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 2369 DESC_HDR_SEL0_AESU | 2370 DESC_HDR_MODE0_AESU_CBC, 2371 }, 2372 { .type = CRYPTO_ALG_TYPE_ABLKCIPHER, 2373 .alg.crypto = { 2374 .cra_name = "ctr(aes)", 2375 .cra_driver_name = "ctr-aes-talitos", 2376 .cra_blocksize = AES_BLOCK_SIZE, 2377 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | 2378 CRYPTO_ALG_ASYNC, 2379 .cra_ablkcipher = { 2380 .min_keysize = AES_MIN_KEY_SIZE, 2381 .max_keysize = AES_MAX_KEY_SIZE, 2382 .ivsize = AES_BLOCK_SIZE, 2383 } 2384 }, 2385 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 2386 DESC_HDR_SEL0_AESU | 2387 DESC_HDR_MODE0_AESU_CTR, 2388 }, 2389 { .type = CRYPTO_ALG_TYPE_ABLKCIPHER, 2390 .alg.crypto = { 2391 .cra_name = "ecb(des)", 2392 .cra_driver_name = "ecb-des-talitos", 2393 .cra_blocksize = DES_BLOCK_SIZE, 2394 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | 2395 CRYPTO_ALG_ASYNC, 2396 .cra_ablkcipher = { 2397 .min_keysize = DES_KEY_SIZE, 2398 .max_keysize = DES_KEY_SIZE, 2399 .ivsize = DES_BLOCK_SIZE, 2400 } 2401 }, 2402 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 2403 DESC_HDR_SEL0_DEU, 2404 }, 2405 { .type = CRYPTO_ALG_TYPE_ABLKCIPHER, 2406 .alg.crypto = { 2407 .cra_name = "cbc(des)", 2408 .cra_driver_name = "cbc-des-talitos", 2409 .cra_blocksize = DES_BLOCK_SIZE, 2410 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | 2411 CRYPTO_ALG_ASYNC, 2412 .cra_ablkcipher = { 2413 .min_keysize = DES_KEY_SIZE, 2414 .max_keysize = DES_KEY_SIZE, 2415 .ivsize = DES_BLOCK_SIZE, 2416 } 2417 }, 2418 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 2419 DESC_HDR_SEL0_DEU | 2420 DESC_HDR_MODE0_DEU_CBC, 2421 }, 2422 { .type = CRYPTO_ALG_TYPE_ABLKCIPHER, 2423 .alg.crypto = { 2424 .cra_name = "ecb(des3_ede)", 2425 .cra_driver_name = "ecb-3des-talitos", 2426 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2427 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | 2428 CRYPTO_ALG_ASYNC, 2429 .cra_ablkcipher = { 2430 .min_keysize = DES3_EDE_KEY_SIZE, 2431 .max_keysize = DES3_EDE_KEY_SIZE, 2432 .ivsize = DES3_EDE_BLOCK_SIZE, 2433 } 2434 }, 2435 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 2436 DESC_HDR_SEL0_DEU | 2437 DESC_HDR_MODE0_DEU_3DES, 2438 }, 2439 { .type = CRYPTO_ALG_TYPE_ABLKCIPHER, 2440 .alg.crypto = { 2441 .cra_name = "cbc(des3_ede)", 2442 .cra_driver_name = "cbc-3des-talitos", 2443 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2444 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | 2445 CRYPTO_ALG_ASYNC, 2446 .cra_ablkcipher = { 2447 .min_keysize = DES3_EDE_KEY_SIZE, 2448 .max_keysize = DES3_EDE_KEY_SIZE, 2449 .ivsize = DES3_EDE_BLOCK_SIZE, 2450 } 2451 }, 2452 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 2453 DESC_HDR_SEL0_DEU | 2454 DESC_HDR_MODE0_DEU_CBC | 2455 DESC_HDR_MODE0_DEU_3DES, 2456 }, 2457 /* AHASH algorithms. */ 2458 { .type = CRYPTO_ALG_TYPE_AHASH, 2459 .alg.hash = { 2460 .halg.digestsize = MD5_DIGEST_SIZE, 2461 .halg.base = { 2462 .cra_name = "md5", 2463 .cra_driver_name = "md5-talitos", 2464 .cra_blocksize = MD5_HMAC_BLOCK_SIZE, 2465 .cra_flags = CRYPTO_ALG_TYPE_AHASH | 2466 CRYPTO_ALG_ASYNC, 2467 } 2468 }, 2469 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 2470 DESC_HDR_SEL0_MDEUA | 2471 DESC_HDR_MODE0_MDEU_MD5, 2472 }, 2473 { .type = CRYPTO_ALG_TYPE_AHASH, 2474 .alg.hash = { 2475 .halg.digestsize = SHA1_DIGEST_SIZE, 2476 .halg.base = { 2477 .cra_name = "sha1", 2478 .cra_driver_name = "sha1-talitos", 2479 .cra_blocksize = SHA1_BLOCK_SIZE, 2480 .cra_flags = CRYPTO_ALG_TYPE_AHASH | 2481 CRYPTO_ALG_ASYNC, 2482 } 2483 }, 2484 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 2485 DESC_HDR_SEL0_MDEUA | 2486 DESC_HDR_MODE0_MDEU_SHA1, 2487 }, 2488 { .type = CRYPTO_ALG_TYPE_AHASH, 2489 .alg.hash = { 2490 .halg.digestsize = SHA224_DIGEST_SIZE, 2491 .halg.base = { 2492 .cra_name = "sha224", 2493 .cra_driver_name = "sha224-talitos", 2494 .cra_blocksize = SHA224_BLOCK_SIZE, 2495 .cra_flags = CRYPTO_ALG_TYPE_AHASH | 2496 CRYPTO_ALG_ASYNC, 2497 } 2498 }, 2499 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 2500 DESC_HDR_SEL0_MDEUA | 2501 DESC_HDR_MODE0_MDEU_SHA224, 2502 }, 2503 { .type = CRYPTO_ALG_TYPE_AHASH, 2504 .alg.hash = { 2505 .halg.digestsize = SHA256_DIGEST_SIZE, 2506 .halg.base = { 2507 .cra_name = "sha256", 2508 .cra_driver_name = "sha256-talitos", 2509 .cra_blocksize = SHA256_BLOCK_SIZE, 2510 .cra_flags = CRYPTO_ALG_TYPE_AHASH | 2511 CRYPTO_ALG_ASYNC, 2512 } 2513 }, 2514 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 2515 DESC_HDR_SEL0_MDEUA | 2516 DESC_HDR_MODE0_MDEU_SHA256, 2517 }, 2518 { .type = CRYPTO_ALG_TYPE_AHASH, 2519 .alg.hash = { 2520 .halg.digestsize = SHA384_DIGEST_SIZE, 2521 .halg.base = { 2522 .cra_name = "sha384", 2523 .cra_driver_name = "sha384-talitos", 2524 .cra_blocksize = SHA384_BLOCK_SIZE, 2525 .cra_flags = CRYPTO_ALG_TYPE_AHASH | 2526 CRYPTO_ALG_ASYNC, 2527 } 2528 }, 2529 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 2530 DESC_HDR_SEL0_MDEUB | 2531 DESC_HDR_MODE0_MDEUB_SHA384, 2532 }, 2533 { .type = CRYPTO_ALG_TYPE_AHASH, 2534 .alg.hash = { 2535 .halg.digestsize = SHA512_DIGEST_SIZE, 2536 .halg.base = { 2537 .cra_name = "sha512", 2538 .cra_driver_name = "sha512-talitos", 2539 .cra_blocksize = SHA512_BLOCK_SIZE, 2540 .cra_flags = CRYPTO_ALG_TYPE_AHASH | 2541 CRYPTO_ALG_ASYNC, 2542 } 2543 }, 2544 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 2545 DESC_HDR_SEL0_MDEUB | 2546 DESC_HDR_MODE0_MDEUB_SHA512, 2547 }, 2548 { .type = CRYPTO_ALG_TYPE_AHASH, 2549 .alg.hash = { 2550 .halg.digestsize = MD5_DIGEST_SIZE, 2551 .halg.base = { 2552 .cra_name = "hmac(md5)", 2553 .cra_driver_name = "hmac-md5-talitos", 2554 .cra_blocksize = MD5_HMAC_BLOCK_SIZE, 2555 .cra_flags = CRYPTO_ALG_TYPE_AHASH | 2556 CRYPTO_ALG_ASYNC, 2557 } 2558 }, 2559 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 2560 DESC_HDR_SEL0_MDEUA | 2561 DESC_HDR_MODE0_MDEU_MD5, 2562 }, 2563 { .type = CRYPTO_ALG_TYPE_AHASH, 2564 .alg.hash = { 2565 .halg.digestsize = SHA1_DIGEST_SIZE, 2566 .halg.base = { 2567 .cra_name = "hmac(sha1)", 2568 .cra_driver_name = "hmac-sha1-talitos", 2569 .cra_blocksize = SHA1_BLOCK_SIZE, 2570 .cra_flags = CRYPTO_ALG_TYPE_AHASH | 2571 CRYPTO_ALG_ASYNC, 2572 } 2573 }, 2574 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 2575 DESC_HDR_SEL0_MDEUA | 2576 DESC_HDR_MODE0_MDEU_SHA1, 2577 }, 2578 { .type = CRYPTO_ALG_TYPE_AHASH, 2579 .alg.hash = { 2580 .halg.digestsize = SHA224_DIGEST_SIZE, 2581 .halg.base = { 2582 .cra_name = "hmac(sha224)", 2583 .cra_driver_name = "hmac-sha224-talitos", 2584 .cra_blocksize = SHA224_BLOCK_SIZE, 2585 .cra_flags = CRYPTO_ALG_TYPE_AHASH | 2586 CRYPTO_ALG_ASYNC, 2587 } 2588 }, 2589 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 2590 DESC_HDR_SEL0_MDEUA | 2591 DESC_HDR_MODE0_MDEU_SHA224, 2592 }, 2593 { .type = CRYPTO_ALG_TYPE_AHASH, 2594 .alg.hash = { 2595 .halg.digestsize = SHA256_DIGEST_SIZE, 2596 .halg.base = { 2597 .cra_name = "hmac(sha256)", 2598 .cra_driver_name = "hmac-sha256-talitos", 2599 .cra_blocksize = SHA256_BLOCK_SIZE, 2600 .cra_flags = CRYPTO_ALG_TYPE_AHASH | 2601 CRYPTO_ALG_ASYNC, 2602 } 2603 }, 2604 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 2605 DESC_HDR_SEL0_MDEUA | 2606 DESC_HDR_MODE0_MDEU_SHA256, 2607 }, 2608 { .type = CRYPTO_ALG_TYPE_AHASH, 2609 .alg.hash = { 2610 .halg.digestsize = SHA384_DIGEST_SIZE, 2611 .halg.base = { 2612 .cra_name = "hmac(sha384)", 2613 .cra_driver_name = "hmac-sha384-talitos", 2614 .cra_blocksize = SHA384_BLOCK_SIZE, 2615 .cra_flags = CRYPTO_ALG_TYPE_AHASH | 2616 CRYPTO_ALG_ASYNC, 2617 } 2618 }, 2619 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 2620 DESC_HDR_SEL0_MDEUB | 2621 DESC_HDR_MODE0_MDEUB_SHA384, 2622 }, 2623 { .type = CRYPTO_ALG_TYPE_AHASH, 2624 .alg.hash = { 2625 .halg.digestsize = SHA512_DIGEST_SIZE, 2626 .halg.base = { 2627 .cra_name = "hmac(sha512)", 2628 .cra_driver_name = "hmac-sha512-talitos", 2629 .cra_blocksize = SHA512_BLOCK_SIZE, 2630 .cra_flags = CRYPTO_ALG_TYPE_AHASH | 2631 CRYPTO_ALG_ASYNC, 2632 } 2633 }, 2634 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 2635 DESC_HDR_SEL0_MDEUB | 2636 DESC_HDR_MODE0_MDEUB_SHA512, 2637 } 2638 }; 2639 2640 struct talitos_crypto_alg { 2641 struct list_head entry; 2642 struct device *dev; 2643 struct talitos_alg_template algt; 2644 }; 2645 2646 static int talitos_init_common(struct talitos_ctx *ctx, 2647 struct talitos_crypto_alg *talitos_alg) 2648 { 2649 struct talitos_private *priv; 2650 2651 /* update context with ptr to dev */ 2652 ctx->dev = talitos_alg->dev; 2653 2654 /* assign SEC channel to tfm in round-robin fashion */ 2655 priv = dev_get_drvdata(ctx->dev); 2656 ctx->ch = atomic_inc_return(&priv->last_chan) & 2657 (priv->num_channels - 1); 2658 2659 /* copy descriptor header template value */ 2660 ctx->desc_hdr_template = talitos_alg->algt.desc_hdr_template; 2661 2662 /* select done notification */ 2663 ctx->desc_hdr_template |= DESC_HDR_DONE_NOTIFY; 2664 2665 return 0; 2666 } 2667 2668 static int talitos_cra_init(struct crypto_tfm *tfm) 2669 { 2670 struct crypto_alg *alg = tfm->__crt_alg; 2671 struct talitos_crypto_alg *talitos_alg; 2672 struct talitos_ctx *ctx = crypto_tfm_ctx(tfm); 2673 2674 if ((alg->cra_flags & CRYPTO_ALG_TYPE_MASK) == CRYPTO_ALG_TYPE_AHASH) 2675 talitos_alg = container_of(__crypto_ahash_alg(alg), 2676 struct talitos_crypto_alg, 2677 algt.alg.hash); 2678 else 2679 talitos_alg = container_of(alg, struct talitos_crypto_alg, 2680 algt.alg.crypto); 2681 2682 return talitos_init_common(ctx, talitos_alg); 2683 } 2684 2685 static int talitos_cra_init_aead(struct crypto_aead *tfm) 2686 { 2687 struct aead_alg *alg = crypto_aead_alg(tfm); 2688 struct talitos_crypto_alg *talitos_alg; 2689 struct talitos_ctx *ctx = crypto_aead_ctx(tfm); 2690 2691 talitos_alg = container_of(alg, struct talitos_crypto_alg, 2692 algt.alg.aead); 2693 2694 return talitos_init_common(ctx, talitos_alg); 2695 } 2696 2697 static int talitos_cra_init_ahash(struct crypto_tfm *tfm) 2698 { 2699 struct talitos_ctx *ctx = crypto_tfm_ctx(tfm); 2700 2701 talitos_cra_init(tfm); 2702 2703 ctx->keylen = 0; 2704 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), 2705 sizeof(struct talitos_ahash_req_ctx)); 2706 2707 return 0; 2708 } 2709 2710 /* 2711 * given the alg's descriptor header template, determine whether descriptor 2712 * type and primary/secondary execution units required match the hw 2713 * capabilities description provided in the device tree node. 2714 */ 2715 static int hw_supports(struct device *dev, __be32 desc_hdr_template) 2716 { 2717 struct talitos_private *priv = dev_get_drvdata(dev); 2718 int ret; 2719 2720 ret = (1 << DESC_TYPE(desc_hdr_template) & priv->desc_types) && 2721 (1 << PRIMARY_EU(desc_hdr_template) & priv->exec_units); 2722 2723 if (SECONDARY_EU(desc_hdr_template)) 2724 ret = ret && (1 << SECONDARY_EU(desc_hdr_template) 2725 & priv->exec_units); 2726 2727 return ret; 2728 } 2729 2730 static int talitos_remove(struct platform_device *ofdev) 2731 { 2732 struct device *dev = &ofdev->dev; 2733 struct talitos_private *priv = dev_get_drvdata(dev); 2734 struct talitos_crypto_alg *t_alg, *n; 2735 int i; 2736 2737 list_for_each_entry_safe(t_alg, n, &priv->alg_list, entry) { 2738 switch (t_alg->algt.type) { 2739 case CRYPTO_ALG_TYPE_ABLKCIPHER: 2740 break; 2741 case CRYPTO_ALG_TYPE_AEAD: 2742 crypto_unregister_aead(&t_alg->algt.alg.aead); 2743 case CRYPTO_ALG_TYPE_AHASH: 2744 crypto_unregister_ahash(&t_alg->algt.alg.hash); 2745 break; 2746 } 2747 list_del(&t_alg->entry); 2748 kfree(t_alg); 2749 } 2750 2751 if (hw_supports(dev, DESC_HDR_SEL0_RNG)) 2752 talitos_unregister_rng(dev); 2753 2754 for (i = 0; priv->chan && i < priv->num_channels; i++) 2755 kfree(priv->chan[i].fifo); 2756 2757 kfree(priv->chan); 2758 2759 for (i = 0; i < 2; i++) 2760 if (priv->irq[i]) { 2761 free_irq(priv->irq[i], dev); 2762 irq_dispose_mapping(priv->irq[i]); 2763 } 2764 2765 tasklet_kill(&priv->done_task[0]); 2766 if (priv->irq[1]) 2767 tasklet_kill(&priv->done_task[1]); 2768 2769 iounmap(priv->reg); 2770 2771 kfree(priv); 2772 2773 return 0; 2774 } 2775 2776 static struct talitos_crypto_alg *talitos_alg_alloc(struct device *dev, 2777 struct talitos_alg_template 2778 *template) 2779 { 2780 struct talitos_private *priv = dev_get_drvdata(dev); 2781 struct talitos_crypto_alg *t_alg; 2782 struct crypto_alg *alg; 2783 2784 t_alg = kzalloc(sizeof(struct talitos_crypto_alg), GFP_KERNEL); 2785 if (!t_alg) 2786 return ERR_PTR(-ENOMEM); 2787 2788 t_alg->algt = *template; 2789 2790 switch (t_alg->algt.type) { 2791 case CRYPTO_ALG_TYPE_ABLKCIPHER: 2792 alg = &t_alg->algt.alg.crypto; 2793 alg->cra_init = talitos_cra_init; 2794 alg->cra_type = &crypto_ablkcipher_type; 2795 alg->cra_ablkcipher.setkey = ablkcipher_setkey; 2796 alg->cra_ablkcipher.encrypt = ablkcipher_encrypt; 2797 alg->cra_ablkcipher.decrypt = ablkcipher_decrypt; 2798 alg->cra_ablkcipher.geniv = "eseqiv"; 2799 break; 2800 case CRYPTO_ALG_TYPE_AEAD: 2801 alg = &t_alg->algt.alg.aead.base; 2802 t_alg->algt.alg.aead.init = talitos_cra_init_aead; 2803 t_alg->algt.alg.aead.setkey = aead_setkey; 2804 t_alg->algt.alg.aead.encrypt = aead_encrypt; 2805 t_alg->algt.alg.aead.decrypt = aead_decrypt; 2806 break; 2807 case CRYPTO_ALG_TYPE_AHASH: 2808 alg = &t_alg->algt.alg.hash.halg.base; 2809 alg->cra_init = talitos_cra_init_ahash; 2810 alg->cra_type = &crypto_ahash_type; 2811 t_alg->algt.alg.hash.init = ahash_init; 2812 t_alg->algt.alg.hash.update = ahash_update; 2813 t_alg->algt.alg.hash.final = ahash_final; 2814 t_alg->algt.alg.hash.finup = ahash_finup; 2815 t_alg->algt.alg.hash.digest = ahash_digest; 2816 t_alg->algt.alg.hash.setkey = ahash_setkey; 2817 2818 if (!(priv->features & TALITOS_FTR_HMAC_OK) && 2819 !strncmp(alg->cra_name, "hmac", 4)) { 2820 kfree(t_alg); 2821 return ERR_PTR(-ENOTSUPP); 2822 } 2823 if (!(priv->features & TALITOS_FTR_SHA224_HWINIT) && 2824 (!strcmp(alg->cra_name, "sha224") || 2825 !strcmp(alg->cra_name, "hmac(sha224)"))) { 2826 t_alg->algt.alg.hash.init = ahash_init_sha224_swinit; 2827 t_alg->algt.desc_hdr_template = 2828 DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | 2829 DESC_HDR_SEL0_MDEUA | 2830 DESC_HDR_MODE0_MDEU_SHA256; 2831 } 2832 break; 2833 default: 2834 dev_err(dev, "unknown algorithm type %d\n", t_alg->algt.type); 2835 kfree(t_alg); 2836 return ERR_PTR(-EINVAL); 2837 } 2838 2839 alg->cra_module = THIS_MODULE; 2840 alg->cra_priority = TALITOS_CRA_PRIORITY; 2841 alg->cra_alignmask = 0; 2842 alg->cra_ctxsize = sizeof(struct talitos_ctx); 2843 alg->cra_flags |= CRYPTO_ALG_KERN_DRIVER_ONLY; 2844 2845 t_alg->dev = dev; 2846 2847 return t_alg; 2848 } 2849 2850 static int talitos_probe_irq(struct platform_device *ofdev) 2851 { 2852 struct device *dev = &ofdev->dev; 2853 struct device_node *np = ofdev->dev.of_node; 2854 struct talitos_private *priv = dev_get_drvdata(dev); 2855 int err; 2856 bool is_sec1 = has_ftr_sec1(priv); 2857 2858 priv->irq[0] = irq_of_parse_and_map(np, 0); 2859 if (!priv->irq[0]) { 2860 dev_err(dev, "failed to map irq\n"); 2861 return -EINVAL; 2862 } 2863 if (is_sec1) { 2864 err = request_irq(priv->irq[0], talitos1_interrupt_4ch, 0, 2865 dev_driver_string(dev), dev); 2866 goto primary_out; 2867 } 2868 2869 priv->irq[1] = irq_of_parse_and_map(np, 1); 2870 2871 /* get the primary irq line */ 2872 if (!priv->irq[1]) { 2873 err = request_irq(priv->irq[0], talitos2_interrupt_4ch, 0, 2874 dev_driver_string(dev), dev); 2875 goto primary_out; 2876 } 2877 2878 err = request_irq(priv->irq[0], talitos2_interrupt_ch0_2, 0, 2879 dev_driver_string(dev), dev); 2880 if (err) 2881 goto primary_out; 2882 2883 /* get the secondary irq line */ 2884 err = request_irq(priv->irq[1], talitos2_interrupt_ch1_3, 0, 2885 dev_driver_string(dev), dev); 2886 if (err) { 2887 dev_err(dev, "failed to request secondary irq\n"); 2888 irq_dispose_mapping(priv->irq[1]); 2889 priv->irq[1] = 0; 2890 } 2891 2892 return err; 2893 2894 primary_out: 2895 if (err) { 2896 dev_err(dev, "failed to request primary irq\n"); 2897 irq_dispose_mapping(priv->irq[0]); 2898 priv->irq[0] = 0; 2899 } 2900 2901 return err; 2902 } 2903 2904 static int talitos_probe(struct platform_device *ofdev) 2905 { 2906 struct device *dev = &ofdev->dev; 2907 struct device_node *np = ofdev->dev.of_node; 2908 struct talitos_private *priv; 2909 const unsigned int *prop; 2910 int i, err; 2911 int stride; 2912 2913 priv = kzalloc(sizeof(struct talitos_private), GFP_KERNEL); 2914 if (!priv) 2915 return -ENOMEM; 2916 2917 INIT_LIST_HEAD(&priv->alg_list); 2918 2919 dev_set_drvdata(dev, priv); 2920 2921 priv->ofdev = ofdev; 2922 2923 spin_lock_init(&priv->reg_lock); 2924 2925 priv->reg = of_iomap(np, 0); 2926 if (!priv->reg) { 2927 dev_err(dev, "failed to of_iomap\n"); 2928 err = -ENOMEM; 2929 goto err_out; 2930 } 2931 2932 /* get SEC version capabilities from device tree */ 2933 prop = of_get_property(np, "fsl,num-channels", NULL); 2934 if (prop) 2935 priv->num_channels = *prop; 2936 2937 prop = of_get_property(np, "fsl,channel-fifo-len", NULL); 2938 if (prop) 2939 priv->chfifo_len = *prop; 2940 2941 prop = of_get_property(np, "fsl,exec-units-mask", NULL); 2942 if (prop) 2943 priv->exec_units = *prop; 2944 2945 prop = of_get_property(np, "fsl,descriptor-types-mask", NULL); 2946 if (prop) 2947 priv->desc_types = *prop; 2948 2949 if (!is_power_of_2(priv->num_channels) || !priv->chfifo_len || 2950 !priv->exec_units || !priv->desc_types) { 2951 dev_err(dev, "invalid property data in device tree node\n"); 2952 err = -EINVAL; 2953 goto err_out; 2954 } 2955 2956 if (of_device_is_compatible(np, "fsl,sec3.0")) 2957 priv->features |= TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT; 2958 2959 if (of_device_is_compatible(np, "fsl,sec2.1")) 2960 priv->features |= TALITOS_FTR_HW_AUTH_CHECK | 2961 TALITOS_FTR_SHA224_HWINIT | 2962 TALITOS_FTR_HMAC_OK; 2963 2964 if (of_device_is_compatible(np, "fsl,sec1.0")) 2965 priv->features |= TALITOS_FTR_SEC1; 2966 2967 if (of_device_is_compatible(np, "fsl,sec1.2")) { 2968 priv->reg_deu = priv->reg + TALITOS12_DEU; 2969 priv->reg_aesu = priv->reg + TALITOS12_AESU; 2970 priv->reg_mdeu = priv->reg + TALITOS12_MDEU; 2971 stride = TALITOS1_CH_STRIDE; 2972 } else if (of_device_is_compatible(np, "fsl,sec1.0")) { 2973 priv->reg_deu = priv->reg + TALITOS10_DEU; 2974 priv->reg_aesu = priv->reg + TALITOS10_AESU; 2975 priv->reg_mdeu = priv->reg + TALITOS10_MDEU; 2976 priv->reg_afeu = priv->reg + TALITOS10_AFEU; 2977 priv->reg_rngu = priv->reg + TALITOS10_RNGU; 2978 priv->reg_pkeu = priv->reg + TALITOS10_PKEU; 2979 stride = TALITOS1_CH_STRIDE; 2980 } else { 2981 priv->reg_deu = priv->reg + TALITOS2_DEU; 2982 priv->reg_aesu = priv->reg + TALITOS2_AESU; 2983 priv->reg_mdeu = priv->reg + TALITOS2_MDEU; 2984 priv->reg_afeu = priv->reg + TALITOS2_AFEU; 2985 priv->reg_rngu = priv->reg + TALITOS2_RNGU; 2986 priv->reg_pkeu = priv->reg + TALITOS2_PKEU; 2987 priv->reg_keu = priv->reg + TALITOS2_KEU; 2988 priv->reg_crcu = priv->reg + TALITOS2_CRCU; 2989 stride = TALITOS2_CH_STRIDE; 2990 } 2991 2992 err = talitos_probe_irq(ofdev); 2993 if (err) 2994 goto err_out; 2995 2996 if (of_device_is_compatible(np, "fsl,sec1.0")) { 2997 tasklet_init(&priv->done_task[0], talitos1_done_4ch, 2998 (unsigned long)dev); 2999 } else { 3000 if (!priv->irq[1]) { 3001 tasklet_init(&priv->done_task[0], talitos2_done_4ch, 3002 (unsigned long)dev); 3003 } else { 3004 tasklet_init(&priv->done_task[0], talitos2_done_ch0_2, 3005 (unsigned long)dev); 3006 tasklet_init(&priv->done_task[1], talitos2_done_ch1_3, 3007 (unsigned long)dev); 3008 } 3009 } 3010 3011 priv->chan = kzalloc(sizeof(struct talitos_channel) * 3012 priv->num_channels, GFP_KERNEL); 3013 if (!priv->chan) { 3014 dev_err(dev, "failed to allocate channel management space\n"); 3015 err = -ENOMEM; 3016 goto err_out; 3017 } 3018 3019 priv->fifo_len = roundup_pow_of_two(priv->chfifo_len); 3020 3021 for (i = 0; i < priv->num_channels; i++) { 3022 priv->chan[i].reg = priv->reg + stride * (i + 1); 3023 if (!priv->irq[1] || !(i & 1)) 3024 priv->chan[i].reg += TALITOS_CH_BASE_OFFSET; 3025 3026 spin_lock_init(&priv->chan[i].head_lock); 3027 spin_lock_init(&priv->chan[i].tail_lock); 3028 3029 priv->chan[i].fifo = kzalloc(sizeof(struct talitos_request) * 3030 priv->fifo_len, GFP_KERNEL); 3031 if (!priv->chan[i].fifo) { 3032 dev_err(dev, "failed to allocate request fifo %d\n", i); 3033 err = -ENOMEM; 3034 goto err_out; 3035 } 3036 3037 atomic_set(&priv->chan[i].submit_count, 3038 -(priv->chfifo_len - 1)); 3039 } 3040 3041 dma_set_mask(dev, DMA_BIT_MASK(36)); 3042 3043 /* reset and initialize the h/w */ 3044 err = init_device(dev); 3045 if (err) { 3046 dev_err(dev, "failed to initialize device\n"); 3047 goto err_out; 3048 } 3049 3050 /* register the RNG, if available */ 3051 if (hw_supports(dev, DESC_HDR_SEL0_RNG)) { 3052 err = talitos_register_rng(dev); 3053 if (err) { 3054 dev_err(dev, "failed to register hwrng: %d\n", err); 3055 goto err_out; 3056 } else 3057 dev_info(dev, "hwrng\n"); 3058 } 3059 3060 /* register crypto algorithms the device supports */ 3061 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) { 3062 if (hw_supports(dev, driver_algs[i].desc_hdr_template)) { 3063 struct talitos_crypto_alg *t_alg; 3064 struct crypto_alg *alg = NULL; 3065 3066 t_alg = talitos_alg_alloc(dev, &driver_algs[i]); 3067 if (IS_ERR(t_alg)) { 3068 err = PTR_ERR(t_alg); 3069 if (err == -ENOTSUPP) 3070 continue; 3071 goto err_out; 3072 } 3073 3074 switch (t_alg->algt.type) { 3075 case CRYPTO_ALG_TYPE_ABLKCIPHER: 3076 err = crypto_register_alg( 3077 &t_alg->algt.alg.crypto); 3078 alg = &t_alg->algt.alg.crypto; 3079 break; 3080 3081 case CRYPTO_ALG_TYPE_AEAD: 3082 err = crypto_register_aead( 3083 &t_alg->algt.alg.aead); 3084 alg = &t_alg->algt.alg.aead.base; 3085 break; 3086 3087 case CRYPTO_ALG_TYPE_AHASH: 3088 err = crypto_register_ahash( 3089 &t_alg->algt.alg.hash); 3090 alg = &t_alg->algt.alg.hash.halg.base; 3091 break; 3092 } 3093 if (err) { 3094 dev_err(dev, "%s alg registration failed\n", 3095 alg->cra_driver_name); 3096 kfree(t_alg); 3097 } else 3098 list_add_tail(&t_alg->entry, &priv->alg_list); 3099 } 3100 } 3101 if (!list_empty(&priv->alg_list)) 3102 dev_info(dev, "%s algorithms registered in /proc/crypto\n", 3103 (char *)of_get_property(np, "compatible", NULL)); 3104 3105 return 0; 3106 3107 err_out: 3108 talitos_remove(ofdev); 3109 3110 return err; 3111 } 3112 3113 static const struct of_device_id talitos_match[] = { 3114 #ifdef CONFIG_CRYPTO_DEV_TALITOS1 3115 { 3116 .compatible = "fsl,sec1.0", 3117 }, 3118 #endif 3119 #ifdef CONFIG_CRYPTO_DEV_TALITOS2 3120 { 3121 .compatible = "fsl,sec2.0", 3122 }, 3123 #endif 3124 {}, 3125 }; 3126 MODULE_DEVICE_TABLE(of, talitos_match); 3127 3128 static struct platform_driver talitos_driver = { 3129 .driver = { 3130 .name = "talitos", 3131 .of_match_table = talitos_match, 3132 }, 3133 .probe = talitos_probe, 3134 .remove = talitos_remove, 3135 }; 3136 3137 module_platform_driver(talitos_driver); 3138 3139 MODULE_LICENSE("GPL"); 3140 MODULE_AUTHOR("Kim Phillips <kim.phillips@freescale.com>"); 3141 MODULE_DESCRIPTION("Freescale integrated security engine (SEC) driver"); 3142