1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * AMD Cryptographic Coprocessor (CCP) driver 4 * 5 * Copyright (C) 2013-2019 Advanced Micro Devices, Inc. 6 * 7 * Author: Tom Lendacky <thomas.lendacky@amd.com> 8 * Author: Gary R Hook <gary.hook@amd.com> 9 */ 10 11 #include <linux/module.h> 12 #include <linux/kernel.h> 13 #include <linux/pci.h> 14 #include <linux/interrupt.h> 15 #include <crypto/scatterwalk.h> 16 #include <crypto/des.h> 17 #include <linux/ccp.h> 18 19 #include "ccp-dev.h" 20 21 /* SHA initial context values */ 22 static const __be32 ccp_sha1_init[SHA1_DIGEST_SIZE / sizeof(__be32)] = { 23 cpu_to_be32(SHA1_H0), cpu_to_be32(SHA1_H1), 24 cpu_to_be32(SHA1_H2), cpu_to_be32(SHA1_H3), 25 cpu_to_be32(SHA1_H4), 26 }; 27 28 static const __be32 ccp_sha224_init[SHA256_DIGEST_SIZE / sizeof(__be32)] = { 29 cpu_to_be32(SHA224_H0), cpu_to_be32(SHA224_H1), 30 cpu_to_be32(SHA224_H2), cpu_to_be32(SHA224_H3), 31 cpu_to_be32(SHA224_H4), cpu_to_be32(SHA224_H5), 32 cpu_to_be32(SHA224_H6), cpu_to_be32(SHA224_H7), 33 }; 34 35 static const __be32 ccp_sha256_init[SHA256_DIGEST_SIZE / sizeof(__be32)] = { 36 cpu_to_be32(SHA256_H0), cpu_to_be32(SHA256_H1), 37 cpu_to_be32(SHA256_H2), cpu_to_be32(SHA256_H3), 38 cpu_to_be32(SHA256_H4), cpu_to_be32(SHA256_H5), 39 cpu_to_be32(SHA256_H6), cpu_to_be32(SHA256_H7), 40 }; 41 42 static const __be64 ccp_sha384_init[SHA512_DIGEST_SIZE / sizeof(__be64)] = { 43 cpu_to_be64(SHA384_H0), cpu_to_be64(SHA384_H1), 44 cpu_to_be64(SHA384_H2), cpu_to_be64(SHA384_H3), 45 cpu_to_be64(SHA384_H4), cpu_to_be64(SHA384_H5), 46 cpu_to_be64(SHA384_H6), cpu_to_be64(SHA384_H7), 47 }; 48 49 static const __be64 ccp_sha512_init[SHA512_DIGEST_SIZE / sizeof(__be64)] = { 50 cpu_to_be64(SHA512_H0), cpu_to_be64(SHA512_H1), 51 cpu_to_be64(SHA512_H2), cpu_to_be64(SHA512_H3), 52 cpu_to_be64(SHA512_H4), cpu_to_be64(SHA512_H5), 53 cpu_to_be64(SHA512_H6), cpu_to_be64(SHA512_H7), 54 }; 55 56 #define CCP_NEW_JOBID(ccp) ((ccp->vdata->version == CCP_VERSION(3, 0)) ? \ 57 ccp_gen_jobid(ccp) : 0) 58 59 static u32 ccp_gen_jobid(struct ccp_device *ccp) 60 { 61 return atomic_inc_return(&ccp->current_id) & CCP_JOBID_MASK; 62 } 63 64 static void ccp_sg_free(struct ccp_sg_workarea *wa) 65 { 66 if (wa->dma_count) 67 dma_unmap_sg(wa->dma_dev, wa->dma_sg, wa->nents, wa->dma_dir); 68 69 wa->dma_count = 0; 70 } 71 72 static int ccp_init_sg_workarea(struct ccp_sg_workarea *wa, struct device *dev, 73 struct scatterlist *sg, u64 len, 74 enum dma_data_direction dma_dir) 75 { 76 memset(wa, 0, sizeof(*wa)); 77 78 wa->sg = sg; 79 if (!sg) 80 return 0; 81 82 wa->nents = sg_nents_for_len(sg, len); 83 if (wa->nents < 0) 84 return wa->nents; 85 86 wa->bytes_left = len; 87 wa->sg_used = 0; 88 89 if (len == 0) 90 return 0; 91 92 if (dma_dir == DMA_NONE) 93 return 0; 94 95 wa->dma_sg = sg; 96 wa->dma_dev = dev; 97 wa->dma_dir = dma_dir; 98 wa->dma_count = dma_map_sg(dev, sg, wa->nents, dma_dir); 99 if (!wa->dma_count) 100 return -ENOMEM; 101 102 return 0; 103 } 104 105 static void ccp_update_sg_workarea(struct ccp_sg_workarea *wa, unsigned int len) 106 { 107 unsigned int nbytes = min_t(u64, len, wa->bytes_left); 108 109 if (!wa->sg) 110 return; 111 112 wa->sg_used += nbytes; 113 wa->bytes_left -= nbytes; 114 if (wa->sg_used == wa->sg->length) { 115 wa->sg = sg_next(wa->sg); 116 wa->sg_used = 0; 117 } 118 } 119 120 static void ccp_dm_free(struct ccp_dm_workarea *wa) 121 { 122 if (wa->length <= CCP_DMAPOOL_MAX_SIZE) { 123 if (wa->address) 124 dma_pool_free(wa->dma_pool, wa->address, 125 wa->dma.address); 126 } else { 127 if (wa->dma.address) 128 dma_unmap_single(wa->dev, wa->dma.address, wa->length, 129 wa->dma.dir); 130 kfree(wa->address); 131 } 132 133 wa->address = NULL; 134 wa->dma.address = 0; 135 } 136 137 static int ccp_init_dm_workarea(struct ccp_dm_workarea *wa, 138 struct ccp_cmd_queue *cmd_q, 139 unsigned int len, 140 enum dma_data_direction dir) 141 { 142 memset(wa, 0, sizeof(*wa)); 143 144 if (!len) 145 return 0; 146 147 wa->dev = cmd_q->ccp->dev; 148 wa->length = len; 149 150 if (len <= CCP_DMAPOOL_MAX_SIZE) { 151 wa->dma_pool = cmd_q->dma_pool; 152 153 wa->address = dma_pool_alloc(wa->dma_pool, GFP_KERNEL, 154 &wa->dma.address); 155 if (!wa->address) 156 return -ENOMEM; 157 158 wa->dma.length = CCP_DMAPOOL_MAX_SIZE; 159 160 memset(wa->address, 0, CCP_DMAPOOL_MAX_SIZE); 161 } else { 162 wa->address = kzalloc(len, GFP_KERNEL); 163 if (!wa->address) 164 return -ENOMEM; 165 166 wa->dma.address = dma_map_single(wa->dev, wa->address, len, 167 dir); 168 if (dma_mapping_error(wa->dev, wa->dma.address)) 169 return -ENOMEM; 170 171 wa->dma.length = len; 172 } 173 wa->dma.dir = dir; 174 175 return 0; 176 } 177 178 static int ccp_set_dm_area(struct ccp_dm_workarea *wa, unsigned int wa_offset, 179 struct scatterlist *sg, unsigned int sg_offset, 180 unsigned int len) 181 { 182 WARN_ON(!wa->address); 183 184 if (len > (wa->length - wa_offset)) 185 return -EINVAL; 186 187 scatterwalk_map_and_copy(wa->address + wa_offset, sg, sg_offset, len, 188 0); 189 return 0; 190 } 191 192 static void ccp_get_dm_area(struct ccp_dm_workarea *wa, unsigned int wa_offset, 193 struct scatterlist *sg, unsigned int sg_offset, 194 unsigned int len) 195 { 196 WARN_ON(!wa->address); 197 198 scatterwalk_map_and_copy(wa->address + wa_offset, sg, sg_offset, len, 199 1); 200 } 201 202 static int ccp_reverse_set_dm_area(struct ccp_dm_workarea *wa, 203 unsigned int wa_offset, 204 struct scatterlist *sg, 205 unsigned int sg_offset, 206 unsigned int len) 207 { 208 u8 *p, *q; 209 int rc; 210 211 rc = ccp_set_dm_area(wa, wa_offset, sg, sg_offset, len); 212 if (rc) 213 return rc; 214 215 p = wa->address + wa_offset; 216 q = p + len - 1; 217 while (p < q) { 218 *p = *p ^ *q; 219 *q = *p ^ *q; 220 *p = *p ^ *q; 221 p++; 222 q--; 223 } 224 return 0; 225 } 226 227 static void ccp_reverse_get_dm_area(struct ccp_dm_workarea *wa, 228 unsigned int wa_offset, 229 struct scatterlist *sg, 230 unsigned int sg_offset, 231 unsigned int len) 232 { 233 u8 *p, *q; 234 235 p = wa->address + wa_offset; 236 q = p + len - 1; 237 while (p < q) { 238 *p = *p ^ *q; 239 *q = *p ^ *q; 240 *p = *p ^ *q; 241 p++; 242 q--; 243 } 244 245 ccp_get_dm_area(wa, wa_offset, sg, sg_offset, len); 246 } 247 248 static void ccp_free_data(struct ccp_data *data, struct ccp_cmd_queue *cmd_q) 249 { 250 ccp_dm_free(&data->dm_wa); 251 ccp_sg_free(&data->sg_wa); 252 } 253 254 static int ccp_init_data(struct ccp_data *data, struct ccp_cmd_queue *cmd_q, 255 struct scatterlist *sg, u64 sg_len, 256 unsigned int dm_len, 257 enum dma_data_direction dir) 258 { 259 int ret; 260 261 memset(data, 0, sizeof(*data)); 262 263 ret = ccp_init_sg_workarea(&data->sg_wa, cmd_q->ccp->dev, sg, sg_len, 264 dir); 265 if (ret) 266 goto e_err; 267 268 ret = ccp_init_dm_workarea(&data->dm_wa, cmd_q, dm_len, dir); 269 if (ret) 270 goto e_err; 271 272 return 0; 273 274 e_err: 275 ccp_free_data(data, cmd_q); 276 277 return ret; 278 } 279 280 static unsigned int ccp_queue_buf(struct ccp_data *data, unsigned int from) 281 { 282 struct ccp_sg_workarea *sg_wa = &data->sg_wa; 283 struct ccp_dm_workarea *dm_wa = &data->dm_wa; 284 unsigned int buf_count, nbytes; 285 286 /* Clear the buffer if setting it */ 287 if (!from) 288 memset(dm_wa->address, 0, dm_wa->length); 289 290 if (!sg_wa->sg) 291 return 0; 292 293 /* Perform the copy operation 294 * nbytes will always be <= UINT_MAX because dm_wa->length is 295 * an unsigned int 296 */ 297 nbytes = min_t(u64, sg_wa->bytes_left, dm_wa->length); 298 scatterwalk_map_and_copy(dm_wa->address, sg_wa->sg, sg_wa->sg_used, 299 nbytes, from); 300 301 /* Update the structures and generate the count */ 302 buf_count = 0; 303 while (sg_wa->bytes_left && (buf_count < dm_wa->length)) { 304 nbytes = min(sg_wa->sg->length - sg_wa->sg_used, 305 dm_wa->length - buf_count); 306 nbytes = min_t(u64, sg_wa->bytes_left, nbytes); 307 308 buf_count += nbytes; 309 ccp_update_sg_workarea(sg_wa, nbytes); 310 } 311 312 return buf_count; 313 } 314 315 static unsigned int ccp_fill_queue_buf(struct ccp_data *data) 316 { 317 return ccp_queue_buf(data, 0); 318 } 319 320 static unsigned int ccp_empty_queue_buf(struct ccp_data *data) 321 { 322 return ccp_queue_buf(data, 1); 323 } 324 325 static void ccp_prepare_data(struct ccp_data *src, struct ccp_data *dst, 326 struct ccp_op *op, unsigned int block_size, 327 bool blocksize_op) 328 { 329 unsigned int sg_src_len, sg_dst_len, op_len; 330 331 /* The CCP can only DMA from/to one address each per operation. This 332 * requires that we find the smallest DMA area between the source 333 * and destination. The resulting len values will always be <= UINT_MAX 334 * because the dma length is an unsigned int. 335 */ 336 sg_src_len = sg_dma_len(src->sg_wa.sg) - src->sg_wa.sg_used; 337 sg_src_len = min_t(u64, src->sg_wa.bytes_left, sg_src_len); 338 339 if (dst) { 340 sg_dst_len = sg_dma_len(dst->sg_wa.sg) - dst->sg_wa.sg_used; 341 sg_dst_len = min_t(u64, src->sg_wa.bytes_left, sg_dst_len); 342 op_len = min(sg_src_len, sg_dst_len); 343 } else { 344 op_len = sg_src_len; 345 } 346 347 /* The data operation length will be at least block_size in length 348 * or the smaller of available sg room remaining for the source or 349 * the destination 350 */ 351 op_len = max(op_len, block_size); 352 353 /* Unless we have to buffer data, there's no reason to wait */ 354 op->soc = 0; 355 356 if (sg_src_len < block_size) { 357 /* Not enough data in the sg element, so it 358 * needs to be buffered into a blocksize chunk 359 */ 360 int cp_len = ccp_fill_queue_buf(src); 361 362 op->soc = 1; 363 op->src.u.dma.address = src->dm_wa.dma.address; 364 op->src.u.dma.offset = 0; 365 op->src.u.dma.length = (blocksize_op) ? block_size : cp_len; 366 } else { 367 /* Enough data in the sg element, but we need to 368 * adjust for any previously copied data 369 */ 370 op->src.u.dma.address = sg_dma_address(src->sg_wa.sg); 371 op->src.u.dma.offset = src->sg_wa.sg_used; 372 op->src.u.dma.length = op_len & ~(block_size - 1); 373 374 ccp_update_sg_workarea(&src->sg_wa, op->src.u.dma.length); 375 } 376 377 if (dst) { 378 if (sg_dst_len < block_size) { 379 /* Not enough room in the sg element or we're on the 380 * last piece of data (when using padding), so the 381 * output needs to be buffered into a blocksize chunk 382 */ 383 op->soc = 1; 384 op->dst.u.dma.address = dst->dm_wa.dma.address; 385 op->dst.u.dma.offset = 0; 386 op->dst.u.dma.length = op->src.u.dma.length; 387 } else { 388 /* Enough room in the sg element, but we need to 389 * adjust for any previously used area 390 */ 391 op->dst.u.dma.address = sg_dma_address(dst->sg_wa.sg); 392 op->dst.u.dma.offset = dst->sg_wa.sg_used; 393 op->dst.u.dma.length = op->src.u.dma.length; 394 } 395 } 396 } 397 398 static void ccp_process_data(struct ccp_data *src, struct ccp_data *dst, 399 struct ccp_op *op) 400 { 401 op->init = 0; 402 403 if (dst) { 404 if (op->dst.u.dma.address == dst->dm_wa.dma.address) 405 ccp_empty_queue_buf(dst); 406 else 407 ccp_update_sg_workarea(&dst->sg_wa, 408 op->dst.u.dma.length); 409 } 410 } 411 412 static int ccp_copy_to_from_sb(struct ccp_cmd_queue *cmd_q, 413 struct ccp_dm_workarea *wa, u32 jobid, u32 sb, 414 u32 byte_swap, bool from) 415 { 416 struct ccp_op op; 417 418 memset(&op, 0, sizeof(op)); 419 420 op.cmd_q = cmd_q; 421 op.jobid = jobid; 422 op.eom = 1; 423 424 if (from) { 425 op.soc = 1; 426 op.src.type = CCP_MEMTYPE_SB; 427 op.src.u.sb = sb; 428 op.dst.type = CCP_MEMTYPE_SYSTEM; 429 op.dst.u.dma.address = wa->dma.address; 430 op.dst.u.dma.length = wa->length; 431 } else { 432 op.src.type = CCP_MEMTYPE_SYSTEM; 433 op.src.u.dma.address = wa->dma.address; 434 op.src.u.dma.length = wa->length; 435 op.dst.type = CCP_MEMTYPE_SB; 436 op.dst.u.sb = sb; 437 } 438 439 op.u.passthru.byte_swap = byte_swap; 440 441 return cmd_q->ccp->vdata->perform->passthru(&op); 442 } 443 444 static int ccp_copy_to_sb(struct ccp_cmd_queue *cmd_q, 445 struct ccp_dm_workarea *wa, u32 jobid, u32 sb, 446 u32 byte_swap) 447 { 448 return ccp_copy_to_from_sb(cmd_q, wa, jobid, sb, byte_swap, false); 449 } 450 451 static int ccp_copy_from_sb(struct ccp_cmd_queue *cmd_q, 452 struct ccp_dm_workarea *wa, u32 jobid, u32 sb, 453 u32 byte_swap) 454 { 455 return ccp_copy_to_from_sb(cmd_q, wa, jobid, sb, byte_swap, true); 456 } 457 458 static int ccp_run_aes_cmac_cmd(struct ccp_cmd_queue *cmd_q, 459 struct ccp_cmd *cmd) 460 { 461 struct ccp_aes_engine *aes = &cmd->u.aes; 462 struct ccp_dm_workarea key, ctx; 463 struct ccp_data src; 464 struct ccp_op op; 465 unsigned int dm_offset; 466 int ret; 467 468 if (!((aes->key_len == AES_KEYSIZE_128) || 469 (aes->key_len == AES_KEYSIZE_192) || 470 (aes->key_len == AES_KEYSIZE_256))) 471 return -EINVAL; 472 473 if (aes->src_len & (AES_BLOCK_SIZE - 1)) 474 return -EINVAL; 475 476 if (aes->iv_len != AES_BLOCK_SIZE) 477 return -EINVAL; 478 479 if (!aes->key || !aes->iv || !aes->src) 480 return -EINVAL; 481 482 if (aes->cmac_final) { 483 if (aes->cmac_key_len != AES_BLOCK_SIZE) 484 return -EINVAL; 485 486 if (!aes->cmac_key) 487 return -EINVAL; 488 } 489 490 BUILD_BUG_ON(CCP_AES_KEY_SB_COUNT != 1); 491 BUILD_BUG_ON(CCP_AES_CTX_SB_COUNT != 1); 492 493 ret = -EIO; 494 memset(&op, 0, sizeof(op)); 495 op.cmd_q = cmd_q; 496 op.jobid = CCP_NEW_JOBID(cmd_q->ccp); 497 op.sb_key = cmd_q->sb_key; 498 op.sb_ctx = cmd_q->sb_ctx; 499 op.init = 1; 500 op.u.aes.type = aes->type; 501 op.u.aes.mode = aes->mode; 502 op.u.aes.action = aes->action; 503 504 /* All supported key sizes fit in a single (32-byte) SB entry 505 * and must be in little endian format. Use the 256-bit byte 506 * swap passthru option to convert from big endian to little 507 * endian. 508 */ 509 ret = ccp_init_dm_workarea(&key, cmd_q, 510 CCP_AES_KEY_SB_COUNT * CCP_SB_BYTES, 511 DMA_TO_DEVICE); 512 if (ret) 513 return ret; 514 515 dm_offset = CCP_SB_BYTES - aes->key_len; 516 ret = ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len); 517 if (ret) 518 goto e_key; 519 ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key, 520 CCP_PASSTHRU_BYTESWAP_256BIT); 521 if (ret) { 522 cmd->engine_error = cmd_q->cmd_error; 523 goto e_key; 524 } 525 526 /* The AES context fits in a single (32-byte) SB entry and 527 * must be in little endian format. Use the 256-bit byte swap 528 * passthru option to convert from big endian to little endian. 529 */ 530 ret = ccp_init_dm_workarea(&ctx, cmd_q, 531 CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES, 532 DMA_BIDIRECTIONAL); 533 if (ret) 534 goto e_key; 535 536 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE; 537 ret = ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len); 538 if (ret) 539 goto e_ctx; 540 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, 541 CCP_PASSTHRU_BYTESWAP_256BIT); 542 if (ret) { 543 cmd->engine_error = cmd_q->cmd_error; 544 goto e_ctx; 545 } 546 547 /* Send data to the CCP AES engine */ 548 ret = ccp_init_data(&src, cmd_q, aes->src, aes->src_len, 549 AES_BLOCK_SIZE, DMA_TO_DEVICE); 550 if (ret) 551 goto e_ctx; 552 553 while (src.sg_wa.bytes_left) { 554 ccp_prepare_data(&src, NULL, &op, AES_BLOCK_SIZE, true); 555 if (aes->cmac_final && !src.sg_wa.bytes_left) { 556 op.eom = 1; 557 558 /* Push the K1/K2 key to the CCP now */ 559 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, 560 op.sb_ctx, 561 CCP_PASSTHRU_BYTESWAP_256BIT); 562 if (ret) { 563 cmd->engine_error = cmd_q->cmd_error; 564 goto e_src; 565 } 566 567 ret = ccp_set_dm_area(&ctx, 0, aes->cmac_key, 0, 568 aes->cmac_key_len); 569 if (ret) 570 goto e_src; 571 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, 572 CCP_PASSTHRU_BYTESWAP_256BIT); 573 if (ret) { 574 cmd->engine_error = cmd_q->cmd_error; 575 goto e_src; 576 } 577 } 578 579 ret = cmd_q->ccp->vdata->perform->aes(&op); 580 if (ret) { 581 cmd->engine_error = cmd_q->cmd_error; 582 goto e_src; 583 } 584 585 ccp_process_data(&src, NULL, &op); 586 } 587 588 /* Retrieve the AES context - convert from LE to BE using 589 * 32-byte (256-bit) byteswapping 590 */ 591 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, 592 CCP_PASSTHRU_BYTESWAP_256BIT); 593 if (ret) { 594 cmd->engine_error = cmd_q->cmd_error; 595 goto e_src; 596 } 597 598 /* ...but we only need AES_BLOCK_SIZE bytes */ 599 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE; 600 ccp_get_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len); 601 602 e_src: 603 ccp_free_data(&src, cmd_q); 604 605 e_ctx: 606 ccp_dm_free(&ctx); 607 608 e_key: 609 ccp_dm_free(&key); 610 611 return ret; 612 } 613 614 static int ccp_run_aes_gcm_cmd(struct ccp_cmd_queue *cmd_q, 615 struct ccp_cmd *cmd) 616 { 617 struct ccp_aes_engine *aes = &cmd->u.aes; 618 struct ccp_dm_workarea key, ctx, final_wa, tag; 619 struct ccp_data src, dst; 620 struct ccp_data aad; 621 struct ccp_op op; 622 623 unsigned long long *final; 624 unsigned int dm_offset; 625 unsigned int authsize; 626 unsigned int jobid; 627 unsigned int ilen; 628 bool in_place = true; /* Default value */ 629 int ret; 630 631 struct scatterlist *p_inp, sg_inp[2]; 632 struct scatterlist *p_tag, sg_tag[2]; 633 struct scatterlist *p_outp, sg_outp[2]; 634 struct scatterlist *p_aad; 635 636 if (!aes->iv) 637 return -EINVAL; 638 639 if (!((aes->key_len == AES_KEYSIZE_128) || 640 (aes->key_len == AES_KEYSIZE_192) || 641 (aes->key_len == AES_KEYSIZE_256))) 642 return -EINVAL; 643 644 if (!aes->key) /* Gotta have a key SGL */ 645 return -EINVAL; 646 647 /* Zero defaults to 16 bytes, the maximum size */ 648 authsize = aes->authsize ? aes->authsize : AES_BLOCK_SIZE; 649 switch (authsize) { 650 case 16: 651 case 15: 652 case 14: 653 case 13: 654 case 12: 655 case 8: 656 case 4: 657 break; 658 default: 659 return -EINVAL; 660 } 661 662 /* First, decompose the source buffer into AAD & PT, 663 * and the destination buffer into AAD, CT & tag, or 664 * the input into CT & tag. 665 * It is expected that the input and output SGs will 666 * be valid, even if the AAD and input lengths are 0. 667 */ 668 p_aad = aes->src; 669 p_inp = scatterwalk_ffwd(sg_inp, aes->src, aes->aad_len); 670 p_outp = scatterwalk_ffwd(sg_outp, aes->dst, aes->aad_len); 671 if (aes->action == CCP_AES_ACTION_ENCRYPT) { 672 ilen = aes->src_len; 673 p_tag = scatterwalk_ffwd(sg_tag, p_outp, ilen); 674 } else { 675 /* Input length for decryption includes tag */ 676 ilen = aes->src_len - authsize; 677 p_tag = scatterwalk_ffwd(sg_tag, p_inp, ilen); 678 } 679 680 jobid = CCP_NEW_JOBID(cmd_q->ccp); 681 682 memset(&op, 0, sizeof(op)); 683 op.cmd_q = cmd_q; 684 op.jobid = jobid; 685 op.sb_key = cmd_q->sb_key; /* Pre-allocated */ 686 op.sb_ctx = cmd_q->sb_ctx; /* Pre-allocated */ 687 op.init = 1; 688 op.u.aes.type = aes->type; 689 690 /* Copy the key to the LSB */ 691 ret = ccp_init_dm_workarea(&key, cmd_q, 692 CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES, 693 DMA_TO_DEVICE); 694 if (ret) 695 return ret; 696 697 dm_offset = CCP_SB_BYTES - aes->key_len; 698 ret = ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len); 699 if (ret) 700 goto e_key; 701 ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key, 702 CCP_PASSTHRU_BYTESWAP_256BIT); 703 if (ret) { 704 cmd->engine_error = cmd_q->cmd_error; 705 goto e_key; 706 } 707 708 /* Copy the context (IV) to the LSB. 709 * There is an assumption here that the IV is 96 bits in length, plus 710 * a nonce of 32 bits. If no IV is present, use a zeroed buffer. 711 */ 712 ret = ccp_init_dm_workarea(&ctx, cmd_q, 713 CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES, 714 DMA_BIDIRECTIONAL); 715 if (ret) 716 goto e_key; 717 718 dm_offset = CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES - aes->iv_len; 719 ret = ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len); 720 if (ret) 721 goto e_ctx; 722 723 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, 724 CCP_PASSTHRU_BYTESWAP_256BIT); 725 if (ret) { 726 cmd->engine_error = cmd_q->cmd_error; 727 goto e_ctx; 728 } 729 730 op.init = 1; 731 if (aes->aad_len > 0) { 732 /* Step 1: Run a GHASH over the Additional Authenticated Data */ 733 ret = ccp_init_data(&aad, cmd_q, p_aad, aes->aad_len, 734 AES_BLOCK_SIZE, 735 DMA_TO_DEVICE); 736 if (ret) 737 goto e_ctx; 738 739 op.u.aes.mode = CCP_AES_MODE_GHASH; 740 op.u.aes.action = CCP_AES_GHASHAAD; 741 742 while (aad.sg_wa.bytes_left) { 743 ccp_prepare_data(&aad, NULL, &op, AES_BLOCK_SIZE, true); 744 745 ret = cmd_q->ccp->vdata->perform->aes(&op); 746 if (ret) { 747 cmd->engine_error = cmd_q->cmd_error; 748 goto e_aad; 749 } 750 751 ccp_process_data(&aad, NULL, &op); 752 op.init = 0; 753 } 754 } 755 756 op.u.aes.mode = CCP_AES_MODE_GCTR; 757 op.u.aes.action = aes->action; 758 759 if (ilen > 0) { 760 /* Step 2: Run a GCTR over the plaintext */ 761 in_place = (sg_virt(p_inp) == sg_virt(p_outp)) ? true : false; 762 763 ret = ccp_init_data(&src, cmd_q, p_inp, ilen, 764 AES_BLOCK_SIZE, 765 in_place ? DMA_BIDIRECTIONAL 766 : DMA_TO_DEVICE); 767 if (ret) 768 goto e_ctx; 769 770 if (in_place) { 771 dst = src; 772 } else { 773 ret = ccp_init_data(&dst, cmd_q, p_outp, ilen, 774 AES_BLOCK_SIZE, DMA_FROM_DEVICE); 775 if (ret) 776 goto e_src; 777 } 778 779 op.soc = 0; 780 op.eom = 0; 781 op.init = 1; 782 while (src.sg_wa.bytes_left) { 783 ccp_prepare_data(&src, &dst, &op, AES_BLOCK_SIZE, true); 784 if (!src.sg_wa.bytes_left) { 785 unsigned int nbytes = ilen % AES_BLOCK_SIZE; 786 787 if (nbytes) { 788 op.eom = 1; 789 op.u.aes.size = (nbytes * 8) - 1; 790 } 791 } 792 793 ret = cmd_q->ccp->vdata->perform->aes(&op); 794 if (ret) { 795 cmd->engine_error = cmd_q->cmd_error; 796 goto e_dst; 797 } 798 799 ccp_process_data(&src, &dst, &op); 800 op.init = 0; 801 } 802 } 803 804 /* Step 3: Update the IV portion of the context with the original IV */ 805 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, 806 CCP_PASSTHRU_BYTESWAP_256BIT); 807 if (ret) { 808 cmd->engine_error = cmd_q->cmd_error; 809 goto e_dst; 810 } 811 812 ret = ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len); 813 if (ret) 814 goto e_dst; 815 816 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, 817 CCP_PASSTHRU_BYTESWAP_256BIT); 818 if (ret) { 819 cmd->engine_error = cmd_q->cmd_error; 820 goto e_dst; 821 } 822 823 /* Step 4: Concatenate the lengths of the AAD and source, and 824 * hash that 16 byte buffer. 825 */ 826 ret = ccp_init_dm_workarea(&final_wa, cmd_q, AES_BLOCK_SIZE, 827 DMA_BIDIRECTIONAL); 828 if (ret) 829 goto e_dst; 830 final = (unsigned long long *) final_wa.address; 831 final[0] = cpu_to_be64(aes->aad_len * 8); 832 final[1] = cpu_to_be64(ilen * 8); 833 834 memset(&op, 0, sizeof(op)); 835 op.cmd_q = cmd_q; 836 op.jobid = jobid; 837 op.sb_key = cmd_q->sb_key; /* Pre-allocated */ 838 op.sb_ctx = cmd_q->sb_ctx; /* Pre-allocated */ 839 op.init = 1; 840 op.u.aes.type = aes->type; 841 op.u.aes.mode = CCP_AES_MODE_GHASH; 842 op.u.aes.action = CCP_AES_GHASHFINAL; 843 op.src.type = CCP_MEMTYPE_SYSTEM; 844 op.src.u.dma.address = final_wa.dma.address; 845 op.src.u.dma.length = AES_BLOCK_SIZE; 846 op.dst.type = CCP_MEMTYPE_SYSTEM; 847 op.dst.u.dma.address = final_wa.dma.address; 848 op.dst.u.dma.length = AES_BLOCK_SIZE; 849 op.eom = 1; 850 op.u.aes.size = 0; 851 ret = cmd_q->ccp->vdata->perform->aes(&op); 852 if (ret) 853 goto e_dst; 854 855 if (aes->action == CCP_AES_ACTION_ENCRYPT) { 856 /* Put the ciphered tag after the ciphertext. */ 857 ccp_get_dm_area(&final_wa, 0, p_tag, 0, authsize); 858 } else { 859 /* Does this ciphered tag match the input? */ 860 ret = ccp_init_dm_workarea(&tag, cmd_q, authsize, 861 DMA_BIDIRECTIONAL); 862 if (ret) 863 goto e_tag; 864 ret = ccp_set_dm_area(&tag, 0, p_tag, 0, authsize); 865 if (ret) 866 goto e_tag; 867 868 ret = crypto_memneq(tag.address, final_wa.address, 869 authsize) ? -EBADMSG : 0; 870 ccp_dm_free(&tag); 871 } 872 873 e_tag: 874 ccp_dm_free(&final_wa); 875 876 e_dst: 877 if (ilen > 0 && !in_place) 878 ccp_free_data(&dst, cmd_q); 879 880 e_src: 881 if (ilen > 0) 882 ccp_free_data(&src, cmd_q); 883 884 e_aad: 885 if (aes->aad_len) 886 ccp_free_data(&aad, cmd_q); 887 888 e_ctx: 889 ccp_dm_free(&ctx); 890 891 e_key: 892 ccp_dm_free(&key); 893 894 return ret; 895 } 896 897 static int ccp_run_aes_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) 898 { 899 struct ccp_aes_engine *aes = &cmd->u.aes; 900 struct ccp_dm_workarea key, ctx; 901 struct ccp_data src, dst; 902 struct ccp_op op; 903 unsigned int dm_offset; 904 bool in_place = false; 905 int ret; 906 907 if (aes->mode == CCP_AES_MODE_CMAC) 908 return ccp_run_aes_cmac_cmd(cmd_q, cmd); 909 910 if (aes->mode == CCP_AES_MODE_GCM) 911 return ccp_run_aes_gcm_cmd(cmd_q, cmd); 912 913 if (!((aes->key_len == AES_KEYSIZE_128) || 914 (aes->key_len == AES_KEYSIZE_192) || 915 (aes->key_len == AES_KEYSIZE_256))) 916 return -EINVAL; 917 918 if (((aes->mode == CCP_AES_MODE_ECB) || 919 (aes->mode == CCP_AES_MODE_CBC)) && 920 (aes->src_len & (AES_BLOCK_SIZE - 1))) 921 return -EINVAL; 922 923 if (!aes->key || !aes->src || !aes->dst) 924 return -EINVAL; 925 926 if (aes->mode != CCP_AES_MODE_ECB) { 927 if (aes->iv_len != AES_BLOCK_SIZE) 928 return -EINVAL; 929 930 if (!aes->iv) 931 return -EINVAL; 932 } 933 934 BUILD_BUG_ON(CCP_AES_KEY_SB_COUNT != 1); 935 BUILD_BUG_ON(CCP_AES_CTX_SB_COUNT != 1); 936 937 ret = -EIO; 938 memset(&op, 0, sizeof(op)); 939 op.cmd_q = cmd_q; 940 op.jobid = CCP_NEW_JOBID(cmd_q->ccp); 941 op.sb_key = cmd_q->sb_key; 942 op.sb_ctx = cmd_q->sb_ctx; 943 op.init = (aes->mode == CCP_AES_MODE_ECB) ? 0 : 1; 944 op.u.aes.type = aes->type; 945 op.u.aes.mode = aes->mode; 946 op.u.aes.action = aes->action; 947 948 /* All supported key sizes fit in a single (32-byte) SB entry 949 * and must be in little endian format. Use the 256-bit byte 950 * swap passthru option to convert from big endian to little 951 * endian. 952 */ 953 ret = ccp_init_dm_workarea(&key, cmd_q, 954 CCP_AES_KEY_SB_COUNT * CCP_SB_BYTES, 955 DMA_TO_DEVICE); 956 if (ret) 957 return ret; 958 959 dm_offset = CCP_SB_BYTES - aes->key_len; 960 ret = ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len); 961 if (ret) 962 goto e_key; 963 ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key, 964 CCP_PASSTHRU_BYTESWAP_256BIT); 965 if (ret) { 966 cmd->engine_error = cmd_q->cmd_error; 967 goto e_key; 968 } 969 970 /* The AES context fits in a single (32-byte) SB entry and 971 * must be in little endian format. Use the 256-bit byte swap 972 * passthru option to convert from big endian to little endian. 973 */ 974 ret = ccp_init_dm_workarea(&ctx, cmd_q, 975 CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES, 976 DMA_BIDIRECTIONAL); 977 if (ret) 978 goto e_key; 979 980 if (aes->mode != CCP_AES_MODE_ECB) { 981 /* Load the AES context - convert to LE */ 982 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE; 983 ret = ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len); 984 if (ret) 985 goto e_ctx; 986 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, 987 CCP_PASSTHRU_BYTESWAP_256BIT); 988 if (ret) { 989 cmd->engine_error = cmd_q->cmd_error; 990 goto e_ctx; 991 } 992 } 993 switch (aes->mode) { 994 case CCP_AES_MODE_CFB: /* CFB128 only */ 995 case CCP_AES_MODE_CTR: 996 op.u.aes.size = AES_BLOCK_SIZE * BITS_PER_BYTE - 1; 997 break; 998 default: 999 op.u.aes.size = 0; 1000 } 1001 1002 /* Prepare the input and output data workareas. For in-place 1003 * operations we need to set the dma direction to BIDIRECTIONAL 1004 * and copy the src workarea to the dst workarea. 1005 */ 1006 if (sg_virt(aes->src) == sg_virt(aes->dst)) 1007 in_place = true; 1008 1009 ret = ccp_init_data(&src, cmd_q, aes->src, aes->src_len, 1010 AES_BLOCK_SIZE, 1011 in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE); 1012 if (ret) 1013 goto e_ctx; 1014 1015 if (in_place) { 1016 dst = src; 1017 } else { 1018 ret = ccp_init_data(&dst, cmd_q, aes->dst, aes->src_len, 1019 AES_BLOCK_SIZE, DMA_FROM_DEVICE); 1020 if (ret) 1021 goto e_src; 1022 } 1023 1024 /* Send data to the CCP AES engine */ 1025 while (src.sg_wa.bytes_left) { 1026 ccp_prepare_data(&src, &dst, &op, AES_BLOCK_SIZE, true); 1027 if (!src.sg_wa.bytes_left) { 1028 op.eom = 1; 1029 1030 /* Since we don't retrieve the AES context in ECB 1031 * mode we have to wait for the operation to complete 1032 * on the last piece of data 1033 */ 1034 if (aes->mode == CCP_AES_MODE_ECB) 1035 op.soc = 1; 1036 } 1037 1038 ret = cmd_q->ccp->vdata->perform->aes(&op); 1039 if (ret) { 1040 cmd->engine_error = cmd_q->cmd_error; 1041 goto e_dst; 1042 } 1043 1044 ccp_process_data(&src, &dst, &op); 1045 } 1046 1047 if (aes->mode != CCP_AES_MODE_ECB) { 1048 /* Retrieve the AES context - convert from LE to BE using 1049 * 32-byte (256-bit) byteswapping 1050 */ 1051 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, 1052 CCP_PASSTHRU_BYTESWAP_256BIT); 1053 if (ret) { 1054 cmd->engine_error = cmd_q->cmd_error; 1055 goto e_dst; 1056 } 1057 1058 /* ...but we only need AES_BLOCK_SIZE bytes */ 1059 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE; 1060 ccp_get_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len); 1061 } 1062 1063 e_dst: 1064 if (!in_place) 1065 ccp_free_data(&dst, cmd_q); 1066 1067 e_src: 1068 ccp_free_data(&src, cmd_q); 1069 1070 e_ctx: 1071 ccp_dm_free(&ctx); 1072 1073 e_key: 1074 ccp_dm_free(&key); 1075 1076 return ret; 1077 } 1078 1079 static int ccp_run_xts_aes_cmd(struct ccp_cmd_queue *cmd_q, 1080 struct ccp_cmd *cmd) 1081 { 1082 struct ccp_xts_aes_engine *xts = &cmd->u.xts; 1083 struct ccp_dm_workarea key, ctx; 1084 struct ccp_data src, dst; 1085 struct ccp_op op; 1086 unsigned int unit_size, dm_offset; 1087 bool in_place = false; 1088 unsigned int sb_count; 1089 enum ccp_aes_type aestype; 1090 int ret; 1091 1092 switch (xts->unit_size) { 1093 case CCP_XTS_AES_UNIT_SIZE_16: 1094 unit_size = 16; 1095 break; 1096 case CCP_XTS_AES_UNIT_SIZE_512: 1097 unit_size = 512; 1098 break; 1099 case CCP_XTS_AES_UNIT_SIZE_1024: 1100 unit_size = 1024; 1101 break; 1102 case CCP_XTS_AES_UNIT_SIZE_2048: 1103 unit_size = 2048; 1104 break; 1105 case CCP_XTS_AES_UNIT_SIZE_4096: 1106 unit_size = 4096; 1107 break; 1108 1109 default: 1110 return -EINVAL; 1111 } 1112 1113 if (xts->key_len == AES_KEYSIZE_128) 1114 aestype = CCP_AES_TYPE_128; 1115 else if (xts->key_len == AES_KEYSIZE_256) 1116 aestype = CCP_AES_TYPE_256; 1117 else 1118 return -EINVAL; 1119 1120 if (!xts->final && (xts->src_len & (AES_BLOCK_SIZE - 1))) 1121 return -EINVAL; 1122 1123 if (xts->iv_len != AES_BLOCK_SIZE) 1124 return -EINVAL; 1125 1126 if (!xts->key || !xts->iv || !xts->src || !xts->dst) 1127 return -EINVAL; 1128 1129 BUILD_BUG_ON(CCP_XTS_AES_KEY_SB_COUNT != 1); 1130 BUILD_BUG_ON(CCP_XTS_AES_CTX_SB_COUNT != 1); 1131 1132 ret = -EIO; 1133 memset(&op, 0, sizeof(op)); 1134 op.cmd_q = cmd_q; 1135 op.jobid = CCP_NEW_JOBID(cmd_q->ccp); 1136 op.sb_key = cmd_q->sb_key; 1137 op.sb_ctx = cmd_q->sb_ctx; 1138 op.init = 1; 1139 op.u.xts.type = aestype; 1140 op.u.xts.action = xts->action; 1141 op.u.xts.unit_size = xts->unit_size; 1142 1143 /* A version 3 device only supports 128-bit keys, which fits into a 1144 * single SB entry. A version 5 device uses a 512-bit vector, so two 1145 * SB entries. 1146 */ 1147 if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0)) 1148 sb_count = CCP_XTS_AES_KEY_SB_COUNT; 1149 else 1150 sb_count = CCP5_XTS_AES_KEY_SB_COUNT; 1151 ret = ccp_init_dm_workarea(&key, cmd_q, 1152 sb_count * CCP_SB_BYTES, 1153 DMA_TO_DEVICE); 1154 if (ret) 1155 return ret; 1156 1157 if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0)) { 1158 /* All supported key sizes must be in little endian format. 1159 * Use the 256-bit byte swap passthru option to convert from 1160 * big endian to little endian. 1161 */ 1162 dm_offset = CCP_SB_BYTES - AES_KEYSIZE_128; 1163 ret = ccp_set_dm_area(&key, dm_offset, xts->key, 0, xts->key_len); 1164 if (ret) 1165 goto e_key; 1166 ret = ccp_set_dm_area(&key, 0, xts->key, xts->key_len, xts->key_len); 1167 if (ret) 1168 goto e_key; 1169 } else { 1170 /* Version 5 CCPs use a 512-bit space for the key: each portion 1171 * occupies 256 bits, or one entire slot, and is zero-padded. 1172 */ 1173 unsigned int pad; 1174 1175 dm_offset = CCP_SB_BYTES; 1176 pad = dm_offset - xts->key_len; 1177 ret = ccp_set_dm_area(&key, pad, xts->key, 0, xts->key_len); 1178 if (ret) 1179 goto e_key; 1180 ret = ccp_set_dm_area(&key, dm_offset + pad, xts->key, 1181 xts->key_len, xts->key_len); 1182 if (ret) 1183 goto e_key; 1184 } 1185 ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key, 1186 CCP_PASSTHRU_BYTESWAP_256BIT); 1187 if (ret) { 1188 cmd->engine_error = cmd_q->cmd_error; 1189 goto e_key; 1190 } 1191 1192 /* The AES context fits in a single (32-byte) SB entry and 1193 * for XTS is already in little endian format so no byte swapping 1194 * is needed. 1195 */ 1196 ret = ccp_init_dm_workarea(&ctx, cmd_q, 1197 CCP_XTS_AES_CTX_SB_COUNT * CCP_SB_BYTES, 1198 DMA_BIDIRECTIONAL); 1199 if (ret) 1200 goto e_key; 1201 1202 ret = ccp_set_dm_area(&ctx, 0, xts->iv, 0, xts->iv_len); 1203 if (ret) 1204 goto e_ctx; 1205 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, 1206 CCP_PASSTHRU_BYTESWAP_NOOP); 1207 if (ret) { 1208 cmd->engine_error = cmd_q->cmd_error; 1209 goto e_ctx; 1210 } 1211 1212 /* Prepare the input and output data workareas. For in-place 1213 * operations we need to set the dma direction to BIDIRECTIONAL 1214 * and copy the src workarea to the dst workarea. 1215 */ 1216 if (sg_virt(xts->src) == sg_virt(xts->dst)) 1217 in_place = true; 1218 1219 ret = ccp_init_data(&src, cmd_q, xts->src, xts->src_len, 1220 unit_size, 1221 in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE); 1222 if (ret) 1223 goto e_ctx; 1224 1225 if (in_place) { 1226 dst = src; 1227 } else { 1228 ret = ccp_init_data(&dst, cmd_q, xts->dst, xts->src_len, 1229 unit_size, DMA_FROM_DEVICE); 1230 if (ret) 1231 goto e_src; 1232 } 1233 1234 /* Send data to the CCP AES engine */ 1235 while (src.sg_wa.bytes_left) { 1236 ccp_prepare_data(&src, &dst, &op, unit_size, true); 1237 if (!src.sg_wa.bytes_left) 1238 op.eom = 1; 1239 1240 ret = cmd_q->ccp->vdata->perform->xts_aes(&op); 1241 if (ret) { 1242 cmd->engine_error = cmd_q->cmd_error; 1243 goto e_dst; 1244 } 1245 1246 ccp_process_data(&src, &dst, &op); 1247 } 1248 1249 /* Retrieve the AES context - convert from LE to BE using 1250 * 32-byte (256-bit) byteswapping 1251 */ 1252 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, 1253 CCP_PASSTHRU_BYTESWAP_256BIT); 1254 if (ret) { 1255 cmd->engine_error = cmd_q->cmd_error; 1256 goto e_dst; 1257 } 1258 1259 /* ...but we only need AES_BLOCK_SIZE bytes */ 1260 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE; 1261 ccp_get_dm_area(&ctx, dm_offset, xts->iv, 0, xts->iv_len); 1262 1263 e_dst: 1264 if (!in_place) 1265 ccp_free_data(&dst, cmd_q); 1266 1267 e_src: 1268 ccp_free_data(&src, cmd_q); 1269 1270 e_ctx: 1271 ccp_dm_free(&ctx); 1272 1273 e_key: 1274 ccp_dm_free(&key); 1275 1276 return ret; 1277 } 1278 1279 static int ccp_run_des3_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) 1280 { 1281 struct ccp_des3_engine *des3 = &cmd->u.des3; 1282 1283 struct ccp_dm_workarea key, ctx; 1284 struct ccp_data src, dst; 1285 struct ccp_op op; 1286 unsigned int dm_offset; 1287 unsigned int len_singlekey; 1288 bool in_place = false; 1289 int ret; 1290 1291 /* Error checks */ 1292 if (cmd_q->ccp->vdata->version < CCP_VERSION(5, 0)) 1293 return -EINVAL; 1294 1295 if (!cmd_q->ccp->vdata->perform->des3) 1296 return -EINVAL; 1297 1298 if (des3->key_len != DES3_EDE_KEY_SIZE) 1299 return -EINVAL; 1300 1301 if (((des3->mode == CCP_DES3_MODE_ECB) || 1302 (des3->mode == CCP_DES3_MODE_CBC)) && 1303 (des3->src_len & (DES3_EDE_BLOCK_SIZE - 1))) 1304 return -EINVAL; 1305 1306 if (!des3->key || !des3->src || !des3->dst) 1307 return -EINVAL; 1308 1309 if (des3->mode != CCP_DES3_MODE_ECB) { 1310 if (des3->iv_len != DES3_EDE_BLOCK_SIZE) 1311 return -EINVAL; 1312 1313 if (!des3->iv) 1314 return -EINVAL; 1315 } 1316 1317 ret = -EIO; 1318 /* Zero out all the fields of the command desc */ 1319 memset(&op, 0, sizeof(op)); 1320 1321 /* Set up the Function field */ 1322 op.cmd_q = cmd_q; 1323 op.jobid = CCP_NEW_JOBID(cmd_q->ccp); 1324 op.sb_key = cmd_q->sb_key; 1325 1326 op.init = (des3->mode == CCP_DES3_MODE_ECB) ? 0 : 1; 1327 op.u.des3.type = des3->type; 1328 op.u.des3.mode = des3->mode; 1329 op.u.des3.action = des3->action; 1330 1331 /* 1332 * All supported key sizes fit in a single (32-byte) KSB entry and 1333 * (like AES) must be in little endian format. Use the 256-bit byte 1334 * swap passthru option to convert from big endian to little endian. 1335 */ 1336 ret = ccp_init_dm_workarea(&key, cmd_q, 1337 CCP_DES3_KEY_SB_COUNT * CCP_SB_BYTES, 1338 DMA_TO_DEVICE); 1339 if (ret) 1340 return ret; 1341 1342 /* 1343 * The contents of the key triplet are in the reverse order of what 1344 * is required by the engine. Copy the 3 pieces individually to put 1345 * them where they belong. 1346 */ 1347 dm_offset = CCP_SB_BYTES - des3->key_len; /* Basic offset */ 1348 1349 len_singlekey = des3->key_len / 3; 1350 ret = ccp_set_dm_area(&key, dm_offset + 2 * len_singlekey, 1351 des3->key, 0, len_singlekey); 1352 if (ret) 1353 goto e_key; 1354 ret = ccp_set_dm_area(&key, dm_offset + len_singlekey, 1355 des3->key, len_singlekey, len_singlekey); 1356 if (ret) 1357 goto e_key; 1358 ret = ccp_set_dm_area(&key, dm_offset, 1359 des3->key, 2 * len_singlekey, len_singlekey); 1360 if (ret) 1361 goto e_key; 1362 1363 /* Copy the key to the SB */ 1364 ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key, 1365 CCP_PASSTHRU_BYTESWAP_256BIT); 1366 if (ret) { 1367 cmd->engine_error = cmd_q->cmd_error; 1368 goto e_key; 1369 } 1370 1371 /* 1372 * The DES3 context fits in a single (32-byte) KSB entry and 1373 * must be in little endian format. Use the 256-bit byte swap 1374 * passthru option to convert from big endian to little endian. 1375 */ 1376 if (des3->mode != CCP_DES3_MODE_ECB) { 1377 op.sb_ctx = cmd_q->sb_ctx; 1378 1379 ret = ccp_init_dm_workarea(&ctx, cmd_q, 1380 CCP_DES3_CTX_SB_COUNT * CCP_SB_BYTES, 1381 DMA_BIDIRECTIONAL); 1382 if (ret) 1383 goto e_key; 1384 1385 /* Load the context into the LSB */ 1386 dm_offset = CCP_SB_BYTES - des3->iv_len; 1387 ret = ccp_set_dm_area(&ctx, dm_offset, des3->iv, 0, 1388 des3->iv_len); 1389 if (ret) 1390 goto e_ctx; 1391 1392 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, 1393 CCP_PASSTHRU_BYTESWAP_256BIT); 1394 if (ret) { 1395 cmd->engine_error = cmd_q->cmd_error; 1396 goto e_ctx; 1397 } 1398 } 1399 1400 /* 1401 * Prepare the input and output data workareas. For in-place 1402 * operations we need to set the dma direction to BIDIRECTIONAL 1403 * and copy the src workarea to the dst workarea. 1404 */ 1405 if (sg_virt(des3->src) == sg_virt(des3->dst)) 1406 in_place = true; 1407 1408 ret = ccp_init_data(&src, cmd_q, des3->src, des3->src_len, 1409 DES3_EDE_BLOCK_SIZE, 1410 in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE); 1411 if (ret) 1412 goto e_ctx; 1413 1414 if (in_place) 1415 dst = src; 1416 else { 1417 ret = ccp_init_data(&dst, cmd_q, des3->dst, des3->src_len, 1418 DES3_EDE_BLOCK_SIZE, DMA_FROM_DEVICE); 1419 if (ret) 1420 goto e_src; 1421 } 1422 1423 /* Send data to the CCP DES3 engine */ 1424 while (src.sg_wa.bytes_left) { 1425 ccp_prepare_data(&src, &dst, &op, DES3_EDE_BLOCK_SIZE, true); 1426 if (!src.sg_wa.bytes_left) { 1427 op.eom = 1; 1428 1429 /* Since we don't retrieve the context in ECB mode 1430 * we have to wait for the operation to complete 1431 * on the last piece of data 1432 */ 1433 op.soc = 0; 1434 } 1435 1436 ret = cmd_q->ccp->vdata->perform->des3(&op); 1437 if (ret) { 1438 cmd->engine_error = cmd_q->cmd_error; 1439 goto e_dst; 1440 } 1441 1442 ccp_process_data(&src, &dst, &op); 1443 } 1444 1445 if (des3->mode != CCP_DES3_MODE_ECB) { 1446 /* Retrieve the context and make BE */ 1447 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, 1448 CCP_PASSTHRU_BYTESWAP_256BIT); 1449 if (ret) { 1450 cmd->engine_error = cmd_q->cmd_error; 1451 goto e_dst; 1452 } 1453 1454 /* ...but we only need the last DES3_EDE_BLOCK_SIZE bytes */ 1455 ccp_get_dm_area(&ctx, dm_offset, des3->iv, 0, 1456 DES3_EDE_BLOCK_SIZE); 1457 } 1458 e_dst: 1459 if (!in_place) 1460 ccp_free_data(&dst, cmd_q); 1461 1462 e_src: 1463 ccp_free_data(&src, cmd_q); 1464 1465 e_ctx: 1466 if (des3->mode != CCP_DES3_MODE_ECB) 1467 ccp_dm_free(&ctx); 1468 1469 e_key: 1470 ccp_dm_free(&key); 1471 1472 return ret; 1473 } 1474 1475 static int ccp_run_sha_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) 1476 { 1477 struct ccp_sha_engine *sha = &cmd->u.sha; 1478 struct ccp_dm_workarea ctx; 1479 struct ccp_data src; 1480 struct ccp_op op; 1481 unsigned int ioffset, ooffset; 1482 unsigned int digest_size; 1483 int sb_count; 1484 const void *init; 1485 u64 block_size; 1486 int ctx_size; 1487 int ret; 1488 1489 switch (sha->type) { 1490 case CCP_SHA_TYPE_1: 1491 if (sha->ctx_len < SHA1_DIGEST_SIZE) 1492 return -EINVAL; 1493 block_size = SHA1_BLOCK_SIZE; 1494 break; 1495 case CCP_SHA_TYPE_224: 1496 if (sha->ctx_len < SHA224_DIGEST_SIZE) 1497 return -EINVAL; 1498 block_size = SHA224_BLOCK_SIZE; 1499 break; 1500 case CCP_SHA_TYPE_256: 1501 if (sha->ctx_len < SHA256_DIGEST_SIZE) 1502 return -EINVAL; 1503 block_size = SHA256_BLOCK_SIZE; 1504 break; 1505 case CCP_SHA_TYPE_384: 1506 if (cmd_q->ccp->vdata->version < CCP_VERSION(4, 0) 1507 || sha->ctx_len < SHA384_DIGEST_SIZE) 1508 return -EINVAL; 1509 block_size = SHA384_BLOCK_SIZE; 1510 break; 1511 case CCP_SHA_TYPE_512: 1512 if (cmd_q->ccp->vdata->version < CCP_VERSION(4, 0) 1513 || sha->ctx_len < SHA512_DIGEST_SIZE) 1514 return -EINVAL; 1515 block_size = SHA512_BLOCK_SIZE; 1516 break; 1517 default: 1518 return -EINVAL; 1519 } 1520 1521 if (!sha->ctx) 1522 return -EINVAL; 1523 1524 if (!sha->final && (sha->src_len & (block_size - 1))) 1525 return -EINVAL; 1526 1527 /* The version 3 device can't handle zero-length input */ 1528 if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0)) { 1529 1530 if (!sha->src_len) { 1531 unsigned int digest_len; 1532 const u8 *sha_zero; 1533 1534 /* Not final, just return */ 1535 if (!sha->final) 1536 return 0; 1537 1538 /* CCP can't do a zero length sha operation so the 1539 * caller must buffer the data. 1540 */ 1541 if (sha->msg_bits) 1542 return -EINVAL; 1543 1544 /* The CCP cannot perform zero-length sha operations 1545 * so the caller is required to buffer data for the 1546 * final operation. However, a sha operation for a 1547 * message with a total length of zero is valid so 1548 * known values are required to supply the result. 1549 */ 1550 switch (sha->type) { 1551 case CCP_SHA_TYPE_1: 1552 sha_zero = sha1_zero_message_hash; 1553 digest_len = SHA1_DIGEST_SIZE; 1554 break; 1555 case CCP_SHA_TYPE_224: 1556 sha_zero = sha224_zero_message_hash; 1557 digest_len = SHA224_DIGEST_SIZE; 1558 break; 1559 case CCP_SHA_TYPE_256: 1560 sha_zero = sha256_zero_message_hash; 1561 digest_len = SHA256_DIGEST_SIZE; 1562 break; 1563 default: 1564 return -EINVAL; 1565 } 1566 1567 scatterwalk_map_and_copy((void *)sha_zero, sha->ctx, 0, 1568 digest_len, 1); 1569 1570 return 0; 1571 } 1572 } 1573 1574 /* Set variables used throughout */ 1575 switch (sha->type) { 1576 case CCP_SHA_TYPE_1: 1577 digest_size = SHA1_DIGEST_SIZE; 1578 init = (void *) ccp_sha1_init; 1579 ctx_size = SHA1_DIGEST_SIZE; 1580 sb_count = 1; 1581 if (cmd_q->ccp->vdata->version != CCP_VERSION(3, 0)) 1582 ooffset = ioffset = CCP_SB_BYTES - SHA1_DIGEST_SIZE; 1583 else 1584 ooffset = ioffset = 0; 1585 break; 1586 case CCP_SHA_TYPE_224: 1587 digest_size = SHA224_DIGEST_SIZE; 1588 init = (void *) ccp_sha224_init; 1589 ctx_size = SHA256_DIGEST_SIZE; 1590 sb_count = 1; 1591 ioffset = 0; 1592 if (cmd_q->ccp->vdata->version != CCP_VERSION(3, 0)) 1593 ooffset = CCP_SB_BYTES - SHA224_DIGEST_SIZE; 1594 else 1595 ooffset = 0; 1596 break; 1597 case CCP_SHA_TYPE_256: 1598 digest_size = SHA256_DIGEST_SIZE; 1599 init = (void *) ccp_sha256_init; 1600 ctx_size = SHA256_DIGEST_SIZE; 1601 sb_count = 1; 1602 ooffset = ioffset = 0; 1603 break; 1604 case CCP_SHA_TYPE_384: 1605 digest_size = SHA384_DIGEST_SIZE; 1606 init = (void *) ccp_sha384_init; 1607 ctx_size = SHA512_DIGEST_SIZE; 1608 sb_count = 2; 1609 ioffset = 0; 1610 ooffset = 2 * CCP_SB_BYTES - SHA384_DIGEST_SIZE; 1611 break; 1612 case CCP_SHA_TYPE_512: 1613 digest_size = SHA512_DIGEST_SIZE; 1614 init = (void *) ccp_sha512_init; 1615 ctx_size = SHA512_DIGEST_SIZE; 1616 sb_count = 2; 1617 ooffset = ioffset = 0; 1618 break; 1619 default: 1620 ret = -EINVAL; 1621 goto e_data; 1622 } 1623 1624 /* For zero-length plaintext the src pointer is ignored; 1625 * otherwise both parts must be valid 1626 */ 1627 if (sha->src_len && !sha->src) 1628 return -EINVAL; 1629 1630 memset(&op, 0, sizeof(op)); 1631 op.cmd_q = cmd_q; 1632 op.jobid = CCP_NEW_JOBID(cmd_q->ccp); 1633 op.sb_ctx = cmd_q->sb_ctx; /* Pre-allocated */ 1634 op.u.sha.type = sha->type; 1635 op.u.sha.msg_bits = sha->msg_bits; 1636 1637 /* For SHA1/224/256 the context fits in a single (32-byte) SB entry; 1638 * SHA384/512 require 2 adjacent SB slots, with the right half in the 1639 * first slot, and the left half in the second. Each portion must then 1640 * be in little endian format: use the 256-bit byte swap option. 1641 */ 1642 ret = ccp_init_dm_workarea(&ctx, cmd_q, sb_count * CCP_SB_BYTES, 1643 DMA_BIDIRECTIONAL); 1644 if (ret) 1645 return ret; 1646 if (sha->first) { 1647 switch (sha->type) { 1648 case CCP_SHA_TYPE_1: 1649 case CCP_SHA_TYPE_224: 1650 case CCP_SHA_TYPE_256: 1651 memcpy(ctx.address + ioffset, init, ctx_size); 1652 break; 1653 case CCP_SHA_TYPE_384: 1654 case CCP_SHA_TYPE_512: 1655 memcpy(ctx.address + ctx_size / 2, init, 1656 ctx_size / 2); 1657 memcpy(ctx.address, init + ctx_size / 2, 1658 ctx_size / 2); 1659 break; 1660 default: 1661 ret = -EINVAL; 1662 goto e_ctx; 1663 } 1664 } else { 1665 /* Restore the context */ 1666 ret = ccp_set_dm_area(&ctx, 0, sha->ctx, 0, 1667 sb_count * CCP_SB_BYTES); 1668 if (ret) 1669 goto e_ctx; 1670 } 1671 1672 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, 1673 CCP_PASSTHRU_BYTESWAP_256BIT); 1674 if (ret) { 1675 cmd->engine_error = cmd_q->cmd_error; 1676 goto e_ctx; 1677 } 1678 1679 if (sha->src) { 1680 /* Send data to the CCP SHA engine; block_size is set above */ 1681 ret = ccp_init_data(&src, cmd_q, sha->src, sha->src_len, 1682 block_size, DMA_TO_DEVICE); 1683 if (ret) 1684 goto e_ctx; 1685 1686 while (src.sg_wa.bytes_left) { 1687 ccp_prepare_data(&src, NULL, &op, block_size, false); 1688 if (sha->final && !src.sg_wa.bytes_left) 1689 op.eom = 1; 1690 1691 ret = cmd_q->ccp->vdata->perform->sha(&op); 1692 if (ret) { 1693 cmd->engine_error = cmd_q->cmd_error; 1694 goto e_data; 1695 } 1696 1697 ccp_process_data(&src, NULL, &op); 1698 } 1699 } else { 1700 op.eom = 1; 1701 ret = cmd_q->ccp->vdata->perform->sha(&op); 1702 if (ret) { 1703 cmd->engine_error = cmd_q->cmd_error; 1704 goto e_data; 1705 } 1706 } 1707 1708 /* Retrieve the SHA context - convert from LE to BE using 1709 * 32-byte (256-bit) byteswapping to BE 1710 */ 1711 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, 1712 CCP_PASSTHRU_BYTESWAP_256BIT); 1713 if (ret) { 1714 cmd->engine_error = cmd_q->cmd_error; 1715 goto e_data; 1716 } 1717 1718 if (sha->final) { 1719 /* Finishing up, so get the digest */ 1720 switch (sha->type) { 1721 case CCP_SHA_TYPE_1: 1722 case CCP_SHA_TYPE_224: 1723 case CCP_SHA_TYPE_256: 1724 ccp_get_dm_area(&ctx, ooffset, 1725 sha->ctx, 0, 1726 digest_size); 1727 break; 1728 case CCP_SHA_TYPE_384: 1729 case CCP_SHA_TYPE_512: 1730 ccp_get_dm_area(&ctx, 0, 1731 sha->ctx, LSB_ITEM_SIZE - ooffset, 1732 LSB_ITEM_SIZE); 1733 ccp_get_dm_area(&ctx, LSB_ITEM_SIZE + ooffset, 1734 sha->ctx, 0, 1735 LSB_ITEM_SIZE - ooffset); 1736 break; 1737 default: 1738 ret = -EINVAL; 1739 goto e_ctx; 1740 } 1741 } else { 1742 /* Stash the context */ 1743 ccp_get_dm_area(&ctx, 0, sha->ctx, 0, 1744 sb_count * CCP_SB_BYTES); 1745 } 1746 1747 if (sha->final && sha->opad) { 1748 /* HMAC operation, recursively perform final SHA */ 1749 struct ccp_cmd hmac_cmd; 1750 struct scatterlist sg; 1751 u8 *hmac_buf; 1752 1753 if (sha->opad_len != block_size) { 1754 ret = -EINVAL; 1755 goto e_data; 1756 } 1757 1758 hmac_buf = kmalloc(block_size + digest_size, GFP_KERNEL); 1759 if (!hmac_buf) { 1760 ret = -ENOMEM; 1761 goto e_data; 1762 } 1763 sg_init_one(&sg, hmac_buf, block_size + digest_size); 1764 1765 scatterwalk_map_and_copy(hmac_buf, sha->opad, 0, block_size, 0); 1766 switch (sha->type) { 1767 case CCP_SHA_TYPE_1: 1768 case CCP_SHA_TYPE_224: 1769 case CCP_SHA_TYPE_256: 1770 memcpy(hmac_buf + block_size, 1771 ctx.address + ooffset, 1772 digest_size); 1773 break; 1774 case CCP_SHA_TYPE_384: 1775 case CCP_SHA_TYPE_512: 1776 memcpy(hmac_buf + block_size, 1777 ctx.address + LSB_ITEM_SIZE + ooffset, 1778 LSB_ITEM_SIZE); 1779 memcpy(hmac_buf + block_size + 1780 (LSB_ITEM_SIZE - ooffset), 1781 ctx.address, 1782 LSB_ITEM_SIZE); 1783 break; 1784 default: 1785 ret = -EINVAL; 1786 goto e_ctx; 1787 } 1788 1789 memset(&hmac_cmd, 0, sizeof(hmac_cmd)); 1790 hmac_cmd.engine = CCP_ENGINE_SHA; 1791 hmac_cmd.u.sha.type = sha->type; 1792 hmac_cmd.u.sha.ctx = sha->ctx; 1793 hmac_cmd.u.sha.ctx_len = sha->ctx_len; 1794 hmac_cmd.u.sha.src = &sg; 1795 hmac_cmd.u.sha.src_len = block_size + digest_size; 1796 hmac_cmd.u.sha.opad = NULL; 1797 hmac_cmd.u.sha.opad_len = 0; 1798 hmac_cmd.u.sha.first = 1; 1799 hmac_cmd.u.sha.final = 1; 1800 hmac_cmd.u.sha.msg_bits = (block_size + digest_size) << 3; 1801 1802 ret = ccp_run_sha_cmd(cmd_q, &hmac_cmd); 1803 if (ret) 1804 cmd->engine_error = hmac_cmd.engine_error; 1805 1806 kfree(hmac_buf); 1807 } 1808 1809 e_data: 1810 if (sha->src) 1811 ccp_free_data(&src, cmd_q); 1812 1813 e_ctx: 1814 ccp_dm_free(&ctx); 1815 1816 return ret; 1817 } 1818 1819 static int ccp_run_rsa_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) 1820 { 1821 struct ccp_rsa_engine *rsa = &cmd->u.rsa; 1822 struct ccp_dm_workarea exp, src, dst; 1823 struct ccp_op op; 1824 unsigned int sb_count, i_len, o_len; 1825 int ret; 1826 1827 /* Check against the maximum allowable size, in bits */ 1828 if (rsa->key_size > cmd_q->ccp->vdata->rsamax) 1829 return -EINVAL; 1830 1831 if (!rsa->exp || !rsa->mod || !rsa->src || !rsa->dst) 1832 return -EINVAL; 1833 1834 memset(&op, 0, sizeof(op)); 1835 op.cmd_q = cmd_q; 1836 op.jobid = CCP_NEW_JOBID(cmd_q->ccp); 1837 1838 /* The RSA modulus must precede the message being acted upon, so 1839 * it must be copied to a DMA area where the message and the 1840 * modulus can be concatenated. Therefore the input buffer 1841 * length required is twice the output buffer length (which 1842 * must be a multiple of 256-bits). Compute o_len, i_len in bytes. 1843 * Buffer sizes must be a multiple of 32 bytes; rounding up may be 1844 * required. 1845 */ 1846 o_len = 32 * ((rsa->key_size + 255) / 256); 1847 i_len = o_len * 2; 1848 1849 sb_count = 0; 1850 if (cmd_q->ccp->vdata->version < CCP_VERSION(5, 0)) { 1851 /* sb_count is the number of storage block slots required 1852 * for the modulus. 1853 */ 1854 sb_count = o_len / CCP_SB_BYTES; 1855 op.sb_key = cmd_q->ccp->vdata->perform->sballoc(cmd_q, 1856 sb_count); 1857 if (!op.sb_key) 1858 return -EIO; 1859 } else { 1860 /* A version 5 device allows a modulus size that will not fit 1861 * in the LSB, so the command will transfer it from memory. 1862 * Set the sb key to the default, even though it's not used. 1863 */ 1864 op.sb_key = cmd_q->sb_key; 1865 } 1866 1867 /* The RSA exponent must be in little endian format. Reverse its 1868 * byte order. 1869 */ 1870 ret = ccp_init_dm_workarea(&exp, cmd_q, o_len, DMA_TO_DEVICE); 1871 if (ret) 1872 goto e_sb; 1873 1874 ret = ccp_reverse_set_dm_area(&exp, 0, rsa->exp, 0, rsa->exp_len); 1875 if (ret) 1876 goto e_exp; 1877 1878 if (cmd_q->ccp->vdata->version < CCP_VERSION(5, 0)) { 1879 /* Copy the exponent to the local storage block, using 1880 * as many 32-byte blocks as were allocated above. It's 1881 * already little endian, so no further change is required. 1882 */ 1883 ret = ccp_copy_to_sb(cmd_q, &exp, op.jobid, op.sb_key, 1884 CCP_PASSTHRU_BYTESWAP_NOOP); 1885 if (ret) { 1886 cmd->engine_error = cmd_q->cmd_error; 1887 goto e_exp; 1888 } 1889 } else { 1890 /* The exponent can be retrieved from memory via DMA. */ 1891 op.exp.u.dma.address = exp.dma.address; 1892 op.exp.u.dma.offset = 0; 1893 } 1894 1895 /* Concatenate the modulus and the message. Both the modulus and 1896 * the operands must be in little endian format. Since the input 1897 * is in big endian format it must be converted. 1898 */ 1899 ret = ccp_init_dm_workarea(&src, cmd_q, i_len, DMA_TO_DEVICE); 1900 if (ret) 1901 goto e_exp; 1902 1903 ret = ccp_reverse_set_dm_area(&src, 0, rsa->mod, 0, rsa->mod_len); 1904 if (ret) 1905 goto e_src; 1906 ret = ccp_reverse_set_dm_area(&src, o_len, rsa->src, 0, rsa->src_len); 1907 if (ret) 1908 goto e_src; 1909 1910 /* Prepare the output area for the operation */ 1911 ret = ccp_init_dm_workarea(&dst, cmd_q, o_len, DMA_FROM_DEVICE); 1912 if (ret) 1913 goto e_src; 1914 1915 op.soc = 1; 1916 op.src.u.dma.address = src.dma.address; 1917 op.src.u.dma.offset = 0; 1918 op.src.u.dma.length = i_len; 1919 op.dst.u.dma.address = dst.dma.address; 1920 op.dst.u.dma.offset = 0; 1921 op.dst.u.dma.length = o_len; 1922 1923 op.u.rsa.mod_size = rsa->key_size; 1924 op.u.rsa.input_len = i_len; 1925 1926 ret = cmd_q->ccp->vdata->perform->rsa(&op); 1927 if (ret) { 1928 cmd->engine_error = cmd_q->cmd_error; 1929 goto e_dst; 1930 } 1931 1932 ccp_reverse_get_dm_area(&dst, 0, rsa->dst, 0, rsa->mod_len); 1933 1934 e_dst: 1935 ccp_dm_free(&dst); 1936 1937 e_src: 1938 ccp_dm_free(&src); 1939 1940 e_exp: 1941 ccp_dm_free(&exp); 1942 1943 e_sb: 1944 if (sb_count) 1945 cmd_q->ccp->vdata->perform->sbfree(cmd_q, op.sb_key, sb_count); 1946 1947 return ret; 1948 } 1949 1950 static int ccp_run_passthru_cmd(struct ccp_cmd_queue *cmd_q, 1951 struct ccp_cmd *cmd) 1952 { 1953 struct ccp_passthru_engine *pt = &cmd->u.passthru; 1954 struct ccp_dm_workarea mask; 1955 struct ccp_data src, dst; 1956 struct ccp_op op; 1957 bool in_place = false; 1958 unsigned int i; 1959 int ret = 0; 1960 1961 if (!pt->final && (pt->src_len & (CCP_PASSTHRU_BLOCKSIZE - 1))) 1962 return -EINVAL; 1963 1964 if (!pt->src || !pt->dst) 1965 return -EINVAL; 1966 1967 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) { 1968 if (pt->mask_len != CCP_PASSTHRU_MASKSIZE) 1969 return -EINVAL; 1970 if (!pt->mask) 1971 return -EINVAL; 1972 } 1973 1974 BUILD_BUG_ON(CCP_PASSTHRU_SB_COUNT != 1); 1975 1976 memset(&op, 0, sizeof(op)); 1977 op.cmd_q = cmd_q; 1978 op.jobid = CCP_NEW_JOBID(cmd_q->ccp); 1979 1980 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) { 1981 /* Load the mask */ 1982 op.sb_key = cmd_q->sb_key; 1983 1984 ret = ccp_init_dm_workarea(&mask, cmd_q, 1985 CCP_PASSTHRU_SB_COUNT * 1986 CCP_SB_BYTES, 1987 DMA_TO_DEVICE); 1988 if (ret) 1989 return ret; 1990 1991 ret = ccp_set_dm_area(&mask, 0, pt->mask, 0, pt->mask_len); 1992 if (ret) 1993 goto e_mask; 1994 ret = ccp_copy_to_sb(cmd_q, &mask, op.jobid, op.sb_key, 1995 CCP_PASSTHRU_BYTESWAP_NOOP); 1996 if (ret) { 1997 cmd->engine_error = cmd_q->cmd_error; 1998 goto e_mask; 1999 } 2000 } 2001 2002 /* Prepare the input and output data workareas. For in-place 2003 * operations we need to set the dma direction to BIDIRECTIONAL 2004 * and copy the src workarea to the dst workarea. 2005 */ 2006 if (sg_virt(pt->src) == sg_virt(pt->dst)) 2007 in_place = true; 2008 2009 ret = ccp_init_data(&src, cmd_q, pt->src, pt->src_len, 2010 CCP_PASSTHRU_MASKSIZE, 2011 in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE); 2012 if (ret) 2013 goto e_mask; 2014 2015 if (in_place) { 2016 dst = src; 2017 } else { 2018 ret = ccp_init_data(&dst, cmd_q, pt->dst, pt->src_len, 2019 CCP_PASSTHRU_MASKSIZE, DMA_FROM_DEVICE); 2020 if (ret) 2021 goto e_src; 2022 } 2023 2024 /* Send data to the CCP Passthru engine 2025 * Because the CCP engine works on a single source and destination 2026 * dma address at a time, each entry in the source scatterlist 2027 * (after the dma_map_sg call) must be less than or equal to the 2028 * (remaining) length in the destination scatterlist entry and the 2029 * length must be a multiple of CCP_PASSTHRU_BLOCKSIZE 2030 */ 2031 dst.sg_wa.sg_used = 0; 2032 for (i = 1; i <= src.sg_wa.dma_count; i++) { 2033 if (!dst.sg_wa.sg || 2034 (dst.sg_wa.sg->length < src.sg_wa.sg->length)) { 2035 ret = -EINVAL; 2036 goto e_dst; 2037 } 2038 2039 if (i == src.sg_wa.dma_count) { 2040 op.eom = 1; 2041 op.soc = 1; 2042 } 2043 2044 op.src.type = CCP_MEMTYPE_SYSTEM; 2045 op.src.u.dma.address = sg_dma_address(src.sg_wa.sg); 2046 op.src.u.dma.offset = 0; 2047 op.src.u.dma.length = sg_dma_len(src.sg_wa.sg); 2048 2049 op.dst.type = CCP_MEMTYPE_SYSTEM; 2050 op.dst.u.dma.address = sg_dma_address(dst.sg_wa.sg); 2051 op.dst.u.dma.offset = dst.sg_wa.sg_used; 2052 op.dst.u.dma.length = op.src.u.dma.length; 2053 2054 ret = cmd_q->ccp->vdata->perform->passthru(&op); 2055 if (ret) { 2056 cmd->engine_error = cmd_q->cmd_error; 2057 goto e_dst; 2058 } 2059 2060 dst.sg_wa.sg_used += src.sg_wa.sg->length; 2061 if (dst.sg_wa.sg_used == dst.sg_wa.sg->length) { 2062 dst.sg_wa.sg = sg_next(dst.sg_wa.sg); 2063 dst.sg_wa.sg_used = 0; 2064 } 2065 src.sg_wa.sg = sg_next(src.sg_wa.sg); 2066 } 2067 2068 e_dst: 2069 if (!in_place) 2070 ccp_free_data(&dst, cmd_q); 2071 2072 e_src: 2073 ccp_free_data(&src, cmd_q); 2074 2075 e_mask: 2076 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) 2077 ccp_dm_free(&mask); 2078 2079 return ret; 2080 } 2081 2082 static int ccp_run_passthru_nomap_cmd(struct ccp_cmd_queue *cmd_q, 2083 struct ccp_cmd *cmd) 2084 { 2085 struct ccp_passthru_nomap_engine *pt = &cmd->u.passthru_nomap; 2086 struct ccp_dm_workarea mask; 2087 struct ccp_op op; 2088 int ret; 2089 2090 if (!pt->final && (pt->src_len & (CCP_PASSTHRU_BLOCKSIZE - 1))) 2091 return -EINVAL; 2092 2093 if (!pt->src_dma || !pt->dst_dma) 2094 return -EINVAL; 2095 2096 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) { 2097 if (pt->mask_len != CCP_PASSTHRU_MASKSIZE) 2098 return -EINVAL; 2099 if (!pt->mask) 2100 return -EINVAL; 2101 } 2102 2103 BUILD_BUG_ON(CCP_PASSTHRU_SB_COUNT != 1); 2104 2105 memset(&op, 0, sizeof(op)); 2106 op.cmd_q = cmd_q; 2107 op.jobid = CCP_NEW_JOBID(cmd_q->ccp); 2108 2109 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) { 2110 /* Load the mask */ 2111 op.sb_key = cmd_q->sb_key; 2112 2113 mask.length = pt->mask_len; 2114 mask.dma.address = pt->mask; 2115 mask.dma.length = pt->mask_len; 2116 2117 ret = ccp_copy_to_sb(cmd_q, &mask, op.jobid, op.sb_key, 2118 CCP_PASSTHRU_BYTESWAP_NOOP); 2119 if (ret) { 2120 cmd->engine_error = cmd_q->cmd_error; 2121 return ret; 2122 } 2123 } 2124 2125 /* Send data to the CCP Passthru engine */ 2126 op.eom = 1; 2127 op.soc = 1; 2128 2129 op.src.type = CCP_MEMTYPE_SYSTEM; 2130 op.src.u.dma.address = pt->src_dma; 2131 op.src.u.dma.offset = 0; 2132 op.src.u.dma.length = pt->src_len; 2133 2134 op.dst.type = CCP_MEMTYPE_SYSTEM; 2135 op.dst.u.dma.address = pt->dst_dma; 2136 op.dst.u.dma.offset = 0; 2137 op.dst.u.dma.length = pt->src_len; 2138 2139 ret = cmd_q->ccp->vdata->perform->passthru(&op); 2140 if (ret) 2141 cmd->engine_error = cmd_q->cmd_error; 2142 2143 return ret; 2144 } 2145 2146 static int ccp_run_ecc_mm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) 2147 { 2148 struct ccp_ecc_engine *ecc = &cmd->u.ecc; 2149 struct ccp_dm_workarea src, dst; 2150 struct ccp_op op; 2151 int ret; 2152 u8 *save; 2153 2154 if (!ecc->u.mm.operand_1 || 2155 (ecc->u.mm.operand_1_len > CCP_ECC_MODULUS_BYTES)) 2156 return -EINVAL; 2157 2158 if (ecc->function != CCP_ECC_FUNCTION_MINV_384BIT) 2159 if (!ecc->u.mm.operand_2 || 2160 (ecc->u.mm.operand_2_len > CCP_ECC_MODULUS_BYTES)) 2161 return -EINVAL; 2162 2163 if (!ecc->u.mm.result || 2164 (ecc->u.mm.result_len < CCP_ECC_MODULUS_BYTES)) 2165 return -EINVAL; 2166 2167 memset(&op, 0, sizeof(op)); 2168 op.cmd_q = cmd_q; 2169 op.jobid = CCP_NEW_JOBID(cmd_q->ccp); 2170 2171 /* Concatenate the modulus and the operands. Both the modulus and 2172 * the operands must be in little endian format. Since the input 2173 * is in big endian format it must be converted and placed in a 2174 * fixed length buffer. 2175 */ 2176 ret = ccp_init_dm_workarea(&src, cmd_q, CCP_ECC_SRC_BUF_SIZE, 2177 DMA_TO_DEVICE); 2178 if (ret) 2179 return ret; 2180 2181 /* Save the workarea address since it is updated in order to perform 2182 * the concatenation 2183 */ 2184 save = src.address; 2185 2186 /* Copy the ECC modulus */ 2187 ret = ccp_reverse_set_dm_area(&src, 0, ecc->mod, 0, ecc->mod_len); 2188 if (ret) 2189 goto e_src; 2190 src.address += CCP_ECC_OPERAND_SIZE; 2191 2192 /* Copy the first operand */ 2193 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.mm.operand_1, 0, 2194 ecc->u.mm.operand_1_len); 2195 if (ret) 2196 goto e_src; 2197 src.address += CCP_ECC_OPERAND_SIZE; 2198 2199 if (ecc->function != CCP_ECC_FUNCTION_MINV_384BIT) { 2200 /* Copy the second operand */ 2201 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.mm.operand_2, 0, 2202 ecc->u.mm.operand_2_len); 2203 if (ret) 2204 goto e_src; 2205 src.address += CCP_ECC_OPERAND_SIZE; 2206 } 2207 2208 /* Restore the workarea address */ 2209 src.address = save; 2210 2211 /* Prepare the output area for the operation */ 2212 ret = ccp_init_dm_workarea(&dst, cmd_q, CCP_ECC_DST_BUF_SIZE, 2213 DMA_FROM_DEVICE); 2214 if (ret) 2215 goto e_src; 2216 2217 op.soc = 1; 2218 op.src.u.dma.address = src.dma.address; 2219 op.src.u.dma.offset = 0; 2220 op.src.u.dma.length = src.length; 2221 op.dst.u.dma.address = dst.dma.address; 2222 op.dst.u.dma.offset = 0; 2223 op.dst.u.dma.length = dst.length; 2224 2225 op.u.ecc.function = cmd->u.ecc.function; 2226 2227 ret = cmd_q->ccp->vdata->perform->ecc(&op); 2228 if (ret) { 2229 cmd->engine_error = cmd_q->cmd_error; 2230 goto e_dst; 2231 } 2232 2233 ecc->ecc_result = le16_to_cpup( 2234 (const __le16 *)(dst.address + CCP_ECC_RESULT_OFFSET)); 2235 if (!(ecc->ecc_result & CCP_ECC_RESULT_SUCCESS)) { 2236 ret = -EIO; 2237 goto e_dst; 2238 } 2239 2240 /* Save the ECC result */ 2241 ccp_reverse_get_dm_area(&dst, 0, ecc->u.mm.result, 0, 2242 CCP_ECC_MODULUS_BYTES); 2243 2244 e_dst: 2245 ccp_dm_free(&dst); 2246 2247 e_src: 2248 ccp_dm_free(&src); 2249 2250 return ret; 2251 } 2252 2253 static int ccp_run_ecc_pm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) 2254 { 2255 struct ccp_ecc_engine *ecc = &cmd->u.ecc; 2256 struct ccp_dm_workarea src, dst; 2257 struct ccp_op op; 2258 int ret; 2259 u8 *save; 2260 2261 if (!ecc->u.pm.point_1.x || 2262 (ecc->u.pm.point_1.x_len > CCP_ECC_MODULUS_BYTES) || 2263 !ecc->u.pm.point_1.y || 2264 (ecc->u.pm.point_1.y_len > CCP_ECC_MODULUS_BYTES)) 2265 return -EINVAL; 2266 2267 if (ecc->function == CCP_ECC_FUNCTION_PADD_384BIT) { 2268 if (!ecc->u.pm.point_2.x || 2269 (ecc->u.pm.point_2.x_len > CCP_ECC_MODULUS_BYTES) || 2270 !ecc->u.pm.point_2.y || 2271 (ecc->u.pm.point_2.y_len > CCP_ECC_MODULUS_BYTES)) 2272 return -EINVAL; 2273 } else { 2274 if (!ecc->u.pm.domain_a || 2275 (ecc->u.pm.domain_a_len > CCP_ECC_MODULUS_BYTES)) 2276 return -EINVAL; 2277 2278 if (ecc->function == CCP_ECC_FUNCTION_PMUL_384BIT) 2279 if (!ecc->u.pm.scalar || 2280 (ecc->u.pm.scalar_len > CCP_ECC_MODULUS_BYTES)) 2281 return -EINVAL; 2282 } 2283 2284 if (!ecc->u.pm.result.x || 2285 (ecc->u.pm.result.x_len < CCP_ECC_MODULUS_BYTES) || 2286 !ecc->u.pm.result.y || 2287 (ecc->u.pm.result.y_len < CCP_ECC_MODULUS_BYTES)) 2288 return -EINVAL; 2289 2290 memset(&op, 0, sizeof(op)); 2291 op.cmd_q = cmd_q; 2292 op.jobid = CCP_NEW_JOBID(cmd_q->ccp); 2293 2294 /* Concatenate the modulus and the operands. Both the modulus and 2295 * the operands must be in little endian format. Since the input 2296 * is in big endian format it must be converted and placed in a 2297 * fixed length buffer. 2298 */ 2299 ret = ccp_init_dm_workarea(&src, cmd_q, CCP_ECC_SRC_BUF_SIZE, 2300 DMA_TO_DEVICE); 2301 if (ret) 2302 return ret; 2303 2304 /* Save the workarea address since it is updated in order to perform 2305 * the concatenation 2306 */ 2307 save = src.address; 2308 2309 /* Copy the ECC modulus */ 2310 ret = ccp_reverse_set_dm_area(&src, 0, ecc->mod, 0, ecc->mod_len); 2311 if (ret) 2312 goto e_src; 2313 src.address += CCP_ECC_OPERAND_SIZE; 2314 2315 /* Copy the first point X and Y coordinate */ 2316 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_1.x, 0, 2317 ecc->u.pm.point_1.x_len); 2318 if (ret) 2319 goto e_src; 2320 src.address += CCP_ECC_OPERAND_SIZE; 2321 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_1.y, 0, 2322 ecc->u.pm.point_1.y_len); 2323 if (ret) 2324 goto e_src; 2325 src.address += CCP_ECC_OPERAND_SIZE; 2326 2327 /* Set the first point Z coordinate to 1 */ 2328 *src.address = 0x01; 2329 src.address += CCP_ECC_OPERAND_SIZE; 2330 2331 if (ecc->function == CCP_ECC_FUNCTION_PADD_384BIT) { 2332 /* Copy the second point X and Y coordinate */ 2333 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_2.x, 0, 2334 ecc->u.pm.point_2.x_len); 2335 if (ret) 2336 goto e_src; 2337 src.address += CCP_ECC_OPERAND_SIZE; 2338 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_2.y, 0, 2339 ecc->u.pm.point_2.y_len); 2340 if (ret) 2341 goto e_src; 2342 src.address += CCP_ECC_OPERAND_SIZE; 2343 2344 /* Set the second point Z coordinate to 1 */ 2345 *src.address = 0x01; 2346 src.address += CCP_ECC_OPERAND_SIZE; 2347 } else { 2348 /* Copy the Domain "a" parameter */ 2349 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.domain_a, 0, 2350 ecc->u.pm.domain_a_len); 2351 if (ret) 2352 goto e_src; 2353 src.address += CCP_ECC_OPERAND_SIZE; 2354 2355 if (ecc->function == CCP_ECC_FUNCTION_PMUL_384BIT) { 2356 /* Copy the scalar value */ 2357 ret = ccp_reverse_set_dm_area(&src, 0, 2358 ecc->u.pm.scalar, 0, 2359 ecc->u.pm.scalar_len); 2360 if (ret) 2361 goto e_src; 2362 src.address += CCP_ECC_OPERAND_SIZE; 2363 } 2364 } 2365 2366 /* Restore the workarea address */ 2367 src.address = save; 2368 2369 /* Prepare the output area for the operation */ 2370 ret = ccp_init_dm_workarea(&dst, cmd_q, CCP_ECC_DST_BUF_SIZE, 2371 DMA_FROM_DEVICE); 2372 if (ret) 2373 goto e_src; 2374 2375 op.soc = 1; 2376 op.src.u.dma.address = src.dma.address; 2377 op.src.u.dma.offset = 0; 2378 op.src.u.dma.length = src.length; 2379 op.dst.u.dma.address = dst.dma.address; 2380 op.dst.u.dma.offset = 0; 2381 op.dst.u.dma.length = dst.length; 2382 2383 op.u.ecc.function = cmd->u.ecc.function; 2384 2385 ret = cmd_q->ccp->vdata->perform->ecc(&op); 2386 if (ret) { 2387 cmd->engine_error = cmd_q->cmd_error; 2388 goto e_dst; 2389 } 2390 2391 ecc->ecc_result = le16_to_cpup( 2392 (const __le16 *)(dst.address + CCP_ECC_RESULT_OFFSET)); 2393 if (!(ecc->ecc_result & CCP_ECC_RESULT_SUCCESS)) { 2394 ret = -EIO; 2395 goto e_dst; 2396 } 2397 2398 /* Save the workarea address since it is updated as we walk through 2399 * to copy the point math result 2400 */ 2401 save = dst.address; 2402 2403 /* Save the ECC result X and Y coordinates */ 2404 ccp_reverse_get_dm_area(&dst, 0, ecc->u.pm.result.x, 0, 2405 CCP_ECC_MODULUS_BYTES); 2406 dst.address += CCP_ECC_OUTPUT_SIZE; 2407 ccp_reverse_get_dm_area(&dst, 0, ecc->u.pm.result.y, 0, 2408 CCP_ECC_MODULUS_BYTES); 2409 dst.address += CCP_ECC_OUTPUT_SIZE; 2410 2411 /* Restore the workarea address */ 2412 dst.address = save; 2413 2414 e_dst: 2415 ccp_dm_free(&dst); 2416 2417 e_src: 2418 ccp_dm_free(&src); 2419 2420 return ret; 2421 } 2422 2423 static int ccp_run_ecc_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) 2424 { 2425 struct ccp_ecc_engine *ecc = &cmd->u.ecc; 2426 2427 ecc->ecc_result = 0; 2428 2429 if (!ecc->mod || 2430 (ecc->mod_len > CCP_ECC_MODULUS_BYTES)) 2431 return -EINVAL; 2432 2433 switch (ecc->function) { 2434 case CCP_ECC_FUNCTION_MMUL_384BIT: 2435 case CCP_ECC_FUNCTION_MADD_384BIT: 2436 case CCP_ECC_FUNCTION_MINV_384BIT: 2437 return ccp_run_ecc_mm_cmd(cmd_q, cmd); 2438 2439 case CCP_ECC_FUNCTION_PADD_384BIT: 2440 case CCP_ECC_FUNCTION_PMUL_384BIT: 2441 case CCP_ECC_FUNCTION_PDBL_384BIT: 2442 return ccp_run_ecc_pm_cmd(cmd_q, cmd); 2443 2444 default: 2445 return -EINVAL; 2446 } 2447 } 2448 2449 int ccp_run_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) 2450 { 2451 int ret; 2452 2453 cmd->engine_error = 0; 2454 cmd_q->cmd_error = 0; 2455 cmd_q->int_rcvd = 0; 2456 cmd_q->free_slots = cmd_q->ccp->vdata->perform->get_free_slots(cmd_q); 2457 2458 switch (cmd->engine) { 2459 case CCP_ENGINE_AES: 2460 ret = ccp_run_aes_cmd(cmd_q, cmd); 2461 break; 2462 case CCP_ENGINE_XTS_AES_128: 2463 ret = ccp_run_xts_aes_cmd(cmd_q, cmd); 2464 break; 2465 case CCP_ENGINE_DES3: 2466 ret = ccp_run_des3_cmd(cmd_q, cmd); 2467 break; 2468 case CCP_ENGINE_SHA: 2469 ret = ccp_run_sha_cmd(cmd_q, cmd); 2470 break; 2471 case CCP_ENGINE_RSA: 2472 ret = ccp_run_rsa_cmd(cmd_q, cmd); 2473 break; 2474 case CCP_ENGINE_PASSTHRU: 2475 if (cmd->flags & CCP_CMD_PASSTHRU_NO_DMA_MAP) 2476 ret = ccp_run_passthru_nomap_cmd(cmd_q, cmd); 2477 else 2478 ret = ccp_run_passthru_cmd(cmd_q, cmd); 2479 break; 2480 case CCP_ENGINE_ECC: 2481 ret = ccp_run_ecc_cmd(cmd_q, cmd); 2482 break; 2483 default: 2484 ret = -EINVAL; 2485 } 2486 2487 return ret; 2488 } 2489