1 /* 2 * AMD Cryptographic Coprocessor (CCP) driver 3 * 4 * Copyright (C) 2013,2018 Advanced Micro Devices, Inc. 5 * 6 * Author: Tom Lendacky <thomas.lendacky@amd.com> 7 * Author: Gary R Hook <gary.hook@amd.com> 8 * 9 * This program is free software; you can redistribute it and/or modify 10 * it under the terms of the GNU General Public License version 2 as 11 * published by the Free Software Foundation. 12 */ 13 14 #include <linux/module.h> 15 #include <linux/kernel.h> 16 #include <linux/pci.h> 17 #include <linux/interrupt.h> 18 #include <crypto/scatterwalk.h> 19 #include <crypto/des.h> 20 #include <linux/ccp.h> 21 22 #include "ccp-dev.h" 23 24 /* SHA initial context values */ 25 static const __be32 ccp_sha1_init[SHA1_DIGEST_SIZE / sizeof(__be32)] = { 26 cpu_to_be32(SHA1_H0), cpu_to_be32(SHA1_H1), 27 cpu_to_be32(SHA1_H2), cpu_to_be32(SHA1_H3), 28 cpu_to_be32(SHA1_H4), 29 }; 30 31 static const __be32 ccp_sha224_init[SHA256_DIGEST_SIZE / sizeof(__be32)] = { 32 cpu_to_be32(SHA224_H0), cpu_to_be32(SHA224_H1), 33 cpu_to_be32(SHA224_H2), cpu_to_be32(SHA224_H3), 34 cpu_to_be32(SHA224_H4), cpu_to_be32(SHA224_H5), 35 cpu_to_be32(SHA224_H6), cpu_to_be32(SHA224_H7), 36 }; 37 38 static const __be32 ccp_sha256_init[SHA256_DIGEST_SIZE / sizeof(__be32)] = { 39 cpu_to_be32(SHA256_H0), cpu_to_be32(SHA256_H1), 40 cpu_to_be32(SHA256_H2), cpu_to_be32(SHA256_H3), 41 cpu_to_be32(SHA256_H4), cpu_to_be32(SHA256_H5), 42 cpu_to_be32(SHA256_H6), cpu_to_be32(SHA256_H7), 43 }; 44 45 static const __be64 ccp_sha384_init[SHA512_DIGEST_SIZE / sizeof(__be64)] = { 46 cpu_to_be64(SHA384_H0), cpu_to_be64(SHA384_H1), 47 cpu_to_be64(SHA384_H2), cpu_to_be64(SHA384_H3), 48 cpu_to_be64(SHA384_H4), cpu_to_be64(SHA384_H5), 49 cpu_to_be64(SHA384_H6), cpu_to_be64(SHA384_H7), 50 }; 51 52 static const __be64 ccp_sha512_init[SHA512_DIGEST_SIZE / sizeof(__be64)] = { 53 cpu_to_be64(SHA512_H0), cpu_to_be64(SHA512_H1), 54 cpu_to_be64(SHA512_H2), cpu_to_be64(SHA512_H3), 55 cpu_to_be64(SHA512_H4), cpu_to_be64(SHA512_H5), 56 cpu_to_be64(SHA512_H6), cpu_to_be64(SHA512_H7), 57 }; 58 59 #define CCP_NEW_JOBID(ccp) ((ccp->vdata->version == CCP_VERSION(3, 0)) ? \ 60 ccp_gen_jobid(ccp) : 0) 61 62 static u32 ccp_gen_jobid(struct ccp_device *ccp) 63 { 64 return atomic_inc_return(&ccp->current_id) & CCP_JOBID_MASK; 65 } 66 67 static void ccp_sg_free(struct ccp_sg_workarea *wa) 68 { 69 if (wa->dma_count) 70 dma_unmap_sg(wa->dma_dev, wa->dma_sg, wa->nents, wa->dma_dir); 71 72 wa->dma_count = 0; 73 } 74 75 static int ccp_init_sg_workarea(struct ccp_sg_workarea *wa, struct device *dev, 76 struct scatterlist *sg, u64 len, 77 enum dma_data_direction dma_dir) 78 { 79 memset(wa, 0, sizeof(*wa)); 80 81 wa->sg = sg; 82 if (!sg) 83 return 0; 84 85 wa->nents = sg_nents_for_len(sg, len); 86 if (wa->nents < 0) 87 return wa->nents; 88 89 wa->bytes_left = len; 90 wa->sg_used = 0; 91 92 if (len == 0) 93 return 0; 94 95 if (dma_dir == DMA_NONE) 96 return 0; 97 98 wa->dma_sg = sg; 99 wa->dma_dev = dev; 100 wa->dma_dir = dma_dir; 101 wa->dma_count = dma_map_sg(dev, sg, wa->nents, dma_dir); 102 if (!wa->dma_count) 103 return -ENOMEM; 104 105 return 0; 106 } 107 108 static void ccp_update_sg_workarea(struct ccp_sg_workarea *wa, unsigned int len) 109 { 110 unsigned int nbytes = min_t(u64, len, wa->bytes_left); 111 112 if (!wa->sg) 113 return; 114 115 wa->sg_used += nbytes; 116 wa->bytes_left -= nbytes; 117 if (wa->sg_used == wa->sg->length) { 118 wa->sg = sg_next(wa->sg); 119 wa->sg_used = 0; 120 } 121 } 122 123 static void ccp_dm_free(struct ccp_dm_workarea *wa) 124 { 125 if (wa->length <= CCP_DMAPOOL_MAX_SIZE) { 126 if (wa->address) 127 dma_pool_free(wa->dma_pool, wa->address, 128 wa->dma.address); 129 } else { 130 if (wa->dma.address) 131 dma_unmap_single(wa->dev, wa->dma.address, wa->length, 132 wa->dma.dir); 133 kfree(wa->address); 134 } 135 136 wa->address = NULL; 137 wa->dma.address = 0; 138 } 139 140 static int ccp_init_dm_workarea(struct ccp_dm_workarea *wa, 141 struct ccp_cmd_queue *cmd_q, 142 unsigned int len, 143 enum dma_data_direction dir) 144 { 145 memset(wa, 0, sizeof(*wa)); 146 147 if (!len) 148 return 0; 149 150 wa->dev = cmd_q->ccp->dev; 151 wa->length = len; 152 153 if (len <= CCP_DMAPOOL_MAX_SIZE) { 154 wa->dma_pool = cmd_q->dma_pool; 155 156 wa->address = dma_pool_alloc(wa->dma_pool, GFP_KERNEL, 157 &wa->dma.address); 158 if (!wa->address) 159 return -ENOMEM; 160 161 wa->dma.length = CCP_DMAPOOL_MAX_SIZE; 162 163 memset(wa->address, 0, CCP_DMAPOOL_MAX_SIZE); 164 } else { 165 wa->address = kzalloc(len, GFP_KERNEL); 166 if (!wa->address) 167 return -ENOMEM; 168 169 wa->dma.address = dma_map_single(wa->dev, wa->address, len, 170 dir); 171 if (dma_mapping_error(wa->dev, wa->dma.address)) 172 return -ENOMEM; 173 174 wa->dma.length = len; 175 } 176 wa->dma.dir = dir; 177 178 return 0; 179 } 180 181 static int ccp_set_dm_area(struct ccp_dm_workarea *wa, unsigned int wa_offset, 182 struct scatterlist *sg, unsigned int sg_offset, 183 unsigned int len) 184 { 185 WARN_ON(!wa->address); 186 187 if (len > (wa->length - wa_offset)) 188 return -EINVAL; 189 190 scatterwalk_map_and_copy(wa->address + wa_offset, sg, sg_offset, len, 191 0); 192 return 0; 193 } 194 195 static void ccp_get_dm_area(struct ccp_dm_workarea *wa, unsigned int wa_offset, 196 struct scatterlist *sg, unsigned int sg_offset, 197 unsigned int len) 198 { 199 WARN_ON(!wa->address); 200 201 scatterwalk_map_and_copy(wa->address + wa_offset, sg, sg_offset, len, 202 1); 203 } 204 205 static int ccp_reverse_set_dm_area(struct ccp_dm_workarea *wa, 206 unsigned int wa_offset, 207 struct scatterlist *sg, 208 unsigned int sg_offset, 209 unsigned int len) 210 { 211 u8 *p, *q; 212 int rc; 213 214 rc = ccp_set_dm_area(wa, wa_offset, sg, sg_offset, len); 215 if (rc) 216 return rc; 217 218 p = wa->address + wa_offset; 219 q = p + len - 1; 220 while (p < q) { 221 *p = *p ^ *q; 222 *q = *p ^ *q; 223 *p = *p ^ *q; 224 p++; 225 q--; 226 } 227 return 0; 228 } 229 230 static void ccp_reverse_get_dm_area(struct ccp_dm_workarea *wa, 231 unsigned int wa_offset, 232 struct scatterlist *sg, 233 unsigned int sg_offset, 234 unsigned int len) 235 { 236 u8 *p, *q; 237 238 p = wa->address + wa_offset; 239 q = p + len - 1; 240 while (p < q) { 241 *p = *p ^ *q; 242 *q = *p ^ *q; 243 *p = *p ^ *q; 244 p++; 245 q--; 246 } 247 248 ccp_get_dm_area(wa, wa_offset, sg, sg_offset, len); 249 } 250 251 static void ccp_free_data(struct ccp_data *data, struct ccp_cmd_queue *cmd_q) 252 { 253 ccp_dm_free(&data->dm_wa); 254 ccp_sg_free(&data->sg_wa); 255 } 256 257 static int ccp_init_data(struct ccp_data *data, struct ccp_cmd_queue *cmd_q, 258 struct scatterlist *sg, u64 sg_len, 259 unsigned int dm_len, 260 enum dma_data_direction dir) 261 { 262 int ret; 263 264 memset(data, 0, sizeof(*data)); 265 266 ret = ccp_init_sg_workarea(&data->sg_wa, cmd_q->ccp->dev, sg, sg_len, 267 dir); 268 if (ret) 269 goto e_err; 270 271 ret = ccp_init_dm_workarea(&data->dm_wa, cmd_q, dm_len, dir); 272 if (ret) 273 goto e_err; 274 275 return 0; 276 277 e_err: 278 ccp_free_data(data, cmd_q); 279 280 return ret; 281 } 282 283 static unsigned int ccp_queue_buf(struct ccp_data *data, unsigned int from) 284 { 285 struct ccp_sg_workarea *sg_wa = &data->sg_wa; 286 struct ccp_dm_workarea *dm_wa = &data->dm_wa; 287 unsigned int buf_count, nbytes; 288 289 /* Clear the buffer if setting it */ 290 if (!from) 291 memset(dm_wa->address, 0, dm_wa->length); 292 293 if (!sg_wa->sg) 294 return 0; 295 296 /* Perform the copy operation 297 * nbytes will always be <= UINT_MAX because dm_wa->length is 298 * an unsigned int 299 */ 300 nbytes = min_t(u64, sg_wa->bytes_left, dm_wa->length); 301 scatterwalk_map_and_copy(dm_wa->address, sg_wa->sg, sg_wa->sg_used, 302 nbytes, from); 303 304 /* Update the structures and generate the count */ 305 buf_count = 0; 306 while (sg_wa->bytes_left && (buf_count < dm_wa->length)) { 307 nbytes = min(sg_wa->sg->length - sg_wa->sg_used, 308 dm_wa->length - buf_count); 309 nbytes = min_t(u64, sg_wa->bytes_left, nbytes); 310 311 buf_count += nbytes; 312 ccp_update_sg_workarea(sg_wa, nbytes); 313 } 314 315 return buf_count; 316 } 317 318 static unsigned int ccp_fill_queue_buf(struct ccp_data *data) 319 { 320 return ccp_queue_buf(data, 0); 321 } 322 323 static unsigned int ccp_empty_queue_buf(struct ccp_data *data) 324 { 325 return ccp_queue_buf(data, 1); 326 } 327 328 static void ccp_prepare_data(struct ccp_data *src, struct ccp_data *dst, 329 struct ccp_op *op, unsigned int block_size, 330 bool blocksize_op) 331 { 332 unsigned int sg_src_len, sg_dst_len, op_len; 333 334 /* The CCP can only DMA from/to one address each per operation. This 335 * requires that we find the smallest DMA area between the source 336 * and destination. The resulting len values will always be <= UINT_MAX 337 * because the dma length is an unsigned int. 338 */ 339 sg_src_len = sg_dma_len(src->sg_wa.sg) - src->sg_wa.sg_used; 340 sg_src_len = min_t(u64, src->sg_wa.bytes_left, sg_src_len); 341 342 if (dst) { 343 sg_dst_len = sg_dma_len(dst->sg_wa.sg) - dst->sg_wa.sg_used; 344 sg_dst_len = min_t(u64, src->sg_wa.bytes_left, sg_dst_len); 345 op_len = min(sg_src_len, sg_dst_len); 346 } else { 347 op_len = sg_src_len; 348 } 349 350 /* The data operation length will be at least block_size in length 351 * or the smaller of available sg room remaining for the source or 352 * the destination 353 */ 354 op_len = max(op_len, block_size); 355 356 /* Unless we have to buffer data, there's no reason to wait */ 357 op->soc = 0; 358 359 if (sg_src_len < block_size) { 360 /* Not enough data in the sg element, so it 361 * needs to be buffered into a blocksize chunk 362 */ 363 int cp_len = ccp_fill_queue_buf(src); 364 365 op->soc = 1; 366 op->src.u.dma.address = src->dm_wa.dma.address; 367 op->src.u.dma.offset = 0; 368 op->src.u.dma.length = (blocksize_op) ? block_size : cp_len; 369 } else { 370 /* Enough data in the sg element, but we need to 371 * adjust for any previously copied data 372 */ 373 op->src.u.dma.address = sg_dma_address(src->sg_wa.sg); 374 op->src.u.dma.offset = src->sg_wa.sg_used; 375 op->src.u.dma.length = op_len & ~(block_size - 1); 376 377 ccp_update_sg_workarea(&src->sg_wa, op->src.u.dma.length); 378 } 379 380 if (dst) { 381 if (sg_dst_len < block_size) { 382 /* Not enough room in the sg element or we're on the 383 * last piece of data (when using padding), so the 384 * output needs to be buffered into a blocksize chunk 385 */ 386 op->soc = 1; 387 op->dst.u.dma.address = dst->dm_wa.dma.address; 388 op->dst.u.dma.offset = 0; 389 op->dst.u.dma.length = op->src.u.dma.length; 390 } else { 391 /* Enough room in the sg element, but we need to 392 * adjust for any previously used area 393 */ 394 op->dst.u.dma.address = sg_dma_address(dst->sg_wa.sg); 395 op->dst.u.dma.offset = dst->sg_wa.sg_used; 396 op->dst.u.dma.length = op->src.u.dma.length; 397 } 398 } 399 } 400 401 static void ccp_process_data(struct ccp_data *src, struct ccp_data *dst, 402 struct ccp_op *op) 403 { 404 op->init = 0; 405 406 if (dst) { 407 if (op->dst.u.dma.address == dst->dm_wa.dma.address) 408 ccp_empty_queue_buf(dst); 409 else 410 ccp_update_sg_workarea(&dst->sg_wa, 411 op->dst.u.dma.length); 412 } 413 } 414 415 static int ccp_copy_to_from_sb(struct ccp_cmd_queue *cmd_q, 416 struct ccp_dm_workarea *wa, u32 jobid, u32 sb, 417 u32 byte_swap, bool from) 418 { 419 struct ccp_op op; 420 421 memset(&op, 0, sizeof(op)); 422 423 op.cmd_q = cmd_q; 424 op.jobid = jobid; 425 op.eom = 1; 426 427 if (from) { 428 op.soc = 1; 429 op.src.type = CCP_MEMTYPE_SB; 430 op.src.u.sb = sb; 431 op.dst.type = CCP_MEMTYPE_SYSTEM; 432 op.dst.u.dma.address = wa->dma.address; 433 op.dst.u.dma.length = wa->length; 434 } else { 435 op.src.type = CCP_MEMTYPE_SYSTEM; 436 op.src.u.dma.address = wa->dma.address; 437 op.src.u.dma.length = wa->length; 438 op.dst.type = CCP_MEMTYPE_SB; 439 op.dst.u.sb = sb; 440 } 441 442 op.u.passthru.byte_swap = byte_swap; 443 444 return cmd_q->ccp->vdata->perform->passthru(&op); 445 } 446 447 static int ccp_copy_to_sb(struct ccp_cmd_queue *cmd_q, 448 struct ccp_dm_workarea *wa, u32 jobid, u32 sb, 449 u32 byte_swap) 450 { 451 return ccp_copy_to_from_sb(cmd_q, wa, jobid, sb, byte_swap, false); 452 } 453 454 static int ccp_copy_from_sb(struct ccp_cmd_queue *cmd_q, 455 struct ccp_dm_workarea *wa, u32 jobid, u32 sb, 456 u32 byte_swap) 457 { 458 return ccp_copy_to_from_sb(cmd_q, wa, jobid, sb, byte_swap, true); 459 } 460 461 static int ccp_run_aes_cmac_cmd(struct ccp_cmd_queue *cmd_q, 462 struct ccp_cmd *cmd) 463 { 464 struct ccp_aes_engine *aes = &cmd->u.aes; 465 struct ccp_dm_workarea key, ctx; 466 struct ccp_data src; 467 struct ccp_op op; 468 unsigned int dm_offset; 469 int ret; 470 471 if (!((aes->key_len == AES_KEYSIZE_128) || 472 (aes->key_len == AES_KEYSIZE_192) || 473 (aes->key_len == AES_KEYSIZE_256))) 474 return -EINVAL; 475 476 if (aes->src_len & (AES_BLOCK_SIZE - 1)) 477 return -EINVAL; 478 479 if (aes->iv_len != AES_BLOCK_SIZE) 480 return -EINVAL; 481 482 if (!aes->key || !aes->iv || !aes->src) 483 return -EINVAL; 484 485 if (aes->cmac_final) { 486 if (aes->cmac_key_len != AES_BLOCK_SIZE) 487 return -EINVAL; 488 489 if (!aes->cmac_key) 490 return -EINVAL; 491 } 492 493 BUILD_BUG_ON(CCP_AES_KEY_SB_COUNT != 1); 494 BUILD_BUG_ON(CCP_AES_CTX_SB_COUNT != 1); 495 496 ret = -EIO; 497 memset(&op, 0, sizeof(op)); 498 op.cmd_q = cmd_q; 499 op.jobid = CCP_NEW_JOBID(cmd_q->ccp); 500 op.sb_key = cmd_q->sb_key; 501 op.sb_ctx = cmd_q->sb_ctx; 502 op.init = 1; 503 op.u.aes.type = aes->type; 504 op.u.aes.mode = aes->mode; 505 op.u.aes.action = aes->action; 506 507 /* All supported key sizes fit in a single (32-byte) SB entry 508 * and must be in little endian format. Use the 256-bit byte 509 * swap passthru option to convert from big endian to little 510 * endian. 511 */ 512 ret = ccp_init_dm_workarea(&key, cmd_q, 513 CCP_AES_KEY_SB_COUNT * CCP_SB_BYTES, 514 DMA_TO_DEVICE); 515 if (ret) 516 return ret; 517 518 dm_offset = CCP_SB_BYTES - aes->key_len; 519 ret = ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len); 520 if (ret) 521 goto e_key; 522 ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key, 523 CCP_PASSTHRU_BYTESWAP_256BIT); 524 if (ret) { 525 cmd->engine_error = cmd_q->cmd_error; 526 goto e_key; 527 } 528 529 /* The AES context fits in a single (32-byte) SB entry and 530 * must be in little endian format. Use the 256-bit byte swap 531 * passthru option to convert from big endian to little endian. 532 */ 533 ret = ccp_init_dm_workarea(&ctx, cmd_q, 534 CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES, 535 DMA_BIDIRECTIONAL); 536 if (ret) 537 goto e_key; 538 539 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE; 540 ret = ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len); 541 if (ret) 542 goto e_ctx; 543 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, 544 CCP_PASSTHRU_BYTESWAP_256BIT); 545 if (ret) { 546 cmd->engine_error = cmd_q->cmd_error; 547 goto e_ctx; 548 } 549 550 /* Send data to the CCP AES engine */ 551 ret = ccp_init_data(&src, cmd_q, aes->src, aes->src_len, 552 AES_BLOCK_SIZE, DMA_TO_DEVICE); 553 if (ret) 554 goto e_ctx; 555 556 while (src.sg_wa.bytes_left) { 557 ccp_prepare_data(&src, NULL, &op, AES_BLOCK_SIZE, true); 558 if (aes->cmac_final && !src.sg_wa.bytes_left) { 559 op.eom = 1; 560 561 /* Push the K1/K2 key to the CCP now */ 562 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, 563 op.sb_ctx, 564 CCP_PASSTHRU_BYTESWAP_256BIT); 565 if (ret) { 566 cmd->engine_error = cmd_q->cmd_error; 567 goto e_src; 568 } 569 570 ret = ccp_set_dm_area(&ctx, 0, aes->cmac_key, 0, 571 aes->cmac_key_len); 572 if (ret) 573 goto e_src; 574 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, 575 CCP_PASSTHRU_BYTESWAP_256BIT); 576 if (ret) { 577 cmd->engine_error = cmd_q->cmd_error; 578 goto e_src; 579 } 580 } 581 582 ret = cmd_q->ccp->vdata->perform->aes(&op); 583 if (ret) { 584 cmd->engine_error = cmd_q->cmd_error; 585 goto e_src; 586 } 587 588 ccp_process_data(&src, NULL, &op); 589 } 590 591 /* Retrieve the AES context - convert from LE to BE using 592 * 32-byte (256-bit) byteswapping 593 */ 594 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, 595 CCP_PASSTHRU_BYTESWAP_256BIT); 596 if (ret) { 597 cmd->engine_error = cmd_q->cmd_error; 598 goto e_src; 599 } 600 601 /* ...but we only need AES_BLOCK_SIZE bytes */ 602 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE; 603 ccp_get_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len); 604 605 e_src: 606 ccp_free_data(&src, cmd_q); 607 608 e_ctx: 609 ccp_dm_free(&ctx); 610 611 e_key: 612 ccp_dm_free(&key); 613 614 return ret; 615 } 616 617 static int ccp_run_aes_gcm_cmd(struct ccp_cmd_queue *cmd_q, 618 struct ccp_cmd *cmd) 619 { 620 struct ccp_aes_engine *aes = &cmd->u.aes; 621 struct ccp_dm_workarea key, ctx, final_wa, tag; 622 struct ccp_data src, dst; 623 struct ccp_data aad; 624 struct ccp_op op; 625 626 unsigned long long *final; 627 unsigned int dm_offset; 628 unsigned int ilen; 629 bool in_place = true; /* Default value */ 630 int ret; 631 632 struct scatterlist *p_inp, sg_inp[2]; 633 struct scatterlist *p_tag, sg_tag[2]; 634 struct scatterlist *p_outp, sg_outp[2]; 635 struct scatterlist *p_aad; 636 637 if (!aes->iv) 638 return -EINVAL; 639 640 if (!((aes->key_len == AES_KEYSIZE_128) || 641 (aes->key_len == AES_KEYSIZE_192) || 642 (aes->key_len == AES_KEYSIZE_256))) 643 return -EINVAL; 644 645 if (!aes->key) /* Gotta have a key SGL */ 646 return -EINVAL; 647 648 /* First, decompose the source buffer into AAD & PT, 649 * and the destination buffer into AAD, CT & tag, or 650 * the input into CT & tag. 651 * It is expected that the input and output SGs will 652 * be valid, even if the AAD and input lengths are 0. 653 */ 654 p_aad = aes->src; 655 p_inp = scatterwalk_ffwd(sg_inp, aes->src, aes->aad_len); 656 p_outp = scatterwalk_ffwd(sg_outp, aes->dst, aes->aad_len); 657 if (aes->action == CCP_AES_ACTION_ENCRYPT) { 658 ilen = aes->src_len; 659 p_tag = scatterwalk_ffwd(sg_tag, p_outp, ilen); 660 } else { 661 /* Input length for decryption includes tag */ 662 ilen = aes->src_len - AES_BLOCK_SIZE; 663 p_tag = scatterwalk_ffwd(sg_tag, p_inp, ilen); 664 } 665 666 memset(&op, 0, sizeof(op)); 667 op.cmd_q = cmd_q; 668 op.jobid = CCP_NEW_JOBID(cmd_q->ccp); 669 op.sb_key = cmd_q->sb_key; /* Pre-allocated */ 670 op.sb_ctx = cmd_q->sb_ctx; /* Pre-allocated */ 671 op.init = 1; 672 op.u.aes.type = aes->type; 673 674 /* Copy the key to the LSB */ 675 ret = ccp_init_dm_workarea(&key, cmd_q, 676 CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES, 677 DMA_TO_DEVICE); 678 if (ret) 679 return ret; 680 681 dm_offset = CCP_SB_BYTES - aes->key_len; 682 ret = ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len); 683 if (ret) 684 goto e_key; 685 ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key, 686 CCP_PASSTHRU_BYTESWAP_256BIT); 687 if (ret) { 688 cmd->engine_error = cmd_q->cmd_error; 689 goto e_key; 690 } 691 692 /* Copy the context (IV) to the LSB. 693 * There is an assumption here that the IV is 96 bits in length, plus 694 * a nonce of 32 bits. If no IV is present, use a zeroed buffer. 695 */ 696 ret = ccp_init_dm_workarea(&ctx, cmd_q, 697 CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES, 698 DMA_BIDIRECTIONAL); 699 if (ret) 700 goto e_key; 701 702 dm_offset = CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES - aes->iv_len; 703 ret = ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len); 704 if (ret) 705 goto e_ctx; 706 707 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, 708 CCP_PASSTHRU_BYTESWAP_256BIT); 709 if (ret) { 710 cmd->engine_error = cmd_q->cmd_error; 711 goto e_ctx; 712 } 713 714 op.init = 1; 715 if (aes->aad_len > 0) { 716 /* Step 1: Run a GHASH over the Additional Authenticated Data */ 717 ret = ccp_init_data(&aad, cmd_q, p_aad, aes->aad_len, 718 AES_BLOCK_SIZE, 719 DMA_TO_DEVICE); 720 if (ret) 721 goto e_ctx; 722 723 op.u.aes.mode = CCP_AES_MODE_GHASH; 724 op.u.aes.action = CCP_AES_GHASHAAD; 725 726 while (aad.sg_wa.bytes_left) { 727 ccp_prepare_data(&aad, NULL, &op, AES_BLOCK_SIZE, true); 728 729 ret = cmd_q->ccp->vdata->perform->aes(&op); 730 if (ret) { 731 cmd->engine_error = cmd_q->cmd_error; 732 goto e_aad; 733 } 734 735 ccp_process_data(&aad, NULL, &op); 736 op.init = 0; 737 } 738 } 739 740 op.u.aes.mode = CCP_AES_MODE_GCTR; 741 op.u.aes.action = aes->action; 742 743 if (ilen > 0) { 744 /* Step 2: Run a GCTR over the plaintext */ 745 in_place = (sg_virt(p_inp) == sg_virt(p_outp)) ? true : false; 746 747 ret = ccp_init_data(&src, cmd_q, p_inp, ilen, 748 AES_BLOCK_SIZE, 749 in_place ? DMA_BIDIRECTIONAL 750 : DMA_TO_DEVICE); 751 if (ret) 752 goto e_ctx; 753 754 if (in_place) { 755 dst = src; 756 } else { 757 ret = ccp_init_data(&dst, cmd_q, p_outp, ilen, 758 AES_BLOCK_SIZE, DMA_FROM_DEVICE); 759 if (ret) 760 goto e_src; 761 } 762 763 op.soc = 0; 764 op.eom = 0; 765 op.init = 1; 766 while (src.sg_wa.bytes_left) { 767 ccp_prepare_data(&src, &dst, &op, AES_BLOCK_SIZE, true); 768 if (!src.sg_wa.bytes_left) { 769 unsigned int nbytes = aes->src_len 770 % AES_BLOCK_SIZE; 771 772 if (nbytes) { 773 op.eom = 1; 774 op.u.aes.size = (nbytes * 8) - 1; 775 } 776 } 777 778 ret = cmd_q->ccp->vdata->perform->aes(&op); 779 if (ret) { 780 cmd->engine_error = cmd_q->cmd_error; 781 goto e_dst; 782 } 783 784 ccp_process_data(&src, &dst, &op); 785 op.init = 0; 786 } 787 } 788 789 /* Step 3: Update the IV portion of the context with the original IV */ 790 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, 791 CCP_PASSTHRU_BYTESWAP_256BIT); 792 if (ret) { 793 cmd->engine_error = cmd_q->cmd_error; 794 goto e_dst; 795 } 796 797 ret = ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len); 798 if (ret) 799 goto e_dst; 800 801 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, 802 CCP_PASSTHRU_BYTESWAP_256BIT); 803 if (ret) { 804 cmd->engine_error = cmd_q->cmd_error; 805 goto e_dst; 806 } 807 808 /* Step 4: Concatenate the lengths of the AAD and source, and 809 * hash that 16 byte buffer. 810 */ 811 ret = ccp_init_dm_workarea(&final_wa, cmd_q, AES_BLOCK_SIZE, 812 DMA_BIDIRECTIONAL); 813 if (ret) 814 goto e_dst; 815 final = (unsigned long long *) final_wa.address; 816 final[0] = cpu_to_be64(aes->aad_len * 8); 817 final[1] = cpu_to_be64(ilen * 8); 818 819 op.u.aes.mode = CCP_AES_MODE_GHASH; 820 op.u.aes.action = CCP_AES_GHASHFINAL; 821 op.src.type = CCP_MEMTYPE_SYSTEM; 822 op.src.u.dma.address = final_wa.dma.address; 823 op.src.u.dma.length = AES_BLOCK_SIZE; 824 op.dst.type = CCP_MEMTYPE_SYSTEM; 825 op.dst.u.dma.address = final_wa.dma.address; 826 op.dst.u.dma.length = AES_BLOCK_SIZE; 827 op.eom = 1; 828 op.u.aes.size = 0; 829 ret = cmd_q->ccp->vdata->perform->aes(&op); 830 if (ret) 831 goto e_dst; 832 833 if (aes->action == CCP_AES_ACTION_ENCRYPT) { 834 /* Put the ciphered tag after the ciphertext. */ 835 ccp_get_dm_area(&final_wa, 0, p_tag, 0, AES_BLOCK_SIZE); 836 } else { 837 /* Does this ciphered tag match the input? */ 838 ret = ccp_init_dm_workarea(&tag, cmd_q, AES_BLOCK_SIZE, 839 DMA_BIDIRECTIONAL); 840 if (ret) 841 goto e_tag; 842 ret = ccp_set_dm_area(&tag, 0, p_tag, 0, AES_BLOCK_SIZE); 843 if (ret) 844 goto e_tag; 845 846 ret = memcmp(tag.address, final_wa.address, AES_BLOCK_SIZE); 847 ccp_dm_free(&tag); 848 } 849 850 e_tag: 851 ccp_dm_free(&final_wa); 852 853 e_dst: 854 if (aes->src_len && !in_place) 855 ccp_free_data(&dst, cmd_q); 856 857 e_src: 858 if (aes->src_len) 859 ccp_free_data(&src, cmd_q); 860 861 e_aad: 862 if (aes->aad_len) 863 ccp_free_data(&aad, cmd_q); 864 865 e_ctx: 866 ccp_dm_free(&ctx); 867 868 e_key: 869 ccp_dm_free(&key); 870 871 return ret; 872 } 873 874 static int ccp_run_aes_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) 875 { 876 struct ccp_aes_engine *aes = &cmd->u.aes; 877 struct ccp_dm_workarea key, ctx; 878 struct ccp_data src, dst; 879 struct ccp_op op; 880 unsigned int dm_offset; 881 bool in_place = false; 882 int ret; 883 884 if (aes->mode == CCP_AES_MODE_CMAC) 885 return ccp_run_aes_cmac_cmd(cmd_q, cmd); 886 887 if (aes->mode == CCP_AES_MODE_GCM) 888 return ccp_run_aes_gcm_cmd(cmd_q, cmd); 889 890 if (!((aes->key_len == AES_KEYSIZE_128) || 891 (aes->key_len == AES_KEYSIZE_192) || 892 (aes->key_len == AES_KEYSIZE_256))) 893 return -EINVAL; 894 895 if (((aes->mode == CCP_AES_MODE_ECB) || 896 (aes->mode == CCP_AES_MODE_CBC) || 897 (aes->mode == CCP_AES_MODE_CFB)) && 898 (aes->src_len & (AES_BLOCK_SIZE - 1))) 899 return -EINVAL; 900 901 if (!aes->key || !aes->src || !aes->dst) 902 return -EINVAL; 903 904 if (aes->mode != CCP_AES_MODE_ECB) { 905 if (aes->iv_len != AES_BLOCK_SIZE) 906 return -EINVAL; 907 908 if (!aes->iv) 909 return -EINVAL; 910 } 911 912 BUILD_BUG_ON(CCP_AES_KEY_SB_COUNT != 1); 913 BUILD_BUG_ON(CCP_AES_CTX_SB_COUNT != 1); 914 915 ret = -EIO; 916 memset(&op, 0, sizeof(op)); 917 op.cmd_q = cmd_q; 918 op.jobid = CCP_NEW_JOBID(cmd_q->ccp); 919 op.sb_key = cmd_q->sb_key; 920 op.sb_ctx = cmd_q->sb_ctx; 921 op.init = (aes->mode == CCP_AES_MODE_ECB) ? 0 : 1; 922 op.u.aes.type = aes->type; 923 op.u.aes.mode = aes->mode; 924 op.u.aes.action = aes->action; 925 926 /* All supported key sizes fit in a single (32-byte) SB entry 927 * and must be in little endian format. Use the 256-bit byte 928 * swap passthru option to convert from big endian to little 929 * endian. 930 */ 931 ret = ccp_init_dm_workarea(&key, cmd_q, 932 CCP_AES_KEY_SB_COUNT * CCP_SB_BYTES, 933 DMA_TO_DEVICE); 934 if (ret) 935 return ret; 936 937 dm_offset = CCP_SB_BYTES - aes->key_len; 938 ret = ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len); 939 if (ret) 940 goto e_key; 941 ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key, 942 CCP_PASSTHRU_BYTESWAP_256BIT); 943 if (ret) { 944 cmd->engine_error = cmd_q->cmd_error; 945 goto e_key; 946 } 947 948 /* The AES context fits in a single (32-byte) SB entry and 949 * must be in little endian format. Use the 256-bit byte swap 950 * passthru option to convert from big endian to little endian. 951 */ 952 ret = ccp_init_dm_workarea(&ctx, cmd_q, 953 CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES, 954 DMA_BIDIRECTIONAL); 955 if (ret) 956 goto e_key; 957 958 if (aes->mode != CCP_AES_MODE_ECB) { 959 /* Load the AES context - convert to LE */ 960 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE; 961 ret = ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len); 962 if (ret) 963 goto e_ctx; 964 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, 965 CCP_PASSTHRU_BYTESWAP_256BIT); 966 if (ret) { 967 cmd->engine_error = cmd_q->cmd_error; 968 goto e_ctx; 969 } 970 } 971 switch (aes->mode) { 972 case CCP_AES_MODE_CFB: /* CFB128 only */ 973 case CCP_AES_MODE_CTR: 974 op.u.aes.size = AES_BLOCK_SIZE * BITS_PER_BYTE - 1; 975 break; 976 default: 977 op.u.aes.size = 0; 978 } 979 980 /* Prepare the input and output data workareas. For in-place 981 * operations we need to set the dma direction to BIDIRECTIONAL 982 * and copy the src workarea to the dst workarea. 983 */ 984 if (sg_virt(aes->src) == sg_virt(aes->dst)) 985 in_place = true; 986 987 ret = ccp_init_data(&src, cmd_q, aes->src, aes->src_len, 988 AES_BLOCK_SIZE, 989 in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE); 990 if (ret) 991 goto e_ctx; 992 993 if (in_place) { 994 dst = src; 995 } else { 996 ret = ccp_init_data(&dst, cmd_q, aes->dst, aes->src_len, 997 AES_BLOCK_SIZE, DMA_FROM_DEVICE); 998 if (ret) 999 goto e_src; 1000 } 1001 1002 /* Send data to the CCP AES engine */ 1003 while (src.sg_wa.bytes_left) { 1004 ccp_prepare_data(&src, &dst, &op, AES_BLOCK_SIZE, true); 1005 if (!src.sg_wa.bytes_left) { 1006 op.eom = 1; 1007 1008 /* Since we don't retrieve the AES context in ECB 1009 * mode we have to wait for the operation to complete 1010 * on the last piece of data 1011 */ 1012 if (aes->mode == CCP_AES_MODE_ECB) 1013 op.soc = 1; 1014 } 1015 1016 ret = cmd_q->ccp->vdata->perform->aes(&op); 1017 if (ret) { 1018 cmd->engine_error = cmd_q->cmd_error; 1019 goto e_dst; 1020 } 1021 1022 ccp_process_data(&src, &dst, &op); 1023 } 1024 1025 if (aes->mode != CCP_AES_MODE_ECB) { 1026 /* Retrieve the AES context - convert from LE to BE using 1027 * 32-byte (256-bit) byteswapping 1028 */ 1029 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, 1030 CCP_PASSTHRU_BYTESWAP_256BIT); 1031 if (ret) { 1032 cmd->engine_error = cmd_q->cmd_error; 1033 goto e_dst; 1034 } 1035 1036 /* ...but we only need AES_BLOCK_SIZE bytes */ 1037 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE; 1038 ccp_get_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len); 1039 } 1040 1041 e_dst: 1042 if (!in_place) 1043 ccp_free_data(&dst, cmd_q); 1044 1045 e_src: 1046 ccp_free_data(&src, cmd_q); 1047 1048 e_ctx: 1049 ccp_dm_free(&ctx); 1050 1051 e_key: 1052 ccp_dm_free(&key); 1053 1054 return ret; 1055 } 1056 1057 static int ccp_run_xts_aes_cmd(struct ccp_cmd_queue *cmd_q, 1058 struct ccp_cmd *cmd) 1059 { 1060 struct ccp_xts_aes_engine *xts = &cmd->u.xts; 1061 struct ccp_dm_workarea key, ctx; 1062 struct ccp_data src, dst; 1063 struct ccp_op op; 1064 unsigned int unit_size, dm_offset; 1065 bool in_place = false; 1066 unsigned int sb_count; 1067 enum ccp_aes_type aestype; 1068 int ret; 1069 1070 switch (xts->unit_size) { 1071 case CCP_XTS_AES_UNIT_SIZE_16: 1072 unit_size = 16; 1073 break; 1074 case CCP_XTS_AES_UNIT_SIZE_512: 1075 unit_size = 512; 1076 break; 1077 case CCP_XTS_AES_UNIT_SIZE_1024: 1078 unit_size = 1024; 1079 break; 1080 case CCP_XTS_AES_UNIT_SIZE_2048: 1081 unit_size = 2048; 1082 break; 1083 case CCP_XTS_AES_UNIT_SIZE_4096: 1084 unit_size = 4096; 1085 break; 1086 1087 default: 1088 return -EINVAL; 1089 } 1090 1091 if (xts->key_len == AES_KEYSIZE_128) 1092 aestype = CCP_AES_TYPE_128; 1093 else if (xts->key_len == AES_KEYSIZE_256) 1094 aestype = CCP_AES_TYPE_256; 1095 else 1096 return -EINVAL; 1097 1098 if (!xts->final && (xts->src_len & (AES_BLOCK_SIZE - 1))) 1099 return -EINVAL; 1100 1101 if (xts->iv_len != AES_BLOCK_SIZE) 1102 return -EINVAL; 1103 1104 if (!xts->key || !xts->iv || !xts->src || !xts->dst) 1105 return -EINVAL; 1106 1107 BUILD_BUG_ON(CCP_XTS_AES_KEY_SB_COUNT != 1); 1108 BUILD_BUG_ON(CCP_XTS_AES_CTX_SB_COUNT != 1); 1109 1110 ret = -EIO; 1111 memset(&op, 0, sizeof(op)); 1112 op.cmd_q = cmd_q; 1113 op.jobid = CCP_NEW_JOBID(cmd_q->ccp); 1114 op.sb_key = cmd_q->sb_key; 1115 op.sb_ctx = cmd_q->sb_ctx; 1116 op.init = 1; 1117 op.u.xts.type = aestype; 1118 op.u.xts.action = xts->action; 1119 op.u.xts.unit_size = xts->unit_size; 1120 1121 /* A version 3 device only supports 128-bit keys, which fits into a 1122 * single SB entry. A version 5 device uses a 512-bit vector, so two 1123 * SB entries. 1124 */ 1125 if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0)) 1126 sb_count = CCP_XTS_AES_KEY_SB_COUNT; 1127 else 1128 sb_count = CCP5_XTS_AES_KEY_SB_COUNT; 1129 ret = ccp_init_dm_workarea(&key, cmd_q, 1130 sb_count * CCP_SB_BYTES, 1131 DMA_TO_DEVICE); 1132 if (ret) 1133 return ret; 1134 1135 if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0)) { 1136 /* All supported key sizes must be in little endian format. 1137 * Use the 256-bit byte swap passthru option to convert from 1138 * big endian to little endian. 1139 */ 1140 dm_offset = CCP_SB_BYTES - AES_KEYSIZE_128; 1141 ret = ccp_set_dm_area(&key, dm_offset, xts->key, 0, xts->key_len); 1142 if (ret) 1143 goto e_key; 1144 ret = ccp_set_dm_area(&key, 0, xts->key, xts->key_len, xts->key_len); 1145 if (ret) 1146 goto e_key; 1147 } else { 1148 /* Version 5 CCPs use a 512-bit space for the key: each portion 1149 * occupies 256 bits, or one entire slot, and is zero-padded. 1150 */ 1151 unsigned int pad; 1152 1153 dm_offset = CCP_SB_BYTES; 1154 pad = dm_offset - xts->key_len; 1155 ret = ccp_set_dm_area(&key, pad, xts->key, 0, xts->key_len); 1156 if (ret) 1157 goto e_key; 1158 ret = ccp_set_dm_area(&key, dm_offset + pad, xts->key, 1159 xts->key_len, xts->key_len); 1160 if (ret) 1161 goto e_key; 1162 } 1163 ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key, 1164 CCP_PASSTHRU_BYTESWAP_256BIT); 1165 if (ret) { 1166 cmd->engine_error = cmd_q->cmd_error; 1167 goto e_key; 1168 } 1169 1170 /* The AES context fits in a single (32-byte) SB entry and 1171 * for XTS is already in little endian format so no byte swapping 1172 * is needed. 1173 */ 1174 ret = ccp_init_dm_workarea(&ctx, cmd_q, 1175 CCP_XTS_AES_CTX_SB_COUNT * CCP_SB_BYTES, 1176 DMA_BIDIRECTIONAL); 1177 if (ret) 1178 goto e_key; 1179 1180 ret = ccp_set_dm_area(&ctx, 0, xts->iv, 0, xts->iv_len); 1181 if (ret) 1182 goto e_ctx; 1183 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, 1184 CCP_PASSTHRU_BYTESWAP_NOOP); 1185 if (ret) { 1186 cmd->engine_error = cmd_q->cmd_error; 1187 goto e_ctx; 1188 } 1189 1190 /* Prepare the input and output data workareas. For in-place 1191 * operations we need to set the dma direction to BIDIRECTIONAL 1192 * and copy the src workarea to the dst workarea. 1193 */ 1194 if (sg_virt(xts->src) == sg_virt(xts->dst)) 1195 in_place = true; 1196 1197 ret = ccp_init_data(&src, cmd_q, xts->src, xts->src_len, 1198 unit_size, 1199 in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE); 1200 if (ret) 1201 goto e_ctx; 1202 1203 if (in_place) { 1204 dst = src; 1205 } else { 1206 ret = ccp_init_data(&dst, cmd_q, xts->dst, xts->src_len, 1207 unit_size, DMA_FROM_DEVICE); 1208 if (ret) 1209 goto e_src; 1210 } 1211 1212 /* Send data to the CCP AES engine */ 1213 while (src.sg_wa.bytes_left) { 1214 ccp_prepare_data(&src, &dst, &op, unit_size, true); 1215 if (!src.sg_wa.bytes_left) 1216 op.eom = 1; 1217 1218 ret = cmd_q->ccp->vdata->perform->xts_aes(&op); 1219 if (ret) { 1220 cmd->engine_error = cmd_q->cmd_error; 1221 goto e_dst; 1222 } 1223 1224 ccp_process_data(&src, &dst, &op); 1225 } 1226 1227 /* Retrieve the AES context - convert from LE to BE using 1228 * 32-byte (256-bit) byteswapping 1229 */ 1230 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, 1231 CCP_PASSTHRU_BYTESWAP_256BIT); 1232 if (ret) { 1233 cmd->engine_error = cmd_q->cmd_error; 1234 goto e_dst; 1235 } 1236 1237 /* ...but we only need AES_BLOCK_SIZE bytes */ 1238 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE; 1239 ccp_get_dm_area(&ctx, dm_offset, xts->iv, 0, xts->iv_len); 1240 1241 e_dst: 1242 if (!in_place) 1243 ccp_free_data(&dst, cmd_q); 1244 1245 e_src: 1246 ccp_free_data(&src, cmd_q); 1247 1248 e_ctx: 1249 ccp_dm_free(&ctx); 1250 1251 e_key: 1252 ccp_dm_free(&key); 1253 1254 return ret; 1255 } 1256 1257 static int ccp_run_des3_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) 1258 { 1259 struct ccp_des3_engine *des3 = &cmd->u.des3; 1260 1261 struct ccp_dm_workarea key, ctx; 1262 struct ccp_data src, dst; 1263 struct ccp_op op; 1264 unsigned int dm_offset; 1265 unsigned int len_singlekey; 1266 bool in_place = false; 1267 int ret; 1268 1269 /* Error checks */ 1270 if (!cmd_q->ccp->vdata->perform->des3) 1271 return -EINVAL; 1272 1273 if (des3->key_len != DES3_EDE_KEY_SIZE) 1274 return -EINVAL; 1275 1276 if (((des3->mode == CCP_DES3_MODE_ECB) || 1277 (des3->mode == CCP_DES3_MODE_CBC)) && 1278 (des3->src_len & (DES3_EDE_BLOCK_SIZE - 1))) 1279 return -EINVAL; 1280 1281 if (!des3->key || !des3->src || !des3->dst) 1282 return -EINVAL; 1283 1284 if (des3->mode != CCP_DES3_MODE_ECB) { 1285 if (des3->iv_len != DES3_EDE_BLOCK_SIZE) 1286 return -EINVAL; 1287 1288 if (!des3->iv) 1289 return -EINVAL; 1290 } 1291 1292 ret = -EIO; 1293 /* Zero out all the fields of the command desc */ 1294 memset(&op, 0, sizeof(op)); 1295 1296 /* Set up the Function field */ 1297 op.cmd_q = cmd_q; 1298 op.jobid = CCP_NEW_JOBID(cmd_q->ccp); 1299 op.sb_key = cmd_q->sb_key; 1300 1301 op.init = (des3->mode == CCP_DES3_MODE_ECB) ? 0 : 1; 1302 op.u.des3.type = des3->type; 1303 op.u.des3.mode = des3->mode; 1304 op.u.des3.action = des3->action; 1305 1306 /* 1307 * All supported key sizes fit in a single (32-byte) KSB entry and 1308 * (like AES) must be in little endian format. Use the 256-bit byte 1309 * swap passthru option to convert from big endian to little endian. 1310 */ 1311 ret = ccp_init_dm_workarea(&key, cmd_q, 1312 CCP_DES3_KEY_SB_COUNT * CCP_SB_BYTES, 1313 DMA_TO_DEVICE); 1314 if (ret) 1315 return ret; 1316 1317 /* 1318 * The contents of the key triplet are in the reverse order of what 1319 * is required by the engine. Copy the 3 pieces individually to put 1320 * them where they belong. 1321 */ 1322 dm_offset = CCP_SB_BYTES - des3->key_len; /* Basic offset */ 1323 1324 len_singlekey = des3->key_len / 3; 1325 ret = ccp_set_dm_area(&key, dm_offset + 2 * len_singlekey, 1326 des3->key, 0, len_singlekey); 1327 if (ret) 1328 goto e_key; 1329 ret = ccp_set_dm_area(&key, dm_offset + len_singlekey, 1330 des3->key, len_singlekey, len_singlekey); 1331 if (ret) 1332 goto e_key; 1333 ret = ccp_set_dm_area(&key, dm_offset, 1334 des3->key, 2 * len_singlekey, len_singlekey); 1335 if (ret) 1336 goto e_key; 1337 1338 /* Copy the key to the SB */ 1339 ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key, 1340 CCP_PASSTHRU_BYTESWAP_256BIT); 1341 if (ret) { 1342 cmd->engine_error = cmd_q->cmd_error; 1343 goto e_key; 1344 } 1345 1346 /* 1347 * The DES3 context fits in a single (32-byte) KSB entry and 1348 * must be in little endian format. Use the 256-bit byte swap 1349 * passthru option to convert from big endian to little endian. 1350 */ 1351 if (des3->mode != CCP_DES3_MODE_ECB) { 1352 u32 load_mode; 1353 1354 op.sb_ctx = cmd_q->sb_ctx; 1355 1356 ret = ccp_init_dm_workarea(&ctx, cmd_q, 1357 CCP_DES3_CTX_SB_COUNT * CCP_SB_BYTES, 1358 DMA_BIDIRECTIONAL); 1359 if (ret) 1360 goto e_key; 1361 1362 /* Load the context into the LSB */ 1363 dm_offset = CCP_SB_BYTES - des3->iv_len; 1364 ret = ccp_set_dm_area(&ctx, dm_offset, des3->iv, 0, 1365 des3->iv_len); 1366 if (ret) 1367 goto e_ctx; 1368 1369 if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0)) 1370 load_mode = CCP_PASSTHRU_BYTESWAP_NOOP; 1371 else 1372 load_mode = CCP_PASSTHRU_BYTESWAP_256BIT; 1373 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, 1374 load_mode); 1375 if (ret) { 1376 cmd->engine_error = cmd_q->cmd_error; 1377 goto e_ctx; 1378 } 1379 } 1380 1381 /* 1382 * Prepare the input and output data workareas. For in-place 1383 * operations we need to set the dma direction to BIDIRECTIONAL 1384 * and copy the src workarea to the dst workarea. 1385 */ 1386 if (sg_virt(des3->src) == sg_virt(des3->dst)) 1387 in_place = true; 1388 1389 ret = ccp_init_data(&src, cmd_q, des3->src, des3->src_len, 1390 DES3_EDE_BLOCK_SIZE, 1391 in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE); 1392 if (ret) 1393 goto e_ctx; 1394 1395 if (in_place) 1396 dst = src; 1397 else { 1398 ret = ccp_init_data(&dst, cmd_q, des3->dst, des3->src_len, 1399 DES3_EDE_BLOCK_SIZE, DMA_FROM_DEVICE); 1400 if (ret) 1401 goto e_src; 1402 } 1403 1404 /* Send data to the CCP DES3 engine */ 1405 while (src.sg_wa.bytes_left) { 1406 ccp_prepare_data(&src, &dst, &op, DES3_EDE_BLOCK_SIZE, true); 1407 if (!src.sg_wa.bytes_left) { 1408 op.eom = 1; 1409 1410 /* Since we don't retrieve the context in ECB mode 1411 * we have to wait for the operation to complete 1412 * on the last piece of data 1413 */ 1414 op.soc = 0; 1415 } 1416 1417 ret = cmd_q->ccp->vdata->perform->des3(&op); 1418 if (ret) { 1419 cmd->engine_error = cmd_q->cmd_error; 1420 goto e_dst; 1421 } 1422 1423 ccp_process_data(&src, &dst, &op); 1424 } 1425 1426 if (des3->mode != CCP_DES3_MODE_ECB) { 1427 /* Retrieve the context and make BE */ 1428 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, 1429 CCP_PASSTHRU_BYTESWAP_256BIT); 1430 if (ret) { 1431 cmd->engine_error = cmd_q->cmd_error; 1432 goto e_dst; 1433 } 1434 1435 /* ...but we only need the last DES3_EDE_BLOCK_SIZE bytes */ 1436 if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0)) 1437 dm_offset = CCP_SB_BYTES - des3->iv_len; 1438 else 1439 dm_offset = 0; 1440 ccp_get_dm_area(&ctx, dm_offset, des3->iv, 0, 1441 DES3_EDE_BLOCK_SIZE); 1442 } 1443 e_dst: 1444 if (!in_place) 1445 ccp_free_data(&dst, cmd_q); 1446 1447 e_src: 1448 ccp_free_data(&src, cmd_q); 1449 1450 e_ctx: 1451 if (des3->mode != CCP_DES3_MODE_ECB) 1452 ccp_dm_free(&ctx); 1453 1454 e_key: 1455 ccp_dm_free(&key); 1456 1457 return ret; 1458 } 1459 1460 static int ccp_run_sha_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) 1461 { 1462 struct ccp_sha_engine *sha = &cmd->u.sha; 1463 struct ccp_dm_workarea ctx; 1464 struct ccp_data src; 1465 struct ccp_op op; 1466 unsigned int ioffset, ooffset; 1467 unsigned int digest_size; 1468 int sb_count; 1469 const void *init; 1470 u64 block_size; 1471 int ctx_size; 1472 int ret; 1473 1474 switch (sha->type) { 1475 case CCP_SHA_TYPE_1: 1476 if (sha->ctx_len < SHA1_DIGEST_SIZE) 1477 return -EINVAL; 1478 block_size = SHA1_BLOCK_SIZE; 1479 break; 1480 case CCP_SHA_TYPE_224: 1481 if (sha->ctx_len < SHA224_DIGEST_SIZE) 1482 return -EINVAL; 1483 block_size = SHA224_BLOCK_SIZE; 1484 break; 1485 case CCP_SHA_TYPE_256: 1486 if (sha->ctx_len < SHA256_DIGEST_SIZE) 1487 return -EINVAL; 1488 block_size = SHA256_BLOCK_SIZE; 1489 break; 1490 case CCP_SHA_TYPE_384: 1491 if (cmd_q->ccp->vdata->version < CCP_VERSION(4, 0) 1492 || sha->ctx_len < SHA384_DIGEST_SIZE) 1493 return -EINVAL; 1494 block_size = SHA384_BLOCK_SIZE; 1495 break; 1496 case CCP_SHA_TYPE_512: 1497 if (cmd_q->ccp->vdata->version < CCP_VERSION(4, 0) 1498 || sha->ctx_len < SHA512_DIGEST_SIZE) 1499 return -EINVAL; 1500 block_size = SHA512_BLOCK_SIZE; 1501 break; 1502 default: 1503 return -EINVAL; 1504 } 1505 1506 if (!sha->ctx) 1507 return -EINVAL; 1508 1509 if (!sha->final && (sha->src_len & (block_size - 1))) 1510 return -EINVAL; 1511 1512 /* The version 3 device can't handle zero-length input */ 1513 if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0)) { 1514 1515 if (!sha->src_len) { 1516 unsigned int digest_len; 1517 const u8 *sha_zero; 1518 1519 /* Not final, just return */ 1520 if (!sha->final) 1521 return 0; 1522 1523 /* CCP can't do a zero length sha operation so the 1524 * caller must buffer the data. 1525 */ 1526 if (sha->msg_bits) 1527 return -EINVAL; 1528 1529 /* The CCP cannot perform zero-length sha operations 1530 * so the caller is required to buffer data for the 1531 * final operation. However, a sha operation for a 1532 * message with a total length of zero is valid so 1533 * known values are required to supply the result. 1534 */ 1535 switch (sha->type) { 1536 case CCP_SHA_TYPE_1: 1537 sha_zero = sha1_zero_message_hash; 1538 digest_len = SHA1_DIGEST_SIZE; 1539 break; 1540 case CCP_SHA_TYPE_224: 1541 sha_zero = sha224_zero_message_hash; 1542 digest_len = SHA224_DIGEST_SIZE; 1543 break; 1544 case CCP_SHA_TYPE_256: 1545 sha_zero = sha256_zero_message_hash; 1546 digest_len = SHA256_DIGEST_SIZE; 1547 break; 1548 default: 1549 return -EINVAL; 1550 } 1551 1552 scatterwalk_map_and_copy((void *)sha_zero, sha->ctx, 0, 1553 digest_len, 1); 1554 1555 return 0; 1556 } 1557 } 1558 1559 /* Set variables used throughout */ 1560 switch (sha->type) { 1561 case CCP_SHA_TYPE_1: 1562 digest_size = SHA1_DIGEST_SIZE; 1563 init = (void *) ccp_sha1_init; 1564 ctx_size = SHA1_DIGEST_SIZE; 1565 sb_count = 1; 1566 if (cmd_q->ccp->vdata->version != CCP_VERSION(3, 0)) 1567 ooffset = ioffset = CCP_SB_BYTES - SHA1_DIGEST_SIZE; 1568 else 1569 ooffset = ioffset = 0; 1570 break; 1571 case CCP_SHA_TYPE_224: 1572 digest_size = SHA224_DIGEST_SIZE; 1573 init = (void *) ccp_sha224_init; 1574 ctx_size = SHA256_DIGEST_SIZE; 1575 sb_count = 1; 1576 ioffset = 0; 1577 if (cmd_q->ccp->vdata->version != CCP_VERSION(3, 0)) 1578 ooffset = CCP_SB_BYTES - SHA224_DIGEST_SIZE; 1579 else 1580 ooffset = 0; 1581 break; 1582 case CCP_SHA_TYPE_256: 1583 digest_size = SHA256_DIGEST_SIZE; 1584 init = (void *) ccp_sha256_init; 1585 ctx_size = SHA256_DIGEST_SIZE; 1586 sb_count = 1; 1587 ooffset = ioffset = 0; 1588 break; 1589 case CCP_SHA_TYPE_384: 1590 digest_size = SHA384_DIGEST_SIZE; 1591 init = (void *) ccp_sha384_init; 1592 ctx_size = SHA512_DIGEST_SIZE; 1593 sb_count = 2; 1594 ioffset = 0; 1595 ooffset = 2 * CCP_SB_BYTES - SHA384_DIGEST_SIZE; 1596 break; 1597 case CCP_SHA_TYPE_512: 1598 digest_size = SHA512_DIGEST_SIZE; 1599 init = (void *) ccp_sha512_init; 1600 ctx_size = SHA512_DIGEST_SIZE; 1601 sb_count = 2; 1602 ooffset = ioffset = 0; 1603 break; 1604 default: 1605 ret = -EINVAL; 1606 goto e_data; 1607 } 1608 1609 /* For zero-length plaintext the src pointer is ignored; 1610 * otherwise both parts must be valid 1611 */ 1612 if (sha->src_len && !sha->src) 1613 return -EINVAL; 1614 1615 memset(&op, 0, sizeof(op)); 1616 op.cmd_q = cmd_q; 1617 op.jobid = CCP_NEW_JOBID(cmd_q->ccp); 1618 op.sb_ctx = cmd_q->sb_ctx; /* Pre-allocated */ 1619 op.u.sha.type = sha->type; 1620 op.u.sha.msg_bits = sha->msg_bits; 1621 1622 /* For SHA1/224/256 the context fits in a single (32-byte) SB entry; 1623 * SHA384/512 require 2 adjacent SB slots, with the right half in the 1624 * first slot, and the left half in the second. Each portion must then 1625 * be in little endian format: use the 256-bit byte swap option. 1626 */ 1627 ret = ccp_init_dm_workarea(&ctx, cmd_q, sb_count * CCP_SB_BYTES, 1628 DMA_BIDIRECTIONAL); 1629 if (ret) 1630 return ret; 1631 if (sha->first) { 1632 switch (sha->type) { 1633 case CCP_SHA_TYPE_1: 1634 case CCP_SHA_TYPE_224: 1635 case CCP_SHA_TYPE_256: 1636 memcpy(ctx.address + ioffset, init, ctx_size); 1637 break; 1638 case CCP_SHA_TYPE_384: 1639 case CCP_SHA_TYPE_512: 1640 memcpy(ctx.address + ctx_size / 2, init, 1641 ctx_size / 2); 1642 memcpy(ctx.address, init + ctx_size / 2, 1643 ctx_size / 2); 1644 break; 1645 default: 1646 ret = -EINVAL; 1647 goto e_ctx; 1648 } 1649 } else { 1650 /* Restore the context */ 1651 ret = ccp_set_dm_area(&ctx, 0, sha->ctx, 0, 1652 sb_count * CCP_SB_BYTES); 1653 if (ret) 1654 goto e_ctx; 1655 } 1656 1657 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, 1658 CCP_PASSTHRU_BYTESWAP_256BIT); 1659 if (ret) { 1660 cmd->engine_error = cmd_q->cmd_error; 1661 goto e_ctx; 1662 } 1663 1664 if (sha->src) { 1665 /* Send data to the CCP SHA engine; block_size is set above */ 1666 ret = ccp_init_data(&src, cmd_q, sha->src, sha->src_len, 1667 block_size, DMA_TO_DEVICE); 1668 if (ret) 1669 goto e_ctx; 1670 1671 while (src.sg_wa.bytes_left) { 1672 ccp_prepare_data(&src, NULL, &op, block_size, false); 1673 if (sha->final && !src.sg_wa.bytes_left) 1674 op.eom = 1; 1675 1676 ret = cmd_q->ccp->vdata->perform->sha(&op); 1677 if (ret) { 1678 cmd->engine_error = cmd_q->cmd_error; 1679 goto e_data; 1680 } 1681 1682 ccp_process_data(&src, NULL, &op); 1683 } 1684 } else { 1685 op.eom = 1; 1686 ret = cmd_q->ccp->vdata->perform->sha(&op); 1687 if (ret) { 1688 cmd->engine_error = cmd_q->cmd_error; 1689 goto e_data; 1690 } 1691 } 1692 1693 /* Retrieve the SHA context - convert from LE to BE using 1694 * 32-byte (256-bit) byteswapping to BE 1695 */ 1696 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, 1697 CCP_PASSTHRU_BYTESWAP_256BIT); 1698 if (ret) { 1699 cmd->engine_error = cmd_q->cmd_error; 1700 goto e_data; 1701 } 1702 1703 if (sha->final) { 1704 /* Finishing up, so get the digest */ 1705 switch (sha->type) { 1706 case CCP_SHA_TYPE_1: 1707 case CCP_SHA_TYPE_224: 1708 case CCP_SHA_TYPE_256: 1709 ccp_get_dm_area(&ctx, ooffset, 1710 sha->ctx, 0, 1711 digest_size); 1712 break; 1713 case CCP_SHA_TYPE_384: 1714 case CCP_SHA_TYPE_512: 1715 ccp_get_dm_area(&ctx, 0, 1716 sha->ctx, LSB_ITEM_SIZE - ooffset, 1717 LSB_ITEM_SIZE); 1718 ccp_get_dm_area(&ctx, LSB_ITEM_SIZE + ooffset, 1719 sha->ctx, 0, 1720 LSB_ITEM_SIZE - ooffset); 1721 break; 1722 default: 1723 ret = -EINVAL; 1724 goto e_ctx; 1725 } 1726 } else { 1727 /* Stash the context */ 1728 ccp_get_dm_area(&ctx, 0, sha->ctx, 0, 1729 sb_count * CCP_SB_BYTES); 1730 } 1731 1732 if (sha->final && sha->opad) { 1733 /* HMAC operation, recursively perform final SHA */ 1734 struct ccp_cmd hmac_cmd; 1735 struct scatterlist sg; 1736 u8 *hmac_buf; 1737 1738 if (sha->opad_len != block_size) { 1739 ret = -EINVAL; 1740 goto e_data; 1741 } 1742 1743 hmac_buf = kmalloc(block_size + digest_size, GFP_KERNEL); 1744 if (!hmac_buf) { 1745 ret = -ENOMEM; 1746 goto e_data; 1747 } 1748 sg_init_one(&sg, hmac_buf, block_size + digest_size); 1749 1750 scatterwalk_map_and_copy(hmac_buf, sha->opad, 0, block_size, 0); 1751 switch (sha->type) { 1752 case CCP_SHA_TYPE_1: 1753 case CCP_SHA_TYPE_224: 1754 case CCP_SHA_TYPE_256: 1755 memcpy(hmac_buf + block_size, 1756 ctx.address + ooffset, 1757 digest_size); 1758 break; 1759 case CCP_SHA_TYPE_384: 1760 case CCP_SHA_TYPE_512: 1761 memcpy(hmac_buf + block_size, 1762 ctx.address + LSB_ITEM_SIZE + ooffset, 1763 LSB_ITEM_SIZE); 1764 memcpy(hmac_buf + block_size + 1765 (LSB_ITEM_SIZE - ooffset), 1766 ctx.address, 1767 LSB_ITEM_SIZE); 1768 break; 1769 default: 1770 ret = -EINVAL; 1771 goto e_ctx; 1772 } 1773 1774 memset(&hmac_cmd, 0, sizeof(hmac_cmd)); 1775 hmac_cmd.engine = CCP_ENGINE_SHA; 1776 hmac_cmd.u.sha.type = sha->type; 1777 hmac_cmd.u.sha.ctx = sha->ctx; 1778 hmac_cmd.u.sha.ctx_len = sha->ctx_len; 1779 hmac_cmd.u.sha.src = &sg; 1780 hmac_cmd.u.sha.src_len = block_size + digest_size; 1781 hmac_cmd.u.sha.opad = NULL; 1782 hmac_cmd.u.sha.opad_len = 0; 1783 hmac_cmd.u.sha.first = 1; 1784 hmac_cmd.u.sha.final = 1; 1785 hmac_cmd.u.sha.msg_bits = (block_size + digest_size) << 3; 1786 1787 ret = ccp_run_sha_cmd(cmd_q, &hmac_cmd); 1788 if (ret) 1789 cmd->engine_error = hmac_cmd.engine_error; 1790 1791 kfree(hmac_buf); 1792 } 1793 1794 e_data: 1795 if (sha->src) 1796 ccp_free_data(&src, cmd_q); 1797 1798 e_ctx: 1799 ccp_dm_free(&ctx); 1800 1801 return ret; 1802 } 1803 1804 static int ccp_run_rsa_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) 1805 { 1806 struct ccp_rsa_engine *rsa = &cmd->u.rsa; 1807 struct ccp_dm_workarea exp, src, dst; 1808 struct ccp_op op; 1809 unsigned int sb_count, i_len, o_len; 1810 int ret; 1811 1812 /* Check against the maximum allowable size, in bits */ 1813 if (rsa->key_size > cmd_q->ccp->vdata->rsamax) 1814 return -EINVAL; 1815 1816 if (!rsa->exp || !rsa->mod || !rsa->src || !rsa->dst) 1817 return -EINVAL; 1818 1819 memset(&op, 0, sizeof(op)); 1820 op.cmd_q = cmd_q; 1821 op.jobid = CCP_NEW_JOBID(cmd_q->ccp); 1822 1823 /* The RSA modulus must precede the message being acted upon, so 1824 * it must be copied to a DMA area where the message and the 1825 * modulus can be concatenated. Therefore the input buffer 1826 * length required is twice the output buffer length (which 1827 * must be a multiple of 256-bits). Compute o_len, i_len in bytes. 1828 * Buffer sizes must be a multiple of 32 bytes; rounding up may be 1829 * required. 1830 */ 1831 o_len = 32 * ((rsa->key_size + 255) / 256); 1832 i_len = o_len * 2; 1833 1834 sb_count = 0; 1835 if (cmd_q->ccp->vdata->version < CCP_VERSION(5, 0)) { 1836 /* sb_count is the number of storage block slots required 1837 * for the modulus. 1838 */ 1839 sb_count = o_len / CCP_SB_BYTES; 1840 op.sb_key = cmd_q->ccp->vdata->perform->sballoc(cmd_q, 1841 sb_count); 1842 if (!op.sb_key) 1843 return -EIO; 1844 } else { 1845 /* A version 5 device allows a modulus size that will not fit 1846 * in the LSB, so the command will transfer it from memory. 1847 * Set the sb key to the default, even though it's not used. 1848 */ 1849 op.sb_key = cmd_q->sb_key; 1850 } 1851 1852 /* The RSA exponent must be in little endian format. Reverse its 1853 * byte order. 1854 */ 1855 ret = ccp_init_dm_workarea(&exp, cmd_q, o_len, DMA_TO_DEVICE); 1856 if (ret) 1857 goto e_sb; 1858 1859 ret = ccp_reverse_set_dm_area(&exp, 0, rsa->exp, 0, rsa->exp_len); 1860 if (ret) 1861 goto e_exp; 1862 1863 if (cmd_q->ccp->vdata->version < CCP_VERSION(5, 0)) { 1864 /* Copy the exponent to the local storage block, using 1865 * as many 32-byte blocks as were allocated above. It's 1866 * already little endian, so no further change is required. 1867 */ 1868 ret = ccp_copy_to_sb(cmd_q, &exp, op.jobid, op.sb_key, 1869 CCP_PASSTHRU_BYTESWAP_NOOP); 1870 if (ret) { 1871 cmd->engine_error = cmd_q->cmd_error; 1872 goto e_exp; 1873 } 1874 } else { 1875 /* The exponent can be retrieved from memory via DMA. */ 1876 op.exp.u.dma.address = exp.dma.address; 1877 op.exp.u.dma.offset = 0; 1878 } 1879 1880 /* Concatenate the modulus and the message. Both the modulus and 1881 * the operands must be in little endian format. Since the input 1882 * is in big endian format it must be converted. 1883 */ 1884 ret = ccp_init_dm_workarea(&src, cmd_q, i_len, DMA_TO_DEVICE); 1885 if (ret) 1886 goto e_exp; 1887 1888 ret = ccp_reverse_set_dm_area(&src, 0, rsa->mod, 0, rsa->mod_len); 1889 if (ret) 1890 goto e_src; 1891 ret = ccp_reverse_set_dm_area(&src, o_len, rsa->src, 0, rsa->src_len); 1892 if (ret) 1893 goto e_src; 1894 1895 /* Prepare the output area for the operation */ 1896 ret = ccp_init_dm_workarea(&dst, cmd_q, o_len, DMA_FROM_DEVICE); 1897 if (ret) 1898 goto e_src; 1899 1900 op.soc = 1; 1901 op.src.u.dma.address = src.dma.address; 1902 op.src.u.dma.offset = 0; 1903 op.src.u.dma.length = i_len; 1904 op.dst.u.dma.address = dst.dma.address; 1905 op.dst.u.dma.offset = 0; 1906 op.dst.u.dma.length = o_len; 1907 1908 op.u.rsa.mod_size = rsa->key_size; 1909 op.u.rsa.input_len = i_len; 1910 1911 ret = cmd_q->ccp->vdata->perform->rsa(&op); 1912 if (ret) { 1913 cmd->engine_error = cmd_q->cmd_error; 1914 goto e_dst; 1915 } 1916 1917 ccp_reverse_get_dm_area(&dst, 0, rsa->dst, 0, rsa->mod_len); 1918 1919 e_dst: 1920 ccp_dm_free(&dst); 1921 1922 e_src: 1923 ccp_dm_free(&src); 1924 1925 e_exp: 1926 ccp_dm_free(&exp); 1927 1928 e_sb: 1929 if (sb_count) 1930 cmd_q->ccp->vdata->perform->sbfree(cmd_q, op.sb_key, sb_count); 1931 1932 return ret; 1933 } 1934 1935 static int ccp_run_passthru_cmd(struct ccp_cmd_queue *cmd_q, 1936 struct ccp_cmd *cmd) 1937 { 1938 struct ccp_passthru_engine *pt = &cmd->u.passthru; 1939 struct ccp_dm_workarea mask; 1940 struct ccp_data src, dst; 1941 struct ccp_op op; 1942 bool in_place = false; 1943 unsigned int i; 1944 int ret = 0; 1945 1946 if (!pt->final && (pt->src_len & (CCP_PASSTHRU_BLOCKSIZE - 1))) 1947 return -EINVAL; 1948 1949 if (!pt->src || !pt->dst) 1950 return -EINVAL; 1951 1952 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) { 1953 if (pt->mask_len != CCP_PASSTHRU_MASKSIZE) 1954 return -EINVAL; 1955 if (!pt->mask) 1956 return -EINVAL; 1957 } 1958 1959 BUILD_BUG_ON(CCP_PASSTHRU_SB_COUNT != 1); 1960 1961 memset(&op, 0, sizeof(op)); 1962 op.cmd_q = cmd_q; 1963 op.jobid = CCP_NEW_JOBID(cmd_q->ccp); 1964 1965 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) { 1966 /* Load the mask */ 1967 op.sb_key = cmd_q->sb_key; 1968 1969 ret = ccp_init_dm_workarea(&mask, cmd_q, 1970 CCP_PASSTHRU_SB_COUNT * 1971 CCP_SB_BYTES, 1972 DMA_TO_DEVICE); 1973 if (ret) 1974 return ret; 1975 1976 ret = ccp_set_dm_area(&mask, 0, pt->mask, 0, pt->mask_len); 1977 if (ret) 1978 goto e_mask; 1979 ret = ccp_copy_to_sb(cmd_q, &mask, op.jobid, op.sb_key, 1980 CCP_PASSTHRU_BYTESWAP_NOOP); 1981 if (ret) { 1982 cmd->engine_error = cmd_q->cmd_error; 1983 goto e_mask; 1984 } 1985 } 1986 1987 /* Prepare the input and output data workareas. For in-place 1988 * operations we need to set the dma direction to BIDIRECTIONAL 1989 * and copy the src workarea to the dst workarea. 1990 */ 1991 if (sg_virt(pt->src) == sg_virt(pt->dst)) 1992 in_place = true; 1993 1994 ret = ccp_init_data(&src, cmd_q, pt->src, pt->src_len, 1995 CCP_PASSTHRU_MASKSIZE, 1996 in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE); 1997 if (ret) 1998 goto e_mask; 1999 2000 if (in_place) { 2001 dst = src; 2002 } else { 2003 ret = ccp_init_data(&dst, cmd_q, pt->dst, pt->src_len, 2004 CCP_PASSTHRU_MASKSIZE, DMA_FROM_DEVICE); 2005 if (ret) 2006 goto e_src; 2007 } 2008 2009 /* Send data to the CCP Passthru engine 2010 * Because the CCP engine works on a single source and destination 2011 * dma address at a time, each entry in the source scatterlist 2012 * (after the dma_map_sg call) must be less than or equal to the 2013 * (remaining) length in the destination scatterlist entry and the 2014 * length must be a multiple of CCP_PASSTHRU_BLOCKSIZE 2015 */ 2016 dst.sg_wa.sg_used = 0; 2017 for (i = 1; i <= src.sg_wa.dma_count; i++) { 2018 if (!dst.sg_wa.sg || 2019 (dst.sg_wa.sg->length < src.sg_wa.sg->length)) { 2020 ret = -EINVAL; 2021 goto e_dst; 2022 } 2023 2024 if (i == src.sg_wa.dma_count) { 2025 op.eom = 1; 2026 op.soc = 1; 2027 } 2028 2029 op.src.type = CCP_MEMTYPE_SYSTEM; 2030 op.src.u.dma.address = sg_dma_address(src.sg_wa.sg); 2031 op.src.u.dma.offset = 0; 2032 op.src.u.dma.length = sg_dma_len(src.sg_wa.sg); 2033 2034 op.dst.type = CCP_MEMTYPE_SYSTEM; 2035 op.dst.u.dma.address = sg_dma_address(dst.sg_wa.sg); 2036 op.dst.u.dma.offset = dst.sg_wa.sg_used; 2037 op.dst.u.dma.length = op.src.u.dma.length; 2038 2039 ret = cmd_q->ccp->vdata->perform->passthru(&op); 2040 if (ret) { 2041 cmd->engine_error = cmd_q->cmd_error; 2042 goto e_dst; 2043 } 2044 2045 dst.sg_wa.sg_used += src.sg_wa.sg->length; 2046 if (dst.sg_wa.sg_used == dst.sg_wa.sg->length) { 2047 dst.sg_wa.sg = sg_next(dst.sg_wa.sg); 2048 dst.sg_wa.sg_used = 0; 2049 } 2050 src.sg_wa.sg = sg_next(src.sg_wa.sg); 2051 } 2052 2053 e_dst: 2054 if (!in_place) 2055 ccp_free_data(&dst, cmd_q); 2056 2057 e_src: 2058 ccp_free_data(&src, cmd_q); 2059 2060 e_mask: 2061 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) 2062 ccp_dm_free(&mask); 2063 2064 return ret; 2065 } 2066 2067 static int ccp_run_passthru_nomap_cmd(struct ccp_cmd_queue *cmd_q, 2068 struct ccp_cmd *cmd) 2069 { 2070 struct ccp_passthru_nomap_engine *pt = &cmd->u.passthru_nomap; 2071 struct ccp_dm_workarea mask; 2072 struct ccp_op op; 2073 int ret; 2074 2075 if (!pt->final && (pt->src_len & (CCP_PASSTHRU_BLOCKSIZE - 1))) 2076 return -EINVAL; 2077 2078 if (!pt->src_dma || !pt->dst_dma) 2079 return -EINVAL; 2080 2081 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) { 2082 if (pt->mask_len != CCP_PASSTHRU_MASKSIZE) 2083 return -EINVAL; 2084 if (!pt->mask) 2085 return -EINVAL; 2086 } 2087 2088 BUILD_BUG_ON(CCP_PASSTHRU_SB_COUNT != 1); 2089 2090 memset(&op, 0, sizeof(op)); 2091 op.cmd_q = cmd_q; 2092 op.jobid = CCP_NEW_JOBID(cmd_q->ccp); 2093 2094 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) { 2095 /* Load the mask */ 2096 op.sb_key = cmd_q->sb_key; 2097 2098 mask.length = pt->mask_len; 2099 mask.dma.address = pt->mask; 2100 mask.dma.length = pt->mask_len; 2101 2102 ret = ccp_copy_to_sb(cmd_q, &mask, op.jobid, op.sb_key, 2103 CCP_PASSTHRU_BYTESWAP_NOOP); 2104 if (ret) { 2105 cmd->engine_error = cmd_q->cmd_error; 2106 return ret; 2107 } 2108 } 2109 2110 /* Send data to the CCP Passthru engine */ 2111 op.eom = 1; 2112 op.soc = 1; 2113 2114 op.src.type = CCP_MEMTYPE_SYSTEM; 2115 op.src.u.dma.address = pt->src_dma; 2116 op.src.u.dma.offset = 0; 2117 op.src.u.dma.length = pt->src_len; 2118 2119 op.dst.type = CCP_MEMTYPE_SYSTEM; 2120 op.dst.u.dma.address = pt->dst_dma; 2121 op.dst.u.dma.offset = 0; 2122 op.dst.u.dma.length = pt->src_len; 2123 2124 ret = cmd_q->ccp->vdata->perform->passthru(&op); 2125 if (ret) 2126 cmd->engine_error = cmd_q->cmd_error; 2127 2128 return ret; 2129 } 2130 2131 static int ccp_run_ecc_mm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) 2132 { 2133 struct ccp_ecc_engine *ecc = &cmd->u.ecc; 2134 struct ccp_dm_workarea src, dst; 2135 struct ccp_op op; 2136 int ret; 2137 u8 *save; 2138 2139 if (!ecc->u.mm.operand_1 || 2140 (ecc->u.mm.operand_1_len > CCP_ECC_MODULUS_BYTES)) 2141 return -EINVAL; 2142 2143 if (ecc->function != CCP_ECC_FUNCTION_MINV_384BIT) 2144 if (!ecc->u.mm.operand_2 || 2145 (ecc->u.mm.operand_2_len > CCP_ECC_MODULUS_BYTES)) 2146 return -EINVAL; 2147 2148 if (!ecc->u.mm.result || 2149 (ecc->u.mm.result_len < CCP_ECC_MODULUS_BYTES)) 2150 return -EINVAL; 2151 2152 memset(&op, 0, sizeof(op)); 2153 op.cmd_q = cmd_q; 2154 op.jobid = CCP_NEW_JOBID(cmd_q->ccp); 2155 2156 /* Concatenate the modulus and the operands. Both the modulus and 2157 * the operands must be in little endian format. Since the input 2158 * is in big endian format it must be converted and placed in a 2159 * fixed length buffer. 2160 */ 2161 ret = ccp_init_dm_workarea(&src, cmd_q, CCP_ECC_SRC_BUF_SIZE, 2162 DMA_TO_DEVICE); 2163 if (ret) 2164 return ret; 2165 2166 /* Save the workarea address since it is updated in order to perform 2167 * the concatenation 2168 */ 2169 save = src.address; 2170 2171 /* Copy the ECC modulus */ 2172 ret = ccp_reverse_set_dm_area(&src, 0, ecc->mod, 0, ecc->mod_len); 2173 if (ret) 2174 goto e_src; 2175 src.address += CCP_ECC_OPERAND_SIZE; 2176 2177 /* Copy the first operand */ 2178 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.mm.operand_1, 0, 2179 ecc->u.mm.operand_1_len); 2180 if (ret) 2181 goto e_src; 2182 src.address += CCP_ECC_OPERAND_SIZE; 2183 2184 if (ecc->function != CCP_ECC_FUNCTION_MINV_384BIT) { 2185 /* Copy the second operand */ 2186 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.mm.operand_2, 0, 2187 ecc->u.mm.operand_2_len); 2188 if (ret) 2189 goto e_src; 2190 src.address += CCP_ECC_OPERAND_SIZE; 2191 } 2192 2193 /* Restore the workarea address */ 2194 src.address = save; 2195 2196 /* Prepare the output area for the operation */ 2197 ret = ccp_init_dm_workarea(&dst, cmd_q, CCP_ECC_DST_BUF_SIZE, 2198 DMA_FROM_DEVICE); 2199 if (ret) 2200 goto e_src; 2201 2202 op.soc = 1; 2203 op.src.u.dma.address = src.dma.address; 2204 op.src.u.dma.offset = 0; 2205 op.src.u.dma.length = src.length; 2206 op.dst.u.dma.address = dst.dma.address; 2207 op.dst.u.dma.offset = 0; 2208 op.dst.u.dma.length = dst.length; 2209 2210 op.u.ecc.function = cmd->u.ecc.function; 2211 2212 ret = cmd_q->ccp->vdata->perform->ecc(&op); 2213 if (ret) { 2214 cmd->engine_error = cmd_q->cmd_error; 2215 goto e_dst; 2216 } 2217 2218 ecc->ecc_result = le16_to_cpup( 2219 (const __le16 *)(dst.address + CCP_ECC_RESULT_OFFSET)); 2220 if (!(ecc->ecc_result & CCP_ECC_RESULT_SUCCESS)) { 2221 ret = -EIO; 2222 goto e_dst; 2223 } 2224 2225 /* Save the ECC result */ 2226 ccp_reverse_get_dm_area(&dst, 0, ecc->u.mm.result, 0, 2227 CCP_ECC_MODULUS_BYTES); 2228 2229 e_dst: 2230 ccp_dm_free(&dst); 2231 2232 e_src: 2233 ccp_dm_free(&src); 2234 2235 return ret; 2236 } 2237 2238 static int ccp_run_ecc_pm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) 2239 { 2240 struct ccp_ecc_engine *ecc = &cmd->u.ecc; 2241 struct ccp_dm_workarea src, dst; 2242 struct ccp_op op; 2243 int ret; 2244 u8 *save; 2245 2246 if (!ecc->u.pm.point_1.x || 2247 (ecc->u.pm.point_1.x_len > CCP_ECC_MODULUS_BYTES) || 2248 !ecc->u.pm.point_1.y || 2249 (ecc->u.pm.point_1.y_len > CCP_ECC_MODULUS_BYTES)) 2250 return -EINVAL; 2251 2252 if (ecc->function == CCP_ECC_FUNCTION_PADD_384BIT) { 2253 if (!ecc->u.pm.point_2.x || 2254 (ecc->u.pm.point_2.x_len > CCP_ECC_MODULUS_BYTES) || 2255 !ecc->u.pm.point_2.y || 2256 (ecc->u.pm.point_2.y_len > CCP_ECC_MODULUS_BYTES)) 2257 return -EINVAL; 2258 } else { 2259 if (!ecc->u.pm.domain_a || 2260 (ecc->u.pm.domain_a_len > CCP_ECC_MODULUS_BYTES)) 2261 return -EINVAL; 2262 2263 if (ecc->function == CCP_ECC_FUNCTION_PMUL_384BIT) 2264 if (!ecc->u.pm.scalar || 2265 (ecc->u.pm.scalar_len > CCP_ECC_MODULUS_BYTES)) 2266 return -EINVAL; 2267 } 2268 2269 if (!ecc->u.pm.result.x || 2270 (ecc->u.pm.result.x_len < CCP_ECC_MODULUS_BYTES) || 2271 !ecc->u.pm.result.y || 2272 (ecc->u.pm.result.y_len < CCP_ECC_MODULUS_BYTES)) 2273 return -EINVAL; 2274 2275 memset(&op, 0, sizeof(op)); 2276 op.cmd_q = cmd_q; 2277 op.jobid = CCP_NEW_JOBID(cmd_q->ccp); 2278 2279 /* Concatenate the modulus and the operands. Both the modulus and 2280 * the operands must be in little endian format. Since the input 2281 * is in big endian format it must be converted and placed in a 2282 * fixed length buffer. 2283 */ 2284 ret = ccp_init_dm_workarea(&src, cmd_q, CCP_ECC_SRC_BUF_SIZE, 2285 DMA_TO_DEVICE); 2286 if (ret) 2287 return ret; 2288 2289 /* Save the workarea address since it is updated in order to perform 2290 * the concatenation 2291 */ 2292 save = src.address; 2293 2294 /* Copy the ECC modulus */ 2295 ret = ccp_reverse_set_dm_area(&src, 0, ecc->mod, 0, ecc->mod_len); 2296 if (ret) 2297 goto e_src; 2298 src.address += CCP_ECC_OPERAND_SIZE; 2299 2300 /* Copy the first point X and Y coordinate */ 2301 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_1.x, 0, 2302 ecc->u.pm.point_1.x_len); 2303 if (ret) 2304 goto e_src; 2305 src.address += CCP_ECC_OPERAND_SIZE; 2306 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_1.y, 0, 2307 ecc->u.pm.point_1.y_len); 2308 if (ret) 2309 goto e_src; 2310 src.address += CCP_ECC_OPERAND_SIZE; 2311 2312 /* Set the first point Z coordinate to 1 */ 2313 *src.address = 0x01; 2314 src.address += CCP_ECC_OPERAND_SIZE; 2315 2316 if (ecc->function == CCP_ECC_FUNCTION_PADD_384BIT) { 2317 /* Copy the second point X and Y coordinate */ 2318 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_2.x, 0, 2319 ecc->u.pm.point_2.x_len); 2320 if (ret) 2321 goto e_src; 2322 src.address += CCP_ECC_OPERAND_SIZE; 2323 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_2.y, 0, 2324 ecc->u.pm.point_2.y_len); 2325 if (ret) 2326 goto e_src; 2327 src.address += CCP_ECC_OPERAND_SIZE; 2328 2329 /* Set the second point Z coordinate to 1 */ 2330 *src.address = 0x01; 2331 src.address += CCP_ECC_OPERAND_SIZE; 2332 } else { 2333 /* Copy the Domain "a" parameter */ 2334 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.domain_a, 0, 2335 ecc->u.pm.domain_a_len); 2336 if (ret) 2337 goto e_src; 2338 src.address += CCP_ECC_OPERAND_SIZE; 2339 2340 if (ecc->function == CCP_ECC_FUNCTION_PMUL_384BIT) { 2341 /* Copy the scalar value */ 2342 ret = ccp_reverse_set_dm_area(&src, 0, 2343 ecc->u.pm.scalar, 0, 2344 ecc->u.pm.scalar_len); 2345 if (ret) 2346 goto e_src; 2347 src.address += CCP_ECC_OPERAND_SIZE; 2348 } 2349 } 2350 2351 /* Restore the workarea address */ 2352 src.address = save; 2353 2354 /* Prepare the output area for the operation */ 2355 ret = ccp_init_dm_workarea(&dst, cmd_q, CCP_ECC_DST_BUF_SIZE, 2356 DMA_FROM_DEVICE); 2357 if (ret) 2358 goto e_src; 2359 2360 op.soc = 1; 2361 op.src.u.dma.address = src.dma.address; 2362 op.src.u.dma.offset = 0; 2363 op.src.u.dma.length = src.length; 2364 op.dst.u.dma.address = dst.dma.address; 2365 op.dst.u.dma.offset = 0; 2366 op.dst.u.dma.length = dst.length; 2367 2368 op.u.ecc.function = cmd->u.ecc.function; 2369 2370 ret = cmd_q->ccp->vdata->perform->ecc(&op); 2371 if (ret) { 2372 cmd->engine_error = cmd_q->cmd_error; 2373 goto e_dst; 2374 } 2375 2376 ecc->ecc_result = le16_to_cpup( 2377 (const __le16 *)(dst.address + CCP_ECC_RESULT_OFFSET)); 2378 if (!(ecc->ecc_result & CCP_ECC_RESULT_SUCCESS)) { 2379 ret = -EIO; 2380 goto e_dst; 2381 } 2382 2383 /* Save the workarea address since it is updated as we walk through 2384 * to copy the point math result 2385 */ 2386 save = dst.address; 2387 2388 /* Save the ECC result X and Y coordinates */ 2389 ccp_reverse_get_dm_area(&dst, 0, ecc->u.pm.result.x, 0, 2390 CCP_ECC_MODULUS_BYTES); 2391 dst.address += CCP_ECC_OUTPUT_SIZE; 2392 ccp_reverse_get_dm_area(&dst, 0, ecc->u.pm.result.y, 0, 2393 CCP_ECC_MODULUS_BYTES); 2394 dst.address += CCP_ECC_OUTPUT_SIZE; 2395 2396 /* Restore the workarea address */ 2397 dst.address = save; 2398 2399 e_dst: 2400 ccp_dm_free(&dst); 2401 2402 e_src: 2403 ccp_dm_free(&src); 2404 2405 return ret; 2406 } 2407 2408 static int ccp_run_ecc_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) 2409 { 2410 struct ccp_ecc_engine *ecc = &cmd->u.ecc; 2411 2412 ecc->ecc_result = 0; 2413 2414 if (!ecc->mod || 2415 (ecc->mod_len > CCP_ECC_MODULUS_BYTES)) 2416 return -EINVAL; 2417 2418 switch (ecc->function) { 2419 case CCP_ECC_FUNCTION_MMUL_384BIT: 2420 case CCP_ECC_FUNCTION_MADD_384BIT: 2421 case CCP_ECC_FUNCTION_MINV_384BIT: 2422 return ccp_run_ecc_mm_cmd(cmd_q, cmd); 2423 2424 case CCP_ECC_FUNCTION_PADD_384BIT: 2425 case CCP_ECC_FUNCTION_PMUL_384BIT: 2426 case CCP_ECC_FUNCTION_PDBL_384BIT: 2427 return ccp_run_ecc_pm_cmd(cmd_q, cmd); 2428 2429 default: 2430 return -EINVAL; 2431 } 2432 } 2433 2434 int ccp_run_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) 2435 { 2436 int ret; 2437 2438 cmd->engine_error = 0; 2439 cmd_q->cmd_error = 0; 2440 cmd_q->int_rcvd = 0; 2441 cmd_q->free_slots = cmd_q->ccp->vdata->perform->get_free_slots(cmd_q); 2442 2443 switch (cmd->engine) { 2444 case CCP_ENGINE_AES: 2445 ret = ccp_run_aes_cmd(cmd_q, cmd); 2446 break; 2447 case CCP_ENGINE_XTS_AES_128: 2448 ret = ccp_run_xts_aes_cmd(cmd_q, cmd); 2449 break; 2450 case CCP_ENGINE_DES3: 2451 ret = ccp_run_des3_cmd(cmd_q, cmd); 2452 break; 2453 case CCP_ENGINE_SHA: 2454 ret = ccp_run_sha_cmd(cmd_q, cmd); 2455 break; 2456 case CCP_ENGINE_RSA: 2457 ret = ccp_run_rsa_cmd(cmd_q, cmd); 2458 break; 2459 case CCP_ENGINE_PASSTHRU: 2460 if (cmd->flags & CCP_CMD_PASSTHRU_NO_DMA_MAP) 2461 ret = ccp_run_passthru_nomap_cmd(cmd_q, cmd); 2462 else 2463 ret = ccp_run_passthru_cmd(cmd_q, cmd); 2464 break; 2465 case CCP_ENGINE_ECC: 2466 ret = ccp_run_ecc_cmd(cmd_q, cmd); 2467 break; 2468 default: 2469 ret = -EINVAL; 2470 } 2471 2472 return ret; 2473 } 2474