1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * AMD Cryptographic Coprocessor (CCP) driver 4 * 5 * Copyright (C) 2013-2019 Advanced Micro Devices, Inc. 6 * 7 * Author: Tom Lendacky <thomas.lendacky@amd.com> 8 * Author: Gary R Hook <gary.hook@amd.com> 9 */ 10 11 #include <linux/module.h> 12 #include <linux/kernel.h> 13 #include <linux/pci.h> 14 #include <linux/interrupt.h> 15 #include <crypto/scatterwalk.h> 16 #include <crypto/des.h> 17 #include <linux/ccp.h> 18 19 #include "ccp-dev.h" 20 21 /* SHA initial context values */ 22 static const __be32 ccp_sha1_init[SHA1_DIGEST_SIZE / sizeof(__be32)] = { 23 cpu_to_be32(SHA1_H0), cpu_to_be32(SHA1_H1), 24 cpu_to_be32(SHA1_H2), cpu_to_be32(SHA1_H3), 25 cpu_to_be32(SHA1_H4), 26 }; 27 28 static const __be32 ccp_sha224_init[SHA256_DIGEST_SIZE / sizeof(__be32)] = { 29 cpu_to_be32(SHA224_H0), cpu_to_be32(SHA224_H1), 30 cpu_to_be32(SHA224_H2), cpu_to_be32(SHA224_H3), 31 cpu_to_be32(SHA224_H4), cpu_to_be32(SHA224_H5), 32 cpu_to_be32(SHA224_H6), cpu_to_be32(SHA224_H7), 33 }; 34 35 static const __be32 ccp_sha256_init[SHA256_DIGEST_SIZE / sizeof(__be32)] = { 36 cpu_to_be32(SHA256_H0), cpu_to_be32(SHA256_H1), 37 cpu_to_be32(SHA256_H2), cpu_to_be32(SHA256_H3), 38 cpu_to_be32(SHA256_H4), cpu_to_be32(SHA256_H5), 39 cpu_to_be32(SHA256_H6), cpu_to_be32(SHA256_H7), 40 }; 41 42 static const __be64 ccp_sha384_init[SHA512_DIGEST_SIZE / sizeof(__be64)] = { 43 cpu_to_be64(SHA384_H0), cpu_to_be64(SHA384_H1), 44 cpu_to_be64(SHA384_H2), cpu_to_be64(SHA384_H3), 45 cpu_to_be64(SHA384_H4), cpu_to_be64(SHA384_H5), 46 cpu_to_be64(SHA384_H6), cpu_to_be64(SHA384_H7), 47 }; 48 49 static const __be64 ccp_sha512_init[SHA512_DIGEST_SIZE / sizeof(__be64)] = { 50 cpu_to_be64(SHA512_H0), cpu_to_be64(SHA512_H1), 51 cpu_to_be64(SHA512_H2), cpu_to_be64(SHA512_H3), 52 cpu_to_be64(SHA512_H4), cpu_to_be64(SHA512_H5), 53 cpu_to_be64(SHA512_H6), cpu_to_be64(SHA512_H7), 54 }; 55 56 #define CCP_NEW_JOBID(ccp) ((ccp->vdata->version == CCP_VERSION(3, 0)) ? \ 57 ccp_gen_jobid(ccp) : 0) 58 59 static u32 ccp_gen_jobid(struct ccp_device *ccp) 60 { 61 return atomic_inc_return(&ccp->current_id) & CCP_JOBID_MASK; 62 } 63 64 static void ccp_sg_free(struct ccp_sg_workarea *wa) 65 { 66 if (wa->dma_count) 67 dma_unmap_sg(wa->dma_dev, wa->dma_sg, wa->nents, wa->dma_dir); 68 69 wa->dma_count = 0; 70 } 71 72 static int ccp_init_sg_workarea(struct ccp_sg_workarea *wa, struct device *dev, 73 struct scatterlist *sg, u64 len, 74 enum dma_data_direction dma_dir) 75 { 76 memset(wa, 0, sizeof(*wa)); 77 78 wa->sg = sg; 79 if (!sg) 80 return 0; 81 82 wa->nents = sg_nents_for_len(sg, len); 83 if (wa->nents < 0) 84 return wa->nents; 85 86 wa->bytes_left = len; 87 wa->sg_used = 0; 88 89 if (len == 0) 90 return 0; 91 92 if (dma_dir == DMA_NONE) 93 return 0; 94 95 wa->dma_sg = sg; 96 wa->dma_dev = dev; 97 wa->dma_dir = dma_dir; 98 wa->dma_count = dma_map_sg(dev, sg, wa->nents, dma_dir); 99 if (!wa->dma_count) 100 return -ENOMEM; 101 102 return 0; 103 } 104 105 static void ccp_update_sg_workarea(struct ccp_sg_workarea *wa, unsigned int len) 106 { 107 unsigned int nbytes = min_t(u64, len, wa->bytes_left); 108 109 if (!wa->sg) 110 return; 111 112 wa->sg_used += nbytes; 113 wa->bytes_left -= nbytes; 114 if (wa->sg_used == wa->sg->length) { 115 wa->sg = sg_next(wa->sg); 116 wa->sg_used = 0; 117 } 118 } 119 120 static void ccp_dm_free(struct ccp_dm_workarea *wa) 121 { 122 if (wa->length <= CCP_DMAPOOL_MAX_SIZE) { 123 if (wa->address) 124 dma_pool_free(wa->dma_pool, wa->address, 125 wa->dma.address); 126 } else { 127 if (wa->dma.address) 128 dma_unmap_single(wa->dev, wa->dma.address, wa->length, 129 wa->dma.dir); 130 kfree(wa->address); 131 } 132 133 wa->address = NULL; 134 wa->dma.address = 0; 135 } 136 137 static int ccp_init_dm_workarea(struct ccp_dm_workarea *wa, 138 struct ccp_cmd_queue *cmd_q, 139 unsigned int len, 140 enum dma_data_direction dir) 141 { 142 memset(wa, 0, sizeof(*wa)); 143 144 if (!len) 145 return 0; 146 147 wa->dev = cmd_q->ccp->dev; 148 wa->length = len; 149 150 if (len <= CCP_DMAPOOL_MAX_SIZE) { 151 wa->dma_pool = cmd_q->dma_pool; 152 153 wa->address = dma_pool_alloc(wa->dma_pool, GFP_KERNEL, 154 &wa->dma.address); 155 if (!wa->address) 156 return -ENOMEM; 157 158 wa->dma.length = CCP_DMAPOOL_MAX_SIZE; 159 160 memset(wa->address, 0, CCP_DMAPOOL_MAX_SIZE); 161 } else { 162 wa->address = kzalloc(len, GFP_KERNEL); 163 if (!wa->address) 164 return -ENOMEM; 165 166 wa->dma.address = dma_map_single(wa->dev, wa->address, len, 167 dir); 168 if (dma_mapping_error(wa->dev, wa->dma.address)) 169 return -ENOMEM; 170 171 wa->dma.length = len; 172 } 173 wa->dma.dir = dir; 174 175 return 0; 176 } 177 178 static int ccp_set_dm_area(struct ccp_dm_workarea *wa, unsigned int wa_offset, 179 struct scatterlist *sg, unsigned int sg_offset, 180 unsigned int len) 181 { 182 WARN_ON(!wa->address); 183 184 if (len > (wa->length - wa_offset)) 185 return -EINVAL; 186 187 scatterwalk_map_and_copy(wa->address + wa_offset, sg, sg_offset, len, 188 0); 189 return 0; 190 } 191 192 static void ccp_get_dm_area(struct ccp_dm_workarea *wa, unsigned int wa_offset, 193 struct scatterlist *sg, unsigned int sg_offset, 194 unsigned int len) 195 { 196 WARN_ON(!wa->address); 197 198 scatterwalk_map_and_copy(wa->address + wa_offset, sg, sg_offset, len, 199 1); 200 } 201 202 static int ccp_reverse_set_dm_area(struct ccp_dm_workarea *wa, 203 unsigned int wa_offset, 204 struct scatterlist *sg, 205 unsigned int sg_offset, 206 unsigned int len) 207 { 208 u8 *p, *q; 209 int rc; 210 211 rc = ccp_set_dm_area(wa, wa_offset, sg, sg_offset, len); 212 if (rc) 213 return rc; 214 215 p = wa->address + wa_offset; 216 q = p + len - 1; 217 while (p < q) { 218 *p = *p ^ *q; 219 *q = *p ^ *q; 220 *p = *p ^ *q; 221 p++; 222 q--; 223 } 224 return 0; 225 } 226 227 static void ccp_reverse_get_dm_area(struct ccp_dm_workarea *wa, 228 unsigned int wa_offset, 229 struct scatterlist *sg, 230 unsigned int sg_offset, 231 unsigned int len) 232 { 233 u8 *p, *q; 234 235 p = wa->address + wa_offset; 236 q = p + len - 1; 237 while (p < q) { 238 *p = *p ^ *q; 239 *q = *p ^ *q; 240 *p = *p ^ *q; 241 p++; 242 q--; 243 } 244 245 ccp_get_dm_area(wa, wa_offset, sg, sg_offset, len); 246 } 247 248 static void ccp_free_data(struct ccp_data *data, struct ccp_cmd_queue *cmd_q) 249 { 250 ccp_dm_free(&data->dm_wa); 251 ccp_sg_free(&data->sg_wa); 252 } 253 254 static int ccp_init_data(struct ccp_data *data, struct ccp_cmd_queue *cmd_q, 255 struct scatterlist *sg, u64 sg_len, 256 unsigned int dm_len, 257 enum dma_data_direction dir) 258 { 259 int ret; 260 261 memset(data, 0, sizeof(*data)); 262 263 ret = ccp_init_sg_workarea(&data->sg_wa, cmd_q->ccp->dev, sg, sg_len, 264 dir); 265 if (ret) 266 goto e_err; 267 268 ret = ccp_init_dm_workarea(&data->dm_wa, cmd_q, dm_len, dir); 269 if (ret) 270 goto e_err; 271 272 return 0; 273 274 e_err: 275 ccp_free_data(data, cmd_q); 276 277 return ret; 278 } 279 280 static unsigned int ccp_queue_buf(struct ccp_data *data, unsigned int from) 281 { 282 struct ccp_sg_workarea *sg_wa = &data->sg_wa; 283 struct ccp_dm_workarea *dm_wa = &data->dm_wa; 284 unsigned int buf_count, nbytes; 285 286 /* Clear the buffer if setting it */ 287 if (!from) 288 memset(dm_wa->address, 0, dm_wa->length); 289 290 if (!sg_wa->sg) 291 return 0; 292 293 /* Perform the copy operation 294 * nbytes will always be <= UINT_MAX because dm_wa->length is 295 * an unsigned int 296 */ 297 nbytes = min_t(u64, sg_wa->bytes_left, dm_wa->length); 298 scatterwalk_map_and_copy(dm_wa->address, sg_wa->sg, sg_wa->sg_used, 299 nbytes, from); 300 301 /* Update the structures and generate the count */ 302 buf_count = 0; 303 while (sg_wa->bytes_left && (buf_count < dm_wa->length)) { 304 nbytes = min(sg_wa->sg->length - sg_wa->sg_used, 305 dm_wa->length - buf_count); 306 nbytes = min_t(u64, sg_wa->bytes_left, nbytes); 307 308 buf_count += nbytes; 309 ccp_update_sg_workarea(sg_wa, nbytes); 310 } 311 312 return buf_count; 313 } 314 315 static unsigned int ccp_fill_queue_buf(struct ccp_data *data) 316 { 317 return ccp_queue_buf(data, 0); 318 } 319 320 static unsigned int ccp_empty_queue_buf(struct ccp_data *data) 321 { 322 return ccp_queue_buf(data, 1); 323 } 324 325 static void ccp_prepare_data(struct ccp_data *src, struct ccp_data *dst, 326 struct ccp_op *op, unsigned int block_size, 327 bool blocksize_op) 328 { 329 unsigned int sg_src_len, sg_dst_len, op_len; 330 331 /* The CCP can only DMA from/to one address each per operation. This 332 * requires that we find the smallest DMA area between the source 333 * and destination. The resulting len values will always be <= UINT_MAX 334 * because the dma length is an unsigned int. 335 */ 336 sg_src_len = sg_dma_len(src->sg_wa.sg) - src->sg_wa.sg_used; 337 sg_src_len = min_t(u64, src->sg_wa.bytes_left, sg_src_len); 338 339 if (dst) { 340 sg_dst_len = sg_dma_len(dst->sg_wa.sg) - dst->sg_wa.sg_used; 341 sg_dst_len = min_t(u64, src->sg_wa.bytes_left, sg_dst_len); 342 op_len = min(sg_src_len, sg_dst_len); 343 } else { 344 op_len = sg_src_len; 345 } 346 347 /* The data operation length will be at least block_size in length 348 * or the smaller of available sg room remaining for the source or 349 * the destination 350 */ 351 op_len = max(op_len, block_size); 352 353 /* Unless we have to buffer data, there's no reason to wait */ 354 op->soc = 0; 355 356 if (sg_src_len < block_size) { 357 /* Not enough data in the sg element, so it 358 * needs to be buffered into a blocksize chunk 359 */ 360 int cp_len = ccp_fill_queue_buf(src); 361 362 op->soc = 1; 363 op->src.u.dma.address = src->dm_wa.dma.address; 364 op->src.u.dma.offset = 0; 365 op->src.u.dma.length = (blocksize_op) ? block_size : cp_len; 366 } else { 367 /* Enough data in the sg element, but we need to 368 * adjust for any previously copied data 369 */ 370 op->src.u.dma.address = sg_dma_address(src->sg_wa.sg); 371 op->src.u.dma.offset = src->sg_wa.sg_used; 372 op->src.u.dma.length = op_len & ~(block_size - 1); 373 374 ccp_update_sg_workarea(&src->sg_wa, op->src.u.dma.length); 375 } 376 377 if (dst) { 378 if (sg_dst_len < block_size) { 379 /* Not enough room in the sg element or we're on the 380 * last piece of data (when using padding), so the 381 * output needs to be buffered into a blocksize chunk 382 */ 383 op->soc = 1; 384 op->dst.u.dma.address = dst->dm_wa.dma.address; 385 op->dst.u.dma.offset = 0; 386 op->dst.u.dma.length = op->src.u.dma.length; 387 } else { 388 /* Enough room in the sg element, but we need to 389 * adjust for any previously used area 390 */ 391 op->dst.u.dma.address = sg_dma_address(dst->sg_wa.sg); 392 op->dst.u.dma.offset = dst->sg_wa.sg_used; 393 op->dst.u.dma.length = op->src.u.dma.length; 394 } 395 } 396 } 397 398 static void ccp_process_data(struct ccp_data *src, struct ccp_data *dst, 399 struct ccp_op *op) 400 { 401 op->init = 0; 402 403 if (dst) { 404 if (op->dst.u.dma.address == dst->dm_wa.dma.address) 405 ccp_empty_queue_buf(dst); 406 else 407 ccp_update_sg_workarea(&dst->sg_wa, 408 op->dst.u.dma.length); 409 } 410 } 411 412 static int ccp_copy_to_from_sb(struct ccp_cmd_queue *cmd_q, 413 struct ccp_dm_workarea *wa, u32 jobid, u32 sb, 414 u32 byte_swap, bool from) 415 { 416 struct ccp_op op; 417 418 memset(&op, 0, sizeof(op)); 419 420 op.cmd_q = cmd_q; 421 op.jobid = jobid; 422 op.eom = 1; 423 424 if (from) { 425 op.soc = 1; 426 op.src.type = CCP_MEMTYPE_SB; 427 op.src.u.sb = sb; 428 op.dst.type = CCP_MEMTYPE_SYSTEM; 429 op.dst.u.dma.address = wa->dma.address; 430 op.dst.u.dma.length = wa->length; 431 } else { 432 op.src.type = CCP_MEMTYPE_SYSTEM; 433 op.src.u.dma.address = wa->dma.address; 434 op.src.u.dma.length = wa->length; 435 op.dst.type = CCP_MEMTYPE_SB; 436 op.dst.u.sb = sb; 437 } 438 439 op.u.passthru.byte_swap = byte_swap; 440 441 return cmd_q->ccp->vdata->perform->passthru(&op); 442 } 443 444 static int ccp_copy_to_sb(struct ccp_cmd_queue *cmd_q, 445 struct ccp_dm_workarea *wa, u32 jobid, u32 sb, 446 u32 byte_swap) 447 { 448 return ccp_copy_to_from_sb(cmd_q, wa, jobid, sb, byte_swap, false); 449 } 450 451 static int ccp_copy_from_sb(struct ccp_cmd_queue *cmd_q, 452 struct ccp_dm_workarea *wa, u32 jobid, u32 sb, 453 u32 byte_swap) 454 { 455 return ccp_copy_to_from_sb(cmd_q, wa, jobid, sb, byte_swap, true); 456 } 457 458 static int ccp_run_aes_cmac_cmd(struct ccp_cmd_queue *cmd_q, 459 struct ccp_cmd *cmd) 460 { 461 struct ccp_aes_engine *aes = &cmd->u.aes; 462 struct ccp_dm_workarea key, ctx; 463 struct ccp_data src; 464 struct ccp_op op; 465 unsigned int dm_offset; 466 int ret; 467 468 if (!((aes->key_len == AES_KEYSIZE_128) || 469 (aes->key_len == AES_KEYSIZE_192) || 470 (aes->key_len == AES_KEYSIZE_256))) 471 return -EINVAL; 472 473 if (aes->src_len & (AES_BLOCK_SIZE - 1)) 474 return -EINVAL; 475 476 if (aes->iv_len != AES_BLOCK_SIZE) 477 return -EINVAL; 478 479 if (!aes->key || !aes->iv || !aes->src) 480 return -EINVAL; 481 482 if (aes->cmac_final) { 483 if (aes->cmac_key_len != AES_BLOCK_SIZE) 484 return -EINVAL; 485 486 if (!aes->cmac_key) 487 return -EINVAL; 488 } 489 490 BUILD_BUG_ON(CCP_AES_KEY_SB_COUNT != 1); 491 BUILD_BUG_ON(CCP_AES_CTX_SB_COUNT != 1); 492 493 ret = -EIO; 494 memset(&op, 0, sizeof(op)); 495 op.cmd_q = cmd_q; 496 op.jobid = CCP_NEW_JOBID(cmd_q->ccp); 497 op.sb_key = cmd_q->sb_key; 498 op.sb_ctx = cmd_q->sb_ctx; 499 op.init = 1; 500 op.u.aes.type = aes->type; 501 op.u.aes.mode = aes->mode; 502 op.u.aes.action = aes->action; 503 504 /* All supported key sizes fit in a single (32-byte) SB entry 505 * and must be in little endian format. Use the 256-bit byte 506 * swap passthru option to convert from big endian to little 507 * endian. 508 */ 509 ret = ccp_init_dm_workarea(&key, cmd_q, 510 CCP_AES_KEY_SB_COUNT * CCP_SB_BYTES, 511 DMA_TO_DEVICE); 512 if (ret) 513 return ret; 514 515 dm_offset = CCP_SB_BYTES - aes->key_len; 516 ret = ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len); 517 if (ret) 518 goto e_key; 519 ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key, 520 CCP_PASSTHRU_BYTESWAP_256BIT); 521 if (ret) { 522 cmd->engine_error = cmd_q->cmd_error; 523 goto e_key; 524 } 525 526 /* The AES context fits in a single (32-byte) SB entry and 527 * must be in little endian format. Use the 256-bit byte swap 528 * passthru option to convert from big endian to little endian. 529 */ 530 ret = ccp_init_dm_workarea(&ctx, cmd_q, 531 CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES, 532 DMA_BIDIRECTIONAL); 533 if (ret) 534 goto e_key; 535 536 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE; 537 ret = ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len); 538 if (ret) 539 goto e_ctx; 540 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, 541 CCP_PASSTHRU_BYTESWAP_256BIT); 542 if (ret) { 543 cmd->engine_error = cmd_q->cmd_error; 544 goto e_ctx; 545 } 546 547 /* Send data to the CCP AES engine */ 548 ret = ccp_init_data(&src, cmd_q, aes->src, aes->src_len, 549 AES_BLOCK_SIZE, DMA_TO_DEVICE); 550 if (ret) 551 goto e_ctx; 552 553 while (src.sg_wa.bytes_left) { 554 ccp_prepare_data(&src, NULL, &op, AES_BLOCK_SIZE, true); 555 if (aes->cmac_final && !src.sg_wa.bytes_left) { 556 op.eom = 1; 557 558 /* Push the K1/K2 key to the CCP now */ 559 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, 560 op.sb_ctx, 561 CCP_PASSTHRU_BYTESWAP_256BIT); 562 if (ret) { 563 cmd->engine_error = cmd_q->cmd_error; 564 goto e_src; 565 } 566 567 ret = ccp_set_dm_area(&ctx, 0, aes->cmac_key, 0, 568 aes->cmac_key_len); 569 if (ret) 570 goto e_src; 571 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, 572 CCP_PASSTHRU_BYTESWAP_256BIT); 573 if (ret) { 574 cmd->engine_error = cmd_q->cmd_error; 575 goto e_src; 576 } 577 } 578 579 ret = cmd_q->ccp->vdata->perform->aes(&op); 580 if (ret) { 581 cmd->engine_error = cmd_q->cmd_error; 582 goto e_src; 583 } 584 585 ccp_process_data(&src, NULL, &op); 586 } 587 588 /* Retrieve the AES context - convert from LE to BE using 589 * 32-byte (256-bit) byteswapping 590 */ 591 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, 592 CCP_PASSTHRU_BYTESWAP_256BIT); 593 if (ret) { 594 cmd->engine_error = cmd_q->cmd_error; 595 goto e_src; 596 } 597 598 /* ...but we only need AES_BLOCK_SIZE bytes */ 599 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE; 600 ccp_get_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len); 601 602 e_src: 603 ccp_free_data(&src, cmd_q); 604 605 e_ctx: 606 ccp_dm_free(&ctx); 607 608 e_key: 609 ccp_dm_free(&key); 610 611 return ret; 612 } 613 614 static int ccp_run_aes_gcm_cmd(struct ccp_cmd_queue *cmd_q, 615 struct ccp_cmd *cmd) 616 { 617 struct ccp_aes_engine *aes = &cmd->u.aes; 618 struct ccp_dm_workarea key, ctx, final_wa, tag; 619 struct ccp_data src, dst; 620 struct ccp_data aad; 621 struct ccp_op op; 622 623 unsigned long long *final; 624 unsigned int dm_offset; 625 unsigned int jobid; 626 unsigned int ilen; 627 bool in_place = true; /* Default value */ 628 int ret; 629 630 struct scatterlist *p_inp, sg_inp[2]; 631 struct scatterlist *p_tag, sg_tag[2]; 632 struct scatterlist *p_outp, sg_outp[2]; 633 struct scatterlist *p_aad; 634 635 if (!aes->iv) 636 return -EINVAL; 637 638 if (!((aes->key_len == AES_KEYSIZE_128) || 639 (aes->key_len == AES_KEYSIZE_192) || 640 (aes->key_len == AES_KEYSIZE_256))) 641 return -EINVAL; 642 643 if (!aes->key) /* Gotta have a key SGL */ 644 return -EINVAL; 645 646 /* First, decompose the source buffer into AAD & PT, 647 * and the destination buffer into AAD, CT & tag, or 648 * the input into CT & tag. 649 * It is expected that the input and output SGs will 650 * be valid, even if the AAD and input lengths are 0. 651 */ 652 p_aad = aes->src; 653 p_inp = scatterwalk_ffwd(sg_inp, aes->src, aes->aad_len); 654 p_outp = scatterwalk_ffwd(sg_outp, aes->dst, aes->aad_len); 655 if (aes->action == CCP_AES_ACTION_ENCRYPT) { 656 ilen = aes->src_len; 657 p_tag = scatterwalk_ffwd(sg_tag, p_outp, ilen); 658 } else { 659 /* Input length for decryption includes tag */ 660 ilen = aes->src_len - AES_BLOCK_SIZE; 661 p_tag = scatterwalk_ffwd(sg_tag, p_inp, ilen); 662 } 663 664 jobid = CCP_NEW_JOBID(cmd_q->ccp); 665 666 memset(&op, 0, sizeof(op)); 667 op.cmd_q = cmd_q; 668 op.jobid = jobid; 669 op.sb_key = cmd_q->sb_key; /* Pre-allocated */ 670 op.sb_ctx = cmd_q->sb_ctx; /* Pre-allocated */ 671 op.init = 1; 672 op.u.aes.type = aes->type; 673 674 /* Copy the key to the LSB */ 675 ret = ccp_init_dm_workarea(&key, cmd_q, 676 CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES, 677 DMA_TO_DEVICE); 678 if (ret) 679 return ret; 680 681 dm_offset = CCP_SB_BYTES - aes->key_len; 682 ret = ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len); 683 if (ret) 684 goto e_key; 685 ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key, 686 CCP_PASSTHRU_BYTESWAP_256BIT); 687 if (ret) { 688 cmd->engine_error = cmd_q->cmd_error; 689 goto e_key; 690 } 691 692 /* Copy the context (IV) to the LSB. 693 * There is an assumption here that the IV is 96 bits in length, plus 694 * a nonce of 32 bits. If no IV is present, use a zeroed buffer. 695 */ 696 ret = ccp_init_dm_workarea(&ctx, cmd_q, 697 CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES, 698 DMA_BIDIRECTIONAL); 699 if (ret) 700 goto e_key; 701 702 dm_offset = CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES - aes->iv_len; 703 ret = ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len); 704 if (ret) 705 goto e_ctx; 706 707 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, 708 CCP_PASSTHRU_BYTESWAP_256BIT); 709 if (ret) { 710 cmd->engine_error = cmd_q->cmd_error; 711 goto e_ctx; 712 } 713 714 op.init = 1; 715 if (aes->aad_len > 0) { 716 /* Step 1: Run a GHASH over the Additional Authenticated Data */ 717 ret = ccp_init_data(&aad, cmd_q, p_aad, aes->aad_len, 718 AES_BLOCK_SIZE, 719 DMA_TO_DEVICE); 720 if (ret) 721 goto e_ctx; 722 723 op.u.aes.mode = CCP_AES_MODE_GHASH; 724 op.u.aes.action = CCP_AES_GHASHAAD; 725 726 while (aad.sg_wa.bytes_left) { 727 ccp_prepare_data(&aad, NULL, &op, AES_BLOCK_SIZE, true); 728 729 ret = cmd_q->ccp->vdata->perform->aes(&op); 730 if (ret) { 731 cmd->engine_error = cmd_q->cmd_error; 732 goto e_aad; 733 } 734 735 ccp_process_data(&aad, NULL, &op); 736 op.init = 0; 737 } 738 } 739 740 op.u.aes.mode = CCP_AES_MODE_GCTR; 741 op.u.aes.action = aes->action; 742 743 if (ilen > 0) { 744 /* Step 2: Run a GCTR over the plaintext */ 745 in_place = (sg_virt(p_inp) == sg_virt(p_outp)) ? true : false; 746 747 ret = ccp_init_data(&src, cmd_q, p_inp, ilen, 748 AES_BLOCK_SIZE, 749 in_place ? DMA_BIDIRECTIONAL 750 : DMA_TO_DEVICE); 751 if (ret) 752 goto e_ctx; 753 754 if (in_place) { 755 dst = src; 756 } else { 757 ret = ccp_init_data(&dst, cmd_q, p_outp, ilen, 758 AES_BLOCK_SIZE, DMA_FROM_DEVICE); 759 if (ret) 760 goto e_src; 761 } 762 763 op.soc = 0; 764 op.eom = 0; 765 op.init = 1; 766 while (src.sg_wa.bytes_left) { 767 ccp_prepare_data(&src, &dst, &op, AES_BLOCK_SIZE, true); 768 if (!src.sg_wa.bytes_left) { 769 unsigned int nbytes = aes->src_len 770 % AES_BLOCK_SIZE; 771 772 if (nbytes) { 773 op.eom = 1; 774 op.u.aes.size = (nbytes * 8) - 1; 775 } 776 } 777 778 ret = cmd_q->ccp->vdata->perform->aes(&op); 779 if (ret) { 780 cmd->engine_error = cmd_q->cmd_error; 781 goto e_dst; 782 } 783 784 ccp_process_data(&src, &dst, &op); 785 op.init = 0; 786 } 787 } 788 789 /* Step 3: Update the IV portion of the context with the original IV */ 790 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, 791 CCP_PASSTHRU_BYTESWAP_256BIT); 792 if (ret) { 793 cmd->engine_error = cmd_q->cmd_error; 794 goto e_dst; 795 } 796 797 ret = ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len); 798 if (ret) 799 goto e_dst; 800 801 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, 802 CCP_PASSTHRU_BYTESWAP_256BIT); 803 if (ret) { 804 cmd->engine_error = cmd_q->cmd_error; 805 goto e_dst; 806 } 807 808 /* Step 4: Concatenate the lengths of the AAD and source, and 809 * hash that 16 byte buffer. 810 */ 811 ret = ccp_init_dm_workarea(&final_wa, cmd_q, AES_BLOCK_SIZE, 812 DMA_BIDIRECTIONAL); 813 if (ret) 814 goto e_dst; 815 final = (unsigned long long *) final_wa.address; 816 final[0] = cpu_to_be64(aes->aad_len * 8); 817 final[1] = cpu_to_be64(ilen * 8); 818 819 memset(&op, 0, sizeof(op)); 820 op.cmd_q = cmd_q; 821 op.jobid = jobid; 822 op.sb_key = cmd_q->sb_key; /* Pre-allocated */ 823 op.sb_ctx = cmd_q->sb_ctx; /* Pre-allocated */ 824 op.init = 1; 825 op.u.aes.type = aes->type; 826 op.u.aes.mode = CCP_AES_MODE_GHASH; 827 op.u.aes.action = CCP_AES_GHASHFINAL; 828 op.src.type = CCP_MEMTYPE_SYSTEM; 829 op.src.u.dma.address = final_wa.dma.address; 830 op.src.u.dma.length = AES_BLOCK_SIZE; 831 op.dst.type = CCP_MEMTYPE_SYSTEM; 832 op.dst.u.dma.address = final_wa.dma.address; 833 op.dst.u.dma.length = AES_BLOCK_SIZE; 834 op.eom = 1; 835 op.u.aes.size = 0; 836 ret = cmd_q->ccp->vdata->perform->aes(&op); 837 if (ret) 838 goto e_dst; 839 840 if (aes->action == CCP_AES_ACTION_ENCRYPT) { 841 /* Put the ciphered tag after the ciphertext. */ 842 ccp_get_dm_area(&final_wa, 0, p_tag, 0, AES_BLOCK_SIZE); 843 } else { 844 /* Does this ciphered tag match the input? */ 845 ret = ccp_init_dm_workarea(&tag, cmd_q, AES_BLOCK_SIZE, 846 DMA_BIDIRECTIONAL); 847 if (ret) 848 goto e_tag; 849 ret = ccp_set_dm_area(&tag, 0, p_tag, 0, AES_BLOCK_SIZE); 850 if (ret) 851 goto e_tag; 852 853 ret = memcmp(tag.address, final_wa.address, AES_BLOCK_SIZE); 854 ccp_dm_free(&tag); 855 } 856 857 e_tag: 858 ccp_dm_free(&final_wa); 859 860 e_dst: 861 if (aes->src_len && !in_place) 862 ccp_free_data(&dst, cmd_q); 863 864 e_src: 865 if (aes->src_len) 866 ccp_free_data(&src, cmd_q); 867 868 e_aad: 869 if (aes->aad_len) 870 ccp_free_data(&aad, cmd_q); 871 872 e_ctx: 873 ccp_dm_free(&ctx); 874 875 e_key: 876 ccp_dm_free(&key); 877 878 return ret; 879 } 880 881 static int ccp_run_aes_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) 882 { 883 struct ccp_aes_engine *aes = &cmd->u.aes; 884 struct ccp_dm_workarea key, ctx; 885 struct ccp_data src, dst; 886 struct ccp_op op; 887 unsigned int dm_offset; 888 bool in_place = false; 889 int ret; 890 891 if (aes->mode == CCP_AES_MODE_CMAC) 892 return ccp_run_aes_cmac_cmd(cmd_q, cmd); 893 894 if (aes->mode == CCP_AES_MODE_GCM) 895 return ccp_run_aes_gcm_cmd(cmd_q, cmd); 896 897 if (!((aes->key_len == AES_KEYSIZE_128) || 898 (aes->key_len == AES_KEYSIZE_192) || 899 (aes->key_len == AES_KEYSIZE_256))) 900 return -EINVAL; 901 902 if (((aes->mode == CCP_AES_MODE_ECB) || 903 (aes->mode == CCP_AES_MODE_CBC)) && 904 (aes->src_len & (AES_BLOCK_SIZE - 1))) 905 return -EINVAL; 906 907 if (!aes->key || !aes->src || !aes->dst) 908 return -EINVAL; 909 910 if (aes->mode != CCP_AES_MODE_ECB) { 911 if (aes->iv_len != AES_BLOCK_SIZE) 912 return -EINVAL; 913 914 if (!aes->iv) 915 return -EINVAL; 916 } 917 918 BUILD_BUG_ON(CCP_AES_KEY_SB_COUNT != 1); 919 BUILD_BUG_ON(CCP_AES_CTX_SB_COUNT != 1); 920 921 ret = -EIO; 922 memset(&op, 0, sizeof(op)); 923 op.cmd_q = cmd_q; 924 op.jobid = CCP_NEW_JOBID(cmd_q->ccp); 925 op.sb_key = cmd_q->sb_key; 926 op.sb_ctx = cmd_q->sb_ctx; 927 op.init = (aes->mode == CCP_AES_MODE_ECB) ? 0 : 1; 928 op.u.aes.type = aes->type; 929 op.u.aes.mode = aes->mode; 930 op.u.aes.action = aes->action; 931 932 /* All supported key sizes fit in a single (32-byte) SB entry 933 * and must be in little endian format. Use the 256-bit byte 934 * swap passthru option to convert from big endian to little 935 * endian. 936 */ 937 ret = ccp_init_dm_workarea(&key, cmd_q, 938 CCP_AES_KEY_SB_COUNT * CCP_SB_BYTES, 939 DMA_TO_DEVICE); 940 if (ret) 941 return ret; 942 943 dm_offset = CCP_SB_BYTES - aes->key_len; 944 ret = ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len); 945 if (ret) 946 goto e_key; 947 ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key, 948 CCP_PASSTHRU_BYTESWAP_256BIT); 949 if (ret) { 950 cmd->engine_error = cmd_q->cmd_error; 951 goto e_key; 952 } 953 954 /* The AES context fits in a single (32-byte) SB entry and 955 * must be in little endian format. Use the 256-bit byte swap 956 * passthru option to convert from big endian to little endian. 957 */ 958 ret = ccp_init_dm_workarea(&ctx, cmd_q, 959 CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES, 960 DMA_BIDIRECTIONAL); 961 if (ret) 962 goto e_key; 963 964 if (aes->mode != CCP_AES_MODE_ECB) { 965 /* Load the AES context - convert to LE */ 966 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE; 967 ret = ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len); 968 if (ret) 969 goto e_ctx; 970 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, 971 CCP_PASSTHRU_BYTESWAP_256BIT); 972 if (ret) { 973 cmd->engine_error = cmd_q->cmd_error; 974 goto e_ctx; 975 } 976 } 977 switch (aes->mode) { 978 case CCP_AES_MODE_CFB: /* CFB128 only */ 979 case CCP_AES_MODE_CTR: 980 op.u.aes.size = AES_BLOCK_SIZE * BITS_PER_BYTE - 1; 981 break; 982 default: 983 op.u.aes.size = 0; 984 } 985 986 /* Prepare the input and output data workareas. For in-place 987 * operations we need to set the dma direction to BIDIRECTIONAL 988 * and copy the src workarea to the dst workarea. 989 */ 990 if (sg_virt(aes->src) == sg_virt(aes->dst)) 991 in_place = true; 992 993 ret = ccp_init_data(&src, cmd_q, aes->src, aes->src_len, 994 AES_BLOCK_SIZE, 995 in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE); 996 if (ret) 997 goto e_ctx; 998 999 if (in_place) { 1000 dst = src; 1001 } else { 1002 ret = ccp_init_data(&dst, cmd_q, aes->dst, aes->src_len, 1003 AES_BLOCK_SIZE, DMA_FROM_DEVICE); 1004 if (ret) 1005 goto e_src; 1006 } 1007 1008 /* Send data to the CCP AES engine */ 1009 while (src.sg_wa.bytes_left) { 1010 ccp_prepare_data(&src, &dst, &op, AES_BLOCK_SIZE, true); 1011 if (!src.sg_wa.bytes_left) { 1012 op.eom = 1; 1013 1014 /* Since we don't retrieve the AES context in ECB 1015 * mode we have to wait for the operation to complete 1016 * on the last piece of data 1017 */ 1018 if (aes->mode == CCP_AES_MODE_ECB) 1019 op.soc = 1; 1020 } 1021 1022 ret = cmd_q->ccp->vdata->perform->aes(&op); 1023 if (ret) { 1024 cmd->engine_error = cmd_q->cmd_error; 1025 goto e_dst; 1026 } 1027 1028 ccp_process_data(&src, &dst, &op); 1029 } 1030 1031 if (aes->mode != CCP_AES_MODE_ECB) { 1032 /* Retrieve the AES context - convert from LE to BE using 1033 * 32-byte (256-bit) byteswapping 1034 */ 1035 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, 1036 CCP_PASSTHRU_BYTESWAP_256BIT); 1037 if (ret) { 1038 cmd->engine_error = cmd_q->cmd_error; 1039 goto e_dst; 1040 } 1041 1042 /* ...but we only need AES_BLOCK_SIZE bytes */ 1043 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE; 1044 ccp_get_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len); 1045 } 1046 1047 e_dst: 1048 if (!in_place) 1049 ccp_free_data(&dst, cmd_q); 1050 1051 e_src: 1052 ccp_free_data(&src, cmd_q); 1053 1054 e_ctx: 1055 ccp_dm_free(&ctx); 1056 1057 e_key: 1058 ccp_dm_free(&key); 1059 1060 return ret; 1061 } 1062 1063 static int ccp_run_xts_aes_cmd(struct ccp_cmd_queue *cmd_q, 1064 struct ccp_cmd *cmd) 1065 { 1066 struct ccp_xts_aes_engine *xts = &cmd->u.xts; 1067 struct ccp_dm_workarea key, ctx; 1068 struct ccp_data src, dst; 1069 struct ccp_op op; 1070 unsigned int unit_size, dm_offset; 1071 bool in_place = false; 1072 unsigned int sb_count; 1073 enum ccp_aes_type aestype; 1074 int ret; 1075 1076 switch (xts->unit_size) { 1077 case CCP_XTS_AES_UNIT_SIZE_16: 1078 unit_size = 16; 1079 break; 1080 case CCP_XTS_AES_UNIT_SIZE_512: 1081 unit_size = 512; 1082 break; 1083 case CCP_XTS_AES_UNIT_SIZE_1024: 1084 unit_size = 1024; 1085 break; 1086 case CCP_XTS_AES_UNIT_SIZE_2048: 1087 unit_size = 2048; 1088 break; 1089 case CCP_XTS_AES_UNIT_SIZE_4096: 1090 unit_size = 4096; 1091 break; 1092 1093 default: 1094 return -EINVAL; 1095 } 1096 1097 if (xts->key_len == AES_KEYSIZE_128) 1098 aestype = CCP_AES_TYPE_128; 1099 else if (xts->key_len == AES_KEYSIZE_256) 1100 aestype = CCP_AES_TYPE_256; 1101 else 1102 return -EINVAL; 1103 1104 if (!xts->final && (xts->src_len & (AES_BLOCK_SIZE - 1))) 1105 return -EINVAL; 1106 1107 if (xts->iv_len != AES_BLOCK_SIZE) 1108 return -EINVAL; 1109 1110 if (!xts->key || !xts->iv || !xts->src || !xts->dst) 1111 return -EINVAL; 1112 1113 BUILD_BUG_ON(CCP_XTS_AES_KEY_SB_COUNT != 1); 1114 BUILD_BUG_ON(CCP_XTS_AES_CTX_SB_COUNT != 1); 1115 1116 ret = -EIO; 1117 memset(&op, 0, sizeof(op)); 1118 op.cmd_q = cmd_q; 1119 op.jobid = CCP_NEW_JOBID(cmd_q->ccp); 1120 op.sb_key = cmd_q->sb_key; 1121 op.sb_ctx = cmd_q->sb_ctx; 1122 op.init = 1; 1123 op.u.xts.type = aestype; 1124 op.u.xts.action = xts->action; 1125 op.u.xts.unit_size = xts->unit_size; 1126 1127 /* A version 3 device only supports 128-bit keys, which fits into a 1128 * single SB entry. A version 5 device uses a 512-bit vector, so two 1129 * SB entries. 1130 */ 1131 if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0)) 1132 sb_count = CCP_XTS_AES_KEY_SB_COUNT; 1133 else 1134 sb_count = CCP5_XTS_AES_KEY_SB_COUNT; 1135 ret = ccp_init_dm_workarea(&key, cmd_q, 1136 sb_count * CCP_SB_BYTES, 1137 DMA_TO_DEVICE); 1138 if (ret) 1139 return ret; 1140 1141 if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0)) { 1142 /* All supported key sizes must be in little endian format. 1143 * Use the 256-bit byte swap passthru option to convert from 1144 * big endian to little endian. 1145 */ 1146 dm_offset = CCP_SB_BYTES - AES_KEYSIZE_128; 1147 ret = ccp_set_dm_area(&key, dm_offset, xts->key, 0, xts->key_len); 1148 if (ret) 1149 goto e_key; 1150 ret = ccp_set_dm_area(&key, 0, xts->key, xts->key_len, xts->key_len); 1151 if (ret) 1152 goto e_key; 1153 } else { 1154 /* Version 5 CCPs use a 512-bit space for the key: each portion 1155 * occupies 256 bits, or one entire slot, and is zero-padded. 1156 */ 1157 unsigned int pad; 1158 1159 dm_offset = CCP_SB_BYTES; 1160 pad = dm_offset - xts->key_len; 1161 ret = ccp_set_dm_area(&key, pad, xts->key, 0, xts->key_len); 1162 if (ret) 1163 goto e_key; 1164 ret = ccp_set_dm_area(&key, dm_offset + pad, xts->key, 1165 xts->key_len, xts->key_len); 1166 if (ret) 1167 goto e_key; 1168 } 1169 ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key, 1170 CCP_PASSTHRU_BYTESWAP_256BIT); 1171 if (ret) { 1172 cmd->engine_error = cmd_q->cmd_error; 1173 goto e_key; 1174 } 1175 1176 /* The AES context fits in a single (32-byte) SB entry and 1177 * for XTS is already in little endian format so no byte swapping 1178 * is needed. 1179 */ 1180 ret = ccp_init_dm_workarea(&ctx, cmd_q, 1181 CCP_XTS_AES_CTX_SB_COUNT * CCP_SB_BYTES, 1182 DMA_BIDIRECTIONAL); 1183 if (ret) 1184 goto e_key; 1185 1186 ret = ccp_set_dm_area(&ctx, 0, xts->iv, 0, xts->iv_len); 1187 if (ret) 1188 goto e_ctx; 1189 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, 1190 CCP_PASSTHRU_BYTESWAP_NOOP); 1191 if (ret) { 1192 cmd->engine_error = cmd_q->cmd_error; 1193 goto e_ctx; 1194 } 1195 1196 /* Prepare the input and output data workareas. For in-place 1197 * operations we need to set the dma direction to BIDIRECTIONAL 1198 * and copy the src workarea to the dst workarea. 1199 */ 1200 if (sg_virt(xts->src) == sg_virt(xts->dst)) 1201 in_place = true; 1202 1203 ret = ccp_init_data(&src, cmd_q, xts->src, xts->src_len, 1204 unit_size, 1205 in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE); 1206 if (ret) 1207 goto e_ctx; 1208 1209 if (in_place) { 1210 dst = src; 1211 } else { 1212 ret = ccp_init_data(&dst, cmd_q, xts->dst, xts->src_len, 1213 unit_size, DMA_FROM_DEVICE); 1214 if (ret) 1215 goto e_src; 1216 } 1217 1218 /* Send data to the CCP AES engine */ 1219 while (src.sg_wa.bytes_left) { 1220 ccp_prepare_data(&src, &dst, &op, unit_size, true); 1221 if (!src.sg_wa.bytes_left) 1222 op.eom = 1; 1223 1224 ret = cmd_q->ccp->vdata->perform->xts_aes(&op); 1225 if (ret) { 1226 cmd->engine_error = cmd_q->cmd_error; 1227 goto e_dst; 1228 } 1229 1230 ccp_process_data(&src, &dst, &op); 1231 } 1232 1233 /* Retrieve the AES context - convert from LE to BE using 1234 * 32-byte (256-bit) byteswapping 1235 */ 1236 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, 1237 CCP_PASSTHRU_BYTESWAP_256BIT); 1238 if (ret) { 1239 cmd->engine_error = cmd_q->cmd_error; 1240 goto e_dst; 1241 } 1242 1243 /* ...but we only need AES_BLOCK_SIZE bytes */ 1244 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE; 1245 ccp_get_dm_area(&ctx, dm_offset, xts->iv, 0, xts->iv_len); 1246 1247 e_dst: 1248 if (!in_place) 1249 ccp_free_data(&dst, cmd_q); 1250 1251 e_src: 1252 ccp_free_data(&src, cmd_q); 1253 1254 e_ctx: 1255 ccp_dm_free(&ctx); 1256 1257 e_key: 1258 ccp_dm_free(&key); 1259 1260 return ret; 1261 } 1262 1263 static int ccp_run_des3_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) 1264 { 1265 struct ccp_des3_engine *des3 = &cmd->u.des3; 1266 1267 struct ccp_dm_workarea key, ctx; 1268 struct ccp_data src, dst; 1269 struct ccp_op op; 1270 unsigned int dm_offset; 1271 unsigned int len_singlekey; 1272 bool in_place = false; 1273 int ret; 1274 1275 /* Error checks */ 1276 if (cmd_q->ccp->vdata->version < CCP_VERSION(5, 0)) 1277 return -EINVAL; 1278 1279 if (!cmd_q->ccp->vdata->perform->des3) 1280 return -EINVAL; 1281 1282 if (des3->key_len != DES3_EDE_KEY_SIZE) 1283 return -EINVAL; 1284 1285 if (((des3->mode == CCP_DES3_MODE_ECB) || 1286 (des3->mode == CCP_DES3_MODE_CBC)) && 1287 (des3->src_len & (DES3_EDE_BLOCK_SIZE - 1))) 1288 return -EINVAL; 1289 1290 if (!des3->key || !des3->src || !des3->dst) 1291 return -EINVAL; 1292 1293 if (des3->mode != CCP_DES3_MODE_ECB) { 1294 if (des3->iv_len != DES3_EDE_BLOCK_SIZE) 1295 return -EINVAL; 1296 1297 if (!des3->iv) 1298 return -EINVAL; 1299 } 1300 1301 ret = -EIO; 1302 /* Zero out all the fields of the command desc */ 1303 memset(&op, 0, sizeof(op)); 1304 1305 /* Set up the Function field */ 1306 op.cmd_q = cmd_q; 1307 op.jobid = CCP_NEW_JOBID(cmd_q->ccp); 1308 op.sb_key = cmd_q->sb_key; 1309 1310 op.init = (des3->mode == CCP_DES3_MODE_ECB) ? 0 : 1; 1311 op.u.des3.type = des3->type; 1312 op.u.des3.mode = des3->mode; 1313 op.u.des3.action = des3->action; 1314 1315 /* 1316 * All supported key sizes fit in a single (32-byte) KSB entry and 1317 * (like AES) must be in little endian format. Use the 256-bit byte 1318 * swap passthru option to convert from big endian to little endian. 1319 */ 1320 ret = ccp_init_dm_workarea(&key, cmd_q, 1321 CCP_DES3_KEY_SB_COUNT * CCP_SB_BYTES, 1322 DMA_TO_DEVICE); 1323 if (ret) 1324 return ret; 1325 1326 /* 1327 * The contents of the key triplet are in the reverse order of what 1328 * is required by the engine. Copy the 3 pieces individually to put 1329 * them where they belong. 1330 */ 1331 dm_offset = CCP_SB_BYTES - des3->key_len; /* Basic offset */ 1332 1333 len_singlekey = des3->key_len / 3; 1334 ret = ccp_set_dm_area(&key, dm_offset + 2 * len_singlekey, 1335 des3->key, 0, len_singlekey); 1336 if (ret) 1337 goto e_key; 1338 ret = ccp_set_dm_area(&key, dm_offset + len_singlekey, 1339 des3->key, len_singlekey, len_singlekey); 1340 if (ret) 1341 goto e_key; 1342 ret = ccp_set_dm_area(&key, dm_offset, 1343 des3->key, 2 * len_singlekey, len_singlekey); 1344 if (ret) 1345 goto e_key; 1346 1347 /* Copy the key to the SB */ 1348 ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key, 1349 CCP_PASSTHRU_BYTESWAP_256BIT); 1350 if (ret) { 1351 cmd->engine_error = cmd_q->cmd_error; 1352 goto e_key; 1353 } 1354 1355 /* 1356 * The DES3 context fits in a single (32-byte) KSB entry and 1357 * must be in little endian format. Use the 256-bit byte swap 1358 * passthru option to convert from big endian to little endian. 1359 */ 1360 if (des3->mode != CCP_DES3_MODE_ECB) { 1361 op.sb_ctx = cmd_q->sb_ctx; 1362 1363 ret = ccp_init_dm_workarea(&ctx, cmd_q, 1364 CCP_DES3_CTX_SB_COUNT * CCP_SB_BYTES, 1365 DMA_BIDIRECTIONAL); 1366 if (ret) 1367 goto e_key; 1368 1369 /* Load the context into the LSB */ 1370 dm_offset = CCP_SB_BYTES - des3->iv_len; 1371 ret = ccp_set_dm_area(&ctx, dm_offset, des3->iv, 0, 1372 des3->iv_len); 1373 if (ret) 1374 goto e_ctx; 1375 1376 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, 1377 CCP_PASSTHRU_BYTESWAP_256BIT); 1378 if (ret) { 1379 cmd->engine_error = cmd_q->cmd_error; 1380 goto e_ctx; 1381 } 1382 } 1383 1384 /* 1385 * Prepare the input and output data workareas. For in-place 1386 * operations we need to set the dma direction to BIDIRECTIONAL 1387 * and copy the src workarea to the dst workarea. 1388 */ 1389 if (sg_virt(des3->src) == sg_virt(des3->dst)) 1390 in_place = true; 1391 1392 ret = ccp_init_data(&src, cmd_q, des3->src, des3->src_len, 1393 DES3_EDE_BLOCK_SIZE, 1394 in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE); 1395 if (ret) 1396 goto e_ctx; 1397 1398 if (in_place) 1399 dst = src; 1400 else { 1401 ret = ccp_init_data(&dst, cmd_q, des3->dst, des3->src_len, 1402 DES3_EDE_BLOCK_SIZE, DMA_FROM_DEVICE); 1403 if (ret) 1404 goto e_src; 1405 } 1406 1407 /* Send data to the CCP DES3 engine */ 1408 while (src.sg_wa.bytes_left) { 1409 ccp_prepare_data(&src, &dst, &op, DES3_EDE_BLOCK_SIZE, true); 1410 if (!src.sg_wa.bytes_left) { 1411 op.eom = 1; 1412 1413 /* Since we don't retrieve the context in ECB mode 1414 * we have to wait for the operation to complete 1415 * on the last piece of data 1416 */ 1417 op.soc = 0; 1418 } 1419 1420 ret = cmd_q->ccp->vdata->perform->des3(&op); 1421 if (ret) { 1422 cmd->engine_error = cmd_q->cmd_error; 1423 goto e_dst; 1424 } 1425 1426 ccp_process_data(&src, &dst, &op); 1427 } 1428 1429 if (des3->mode != CCP_DES3_MODE_ECB) { 1430 /* Retrieve the context and make BE */ 1431 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, 1432 CCP_PASSTHRU_BYTESWAP_256BIT); 1433 if (ret) { 1434 cmd->engine_error = cmd_q->cmd_error; 1435 goto e_dst; 1436 } 1437 1438 /* ...but we only need the last DES3_EDE_BLOCK_SIZE bytes */ 1439 ccp_get_dm_area(&ctx, dm_offset, des3->iv, 0, 1440 DES3_EDE_BLOCK_SIZE); 1441 } 1442 e_dst: 1443 if (!in_place) 1444 ccp_free_data(&dst, cmd_q); 1445 1446 e_src: 1447 ccp_free_data(&src, cmd_q); 1448 1449 e_ctx: 1450 if (des3->mode != CCP_DES3_MODE_ECB) 1451 ccp_dm_free(&ctx); 1452 1453 e_key: 1454 ccp_dm_free(&key); 1455 1456 return ret; 1457 } 1458 1459 static int ccp_run_sha_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) 1460 { 1461 struct ccp_sha_engine *sha = &cmd->u.sha; 1462 struct ccp_dm_workarea ctx; 1463 struct ccp_data src; 1464 struct ccp_op op; 1465 unsigned int ioffset, ooffset; 1466 unsigned int digest_size; 1467 int sb_count; 1468 const void *init; 1469 u64 block_size; 1470 int ctx_size; 1471 int ret; 1472 1473 switch (sha->type) { 1474 case CCP_SHA_TYPE_1: 1475 if (sha->ctx_len < SHA1_DIGEST_SIZE) 1476 return -EINVAL; 1477 block_size = SHA1_BLOCK_SIZE; 1478 break; 1479 case CCP_SHA_TYPE_224: 1480 if (sha->ctx_len < SHA224_DIGEST_SIZE) 1481 return -EINVAL; 1482 block_size = SHA224_BLOCK_SIZE; 1483 break; 1484 case CCP_SHA_TYPE_256: 1485 if (sha->ctx_len < SHA256_DIGEST_SIZE) 1486 return -EINVAL; 1487 block_size = SHA256_BLOCK_SIZE; 1488 break; 1489 case CCP_SHA_TYPE_384: 1490 if (cmd_q->ccp->vdata->version < CCP_VERSION(4, 0) 1491 || sha->ctx_len < SHA384_DIGEST_SIZE) 1492 return -EINVAL; 1493 block_size = SHA384_BLOCK_SIZE; 1494 break; 1495 case CCP_SHA_TYPE_512: 1496 if (cmd_q->ccp->vdata->version < CCP_VERSION(4, 0) 1497 || sha->ctx_len < SHA512_DIGEST_SIZE) 1498 return -EINVAL; 1499 block_size = SHA512_BLOCK_SIZE; 1500 break; 1501 default: 1502 return -EINVAL; 1503 } 1504 1505 if (!sha->ctx) 1506 return -EINVAL; 1507 1508 if (!sha->final && (sha->src_len & (block_size - 1))) 1509 return -EINVAL; 1510 1511 /* The version 3 device can't handle zero-length input */ 1512 if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0)) { 1513 1514 if (!sha->src_len) { 1515 unsigned int digest_len; 1516 const u8 *sha_zero; 1517 1518 /* Not final, just return */ 1519 if (!sha->final) 1520 return 0; 1521 1522 /* CCP can't do a zero length sha operation so the 1523 * caller must buffer the data. 1524 */ 1525 if (sha->msg_bits) 1526 return -EINVAL; 1527 1528 /* The CCP cannot perform zero-length sha operations 1529 * so the caller is required to buffer data for the 1530 * final operation. However, a sha operation for a 1531 * message with a total length of zero is valid so 1532 * known values are required to supply the result. 1533 */ 1534 switch (sha->type) { 1535 case CCP_SHA_TYPE_1: 1536 sha_zero = sha1_zero_message_hash; 1537 digest_len = SHA1_DIGEST_SIZE; 1538 break; 1539 case CCP_SHA_TYPE_224: 1540 sha_zero = sha224_zero_message_hash; 1541 digest_len = SHA224_DIGEST_SIZE; 1542 break; 1543 case CCP_SHA_TYPE_256: 1544 sha_zero = sha256_zero_message_hash; 1545 digest_len = SHA256_DIGEST_SIZE; 1546 break; 1547 default: 1548 return -EINVAL; 1549 } 1550 1551 scatterwalk_map_and_copy((void *)sha_zero, sha->ctx, 0, 1552 digest_len, 1); 1553 1554 return 0; 1555 } 1556 } 1557 1558 /* Set variables used throughout */ 1559 switch (sha->type) { 1560 case CCP_SHA_TYPE_1: 1561 digest_size = SHA1_DIGEST_SIZE; 1562 init = (void *) ccp_sha1_init; 1563 ctx_size = SHA1_DIGEST_SIZE; 1564 sb_count = 1; 1565 if (cmd_q->ccp->vdata->version != CCP_VERSION(3, 0)) 1566 ooffset = ioffset = CCP_SB_BYTES - SHA1_DIGEST_SIZE; 1567 else 1568 ooffset = ioffset = 0; 1569 break; 1570 case CCP_SHA_TYPE_224: 1571 digest_size = SHA224_DIGEST_SIZE; 1572 init = (void *) ccp_sha224_init; 1573 ctx_size = SHA256_DIGEST_SIZE; 1574 sb_count = 1; 1575 ioffset = 0; 1576 if (cmd_q->ccp->vdata->version != CCP_VERSION(3, 0)) 1577 ooffset = CCP_SB_BYTES - SHA224_DIGEST_SIZE; 1578 else 1579 ooffset = 0; 1580 break; 1581 case CCP_SHA_TYPE_256: 1582 digest_size = SHA256_DIGEST_SIZE; 1583 init = (void *) ccp_sha256_init; 1584 ctx_size = SHA256_DIGEST_SIZE; 1585 sb_count = 1; 1586 ooffset = ioffset = 0; 1587 break; 1588 case CCP_SHA_TYPE_384: 1589 digest_size = SHA384_DIGEST_SIZE; 1590 init = (void *) ccp_sha384_init; 1591 ctx_size = SHA512_DIGEST_SIZE; 1592 sb_count = 2; 1593 ioffset = 0; 1594 ooffset = 2 * CCP_SB_BYTES - SHA384_DIGEST_SIZE; 1595 break; 1596 case CCP_SHA_TYPE_512: 1597 digest_size = SHA512_DIGEST_SIZE; 1598 init = (void *) ccp_sha512_init; 1599 ctx_size = SHA512_DIGEST_SIZE; 1600 sb_count = 2; 1601 ooffset = ioffset = 0; 1602 break; 1603 default: 1604 ret = -EINVAL; 1605 goto e_data; 1606 } 1607 1608 /* For zero-length plaintext the src pointer is ignored; 1609 * otherwise both parts must be valid 1610 */ 1611 if (sha->src_len && !sha->src) 1612 return -EINVAL; 1613 1614 memset(&op, 0, sizeof(op)); 1615 op.cmd_q = cmd_q; 1616 op.jobid = CCP_NEW_JOBID(cmd_q->ccp); 1617 op.sb_ctx = cmd_q->sb_ctx; /* Pre-allocated */ 1618 op.u.sha.type = sha->type; 1619 op.u.sha.msg_bits = sha->msg_bits; 1620 1621 /* For SHA1/224/256 the context fits in a single (32-byte) SB entry; 1622 * SHA384/512 require 2 adjacent SB slots, with the right half in the 1623 * first slot, and the left half in the second. Each portion must then 1624 * be in little endian format: use the 256-bit byte swap option. 1625 */ 1626 ret = ccp_init_dm_workarea(&ctx, cmd_q, sb_count * CCP_SB_BYTES, 1627 DMA_BIDIRECTIONAL); 1628 if (ret) 1629 return ret; 1630 if (sha->first) { 1631 switch (sha->type) { 1632 case CCP_SHA_TYPE_1: 1633 case CCP_SHA_TYPE_224: 1634 case CCP_SHA_TYPE_256: 1635 memcpy(ctx.address + ioffset, init, ctx_size); 1636 break; 1637 case CCP_SHA_TYPE_384: 1638 case CCP_SHA_TYPE_512: 1639 memcpy(ctx.address + ctx_size / 2, init, 1640 ctx_size / 2); 1641 memcpy(ctx.address, init + ctx_size / 2, 1642 ctx_size / 2); 1643 break; 1644 default: 1645 ret = -EINVAL; 1646 goto e_ctx; 1647 } 1648 } else { 1649 /* Restore the context */ 1650 ret = ccp_set_dm_area(&ctx, 0, sha->ctx, 0, 1651 sb_count * CCP_SB_BYTES); 1652 if (ret) 1653 goto e_ctx; 1654 } 1655 1656 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, 1657 CCP_PASSTHRU_BYTESWAP_256BIT); 1658 if (ret) { 1659 cmd->engine_error = cmd_q->cmd_error; 1660 goto e_ctx; 1661 } 1662 1663 if (sha->src) { 1664 /* Send data to the CCP SHA engine; block_size is set above */ 1665 ret = ccp_init_data(&src, cmd_q, sha->src, sha->src_len, 1666 block_size, DMA_TO_DEVICE); 1667 if (ret) 1668 goto e_ctx; 1669 1670 while (src.sg_wa.bytes_left) { 1671 ccp_prepare_data(&src, NULL, &op, block_size, false); 1672 if (sha->final && !src.sg_wa.bytes_left) 1673 op.eom = 1; 1674 1675 ret = cmd_q->ccp->vdata->perform->sha(&op); 1676 if (ret) { 1677 cmd->engine_error = cmd_q->cmd_error; 1678 goto e_data; 1679 } 1680 1681 ccp_process_data(&src, NULL, &op); 1682 } 1683 } else { 1684 op.eom = 1; 1685 ret = cmd_q->ccp->vdata->perform->sha(&op); 1686 if (ret) { 1687 cmd->engine_error = cmd_q->cmd_error; 1688 goto e_data; 1689 } 1690 } 1691 1692 /* Retrieve the SHA context - convert from LE to BE using 1693 * 32-byte (256-bit) byteswapping to BE 1694 */ 1695 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, 1696 CCP_PASSTHRU_BYTESWAP_256BIT); 1697 if (ret) { 1698 cmd->engine_error = cmd_q->cmd_error; 1699 goto e_data; 1700 } 1701 1702 if (sha->final) { 1703 /* Finishing up, so get the digest */ 1704 switch (sha->type) { 1705 case CCP_SHA_TYPE_1: 1706 case CCP_SHA_TYPE_224: 1707 case CCP_SHA_TYPE_256: 1708 ccp_get_dm_area(&ctx, ooffset, 1709 sha->ctx, 0, 1710 digest_size); 1711 break; 1712 case CCP_SHA_TYPE_384: 1713 case CCP_SHA_TYPE_512: 1714 ccp_get_dm_area(&ctx, 0, 1715 sha->ctx, LSB_ITEM_SIZE - ooffset, 1716 LSB_ITEM_SIZE); 1717 ccp_get_dm_area(&ctx, LSB_ITEM_SIZE + ooffset, 1718 sha->ctx, 0, 1719 LSB_ITEM_SIZE - ooffset); 1720 break; 1721 default: 1722 ret = -EINVAL; 1723 goto e_ctx; 1724 } 1725 } else { 1726 /* Stash the context */ 1727 ccp_get_dm_area(&ctx, 0, sha->ctx, 0, 1728 sb_count * CCP_SB_BYTES); 1729 } 1730 1731 if (sha->final && sha->opad) { 1732 /* HMAC operation, recursively perform final SHA */ 1733 struct ccp_cmd hmac_cmd; 1734 struct scatterlist sg; 1735 u8 *hmac_buf; 1736 1737 if (sha->opad_len != block_size) { 1738 ret = -EINVAL; 1739 goto e_data; 1740 } 1741 1742 hmac_buf = kmalloc(block_size + digest_size, GFP_KERNEL); 1743 if (!hmac_buf) { 1744 ret = -ENOMEM; 1745 goto e_data; 1746 } 1747 sg_init_one(&sg, hmac_buf, block_size + digest_size); 1748 1749 scatterwalk_map_and_copy(hmac_buf, sha->opad, 0, block_size, 0); 1750 switch (sha->type) { 1751 case CCP_SHA_TYPE_1: 1752 case CCP_SHA_TYPE_224: 1753 case CCP_SHA_TYPE_256: 1754 memcpy(hmac_buf + block_size, 1755 ctx.address + ooffset, 1756 digest_size); 1757 break; 1758 case CCP_SHA_TYPE_384: 1759 case CCP_SHA_TYPE_512: 1760 memcpy(hmac_buf + block_size, 1761 ctx.address + LSB_ITEM_SIZE + ooffset, 1762 LSB_ITEM_SIZE); 1763 memcpy(hmac_buf + block_size + 1764 (LSB_ITEM_SIZE - ooffset), 1765 ctx.address, 1766 LSB_ITEM_SIZE); 1767 break; 1768 default: 1769 ret = -EINVAL; 1770 goto e_ctx; 1771 } 1772 1773 memset(&hmac_cmd, 0, sizeof(hmac_cmd)); 1774 hmac_cmd.engine = CCP_ENGINE_SHA; 1775 hmac_cmd.u.sha.type = sha->type; 1776 hmac_cmd.u.sha.ctx = sha->ctx; 1777 hmac_cmd.u.sha.ctx_len = sha->ctx_len; 1778 hmac_cmd.u.sha.src = &sg; 1779 hmac_cmd.u.sha.src_len = block_size + digest_size; 1780 hmac_cmd.u.sha.opad = NULL; 1781 hmac_cmd.u.sha.opad_len = 0; 1782 hmac_cmd.u.sha.first = 1; 1783 hmac_cmd.u.sha.final = 1; 1784 hmac_cmd.u.sha.msg_bits = (block_size + digest_size) << 3; 1785 1786 ret = ccp_run_sha_cmd(cmd_q, &hmac_cmd); 1787 if (ret) 1788 cmd->engine_error = hmac_cmd.engine_error; 1789 1790 kfree(hmac_buf); 1791 } 1792 1793 e_data: 1794 if (sha->src) 1795 ccp_free_data(&src, cmd_q); 1796 1797 e_ctx: 1798 ccp_dm_free(&ctx); 1799 1800 return ret; 1801 } 1802 1803 static int ccp_run_rsa_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) 1804 { 1805 struct ccp_rsa_engine *rsa = &cmd->u.rsa; 1806 struct ccp_dm_workarea exp, src, dst; 1807 struct ccp_op op; 1808 unsigned int sb_count, i_len, o_len; 1809 int ret; 1810 1811 /* Check against the maximum allowable size, in bits */ 1812 if (rsa->key_size > cmd_q->ccp->vdata->rsamax) 1813 return -EINVAL; 1814 1815 if (!rsa->exp || !rsa->mod || !rsa->src || !rsa->dst) 1816 return -EINVAL; 1817 1818 memset(&op, 0, sizeof(op)); 1819 op.cmd_q = cmd_q; 1820 op.jobid = CCP_NEW_JOBID(cmd_q->ccp); 1821 1822 /* The RSA modulus must precede the message being acted upon, so 1823 * it must be copied to a DMA area where the message and the 1824 * modulus can be concatenated. Therefore the input buffer 1825 * length required is twice the output buffer length (which 1826 * must be a multiple of 256-bits). Compute o_len, i_len in bytes. 1827 * Buffer sizes must be a multiple of 32 bytes; rounding up may be 1828 * required. 1829 */ 1830 o_len = 32 * ((rsa->key_size + 255) / 256); 1831 i_len = o_len * 2; 1832 1833 sb_count = 0; 1834 if (cmd_q->ccp->vdata->version < CCP_VERSION(5, 0)) { 1835 /* sb_count is the number of storage block slots required 1836 * for the modulus. 1837 */ 1838 sb_count = o_len / CCP_SB_BYTES; 1839 op.sb_key = cmd_q->ccp->vdata->perform->sballoc(cmd_q, 1840 sb_count); 1841 if (!op.sb_key) 1842 return -EIO; 1843 } else { 1844 /* A version 5 device allows a modulus size that will not fit 1845 * in the LSB, so the command will transfer it from memory. 1846 * Set the sb key to the default, even though it's not used. 1847 */ 1848 op.sb_key = cmd_q->sb_key; 1849 } 1850 1851 /* The RSA exponent must be in little endian format. Reverse its 1852 * byte order. 1853 */ 1854 ret = ccp_init_dm_workarea(&exp, cmd_q, o_len, DMA_TO_DEVICE); 1855 if (ret) 1856 goto e_sb; 1857 1858 ret = ccp_reverse_set_dm_area(&exp, 0, rsa->exp, 0, rsa->exp_len); 1859 if (ret) 1860 goto e_exp; 1861 1862 if (cmd_q->ccp->vdata->version < CCP_VERSION(5, 0)) { 1863 /* Copy the exponent to the local storage block, using 1864 * as many 32-byte blocks as were allocated above. It's 1865 * already little endian, so no further change is required. 1866 */ 1867 ret = ccp_copy_to_sb(cmd_q, &exp, op.jobid, op.sb_key, 1868 CCP_PASSTHRU_BYTESWAP_NOOP); 1869 if (ret) { 1870 cmd->engine_error = cmd_q->cmd_error; 1871 goto e_exp; 1872 } 1873 } else { 1874 /* The exponent can be retrieved from memory via DMA. */ 1875 op.exp.u.dma.address = exp.dma.address; 1876 op.exp.u.dma.offset = 0; 1877 } 1878 1879 /* Concatenate the modulus and the message. Both the modulus and 1880 * the operands must be in little endian format. Since the input 1881 * is in big endian format it must be converted. 1882 */ 1883 ret = ccp_init_dm_workarea(&src, cmd_q, i_len, DMA_TO_DEVICE); 1884 if (ret) 1885 goto e_exp; 1886 1887 ret = ccp_reverse_set_dm_area(&src, 0, rsa->mod, 0, rsa->mod_len); 1888 if (ret) 1889 goto e_src; 1890 ret = ccp_reverse_set_dm_area(&src, o_len, rsa->src, 0, rsa->src_len); 1891 if (ret) 1892 goto e_src; 1893 1894 /* Prepare the output area for the operation */ 1895 ret = ccp_init_dm_workarea(&dst, cmd_q, o_len, DMA_FROM_DEVICE); 1896 if (ret) 1897 goto e_src; 1898 1899 op.soc = 1; 1900 op.src.u.dma.address = src.dma.address; 1901 op.src.u.dma.offset = 0; 1902 op.src.u.dma.length = i_len; 1903 op.dst.u.dma.address = dst.dma.address; 1904 op.dst.u.dma.offset = 0; 1905 op.dst.u.dma.length = o_len; 1906 1907 op.u.rsa.mod_size = rsa->key_size; 1908 op.u.rsa.input_len = i_len; 1909 1910 ret = cmd_q->ccp->vdata->perform->rsa(&op); 1911 if (ret) { 1912 cmd->engine_error = cmd_q->cmd_error; 1913 goto e_dst; 1914 } 1915 1916 ccp_reverse_get_dm_area(&dst, 0, rsa->dst, 0, rsa->mod_len); 1917 1918 e_dst: 1919 ccp_dm_free(&dst); 1920 1921 e_src: 1922 ccp_dm_free(&src); 1923 1924 e_exp: 1925 ccp_dm_free(&exp); 1926 1927 e_sb: 1928 if (sb_count) 1929 cmd_q->ccp->vdata->perform->sbfree(cmd_q, op.sb_key, sb_count); 1930 1931 return ret; 1932 } 1933 1934 static int ccp_run_passthru_cmd(struct ccp_cmd_queue *cmd_q, 1935 struct ccp_cmd *cmd) 1936 { 1937 struct ccp_passthru_engine *pt = &cmd->u.passthru; 1938 struct ccp_dm_workarea mask; 1939 struct ccp_data src, dst; 1940 struct ccp_op op; 1941 bool in_place = false; 1942 unsigned int i; 1943 int ret = 0; 1944 1945 if (!pt->final && (pt->src_len & (CCP_PASSTHRU_BLOCKSIZE - 1))) 1946 return -EINVAL; 1947 1948 if (!pt->src || !pt->dst) 1949 return -EINVAL; 1950 1951 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) { 1952 if (pt->mask_len != CCP_PASSTHRU_MASKSIZE) 1953 return -EINVAL; 1954 if (!pt->mask) 1955 return -EINVAL; 1956 } 1957 1958 BUILD_BUG_ON(CCP_PASSTHRU_SB_COUNT != 1); 1959 1960 memset(&op, 0, sizeof(op)); 1961 op.cmd_q = cmd_q; 1962 op.jobid = CCP_NEW_JOBID(cmd_q->ccp); 1963 1964 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) { 1965 /* Load the mask */ 1966 op.sb_key = cmd_q->sb_key; 1967 1968 ret = ccp_init_dm_workarea(&mask, cmd_q, 1969 CCP_PASSTHRU_SB_COUNT * 1970 CCP_SB_BYTES, 1971 DMA_TO_DEVICE); 1972 if (ret) 1973 return ret; 1974 1975 ret = ccp_set_dm_area(&mask, 0, pt->mask, 0, pt->mask_len); 1976 if (ret) 1977 goto e_mask; 1978 ret = ccp_copy_to_sb(cmd_q, &mask, op.jobid, op.sb_key, 1979 CCP_PASSTHRU_BYTESWAP_NOOP); 1980 if (ret) { 1981 cmd->engine_error = cmd_q->cmd_error; 1982 goto e_mask; 1983 } 1984 } 1985 1986 /* Prepare the input and output data workareas. For in-place 1987 * operations we need to set the dma direction to BIDIRECTIONAL 1988 * and copy the src workarea to the dst workarea. 1989 */ 1990 if (sg_virt(pt->src) == sg_virt(pt->dst)) 1991 in_place = true; 1992 1993 ret = ccp_init_data(&src, cmd_q, pt->src, pt->src_len, 1994 CCP_PASSTHRU_MASKSIZE, 1995 in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE); 1996 if (ret) 1997 goto e_mask; 1998 1999 if (in_place) { 2000 dst = src; 2001 } else { 2002 ret = ccp_init_data(&dst, cmd_q, pt->dst, pt->src_len, 2003 CCP_PASSTHRU_MASKSIZE, DMA_FROM_DEVICE); 2004 if (ret) 2005 goto e_src; 2006 } 2007 2008 /* Send data to the CCP Passthru engine 2009 * Because the CCP engine works on a single source and destination 2010 * dma address at a time, each entry in the source scatterlist 2011 * (after the dma_map_sg call) must be less than or equal to the 2012 * (remaining) length in the destination scatterlist entry and the 2013 * length must be a multiple of CCP_PASSTHRU_BLOCKSIZE 2014 */ 2015 dst.sg_wa.sg_used = 0; 2016 for (i = 1; i <= src.sg_wa.dma_count; i++) { 2017 if (!dst.sg_wa.sg || 2018 (dst.sg_wa.sg->length < src.sg_wa.sg->length)) { 2019 ret = -EINVAL; 2020 goto e_dst; 2021 } 2022 2023 if (i == src.sg_wa.dma_count) { 2024 op.eom = 1; 2025 op.soc = 1; 2026 } 2027 2028 op.src.type = CCP_MEMTYPE_SYSTEM; 2029 op.src.u.dma.address = sg_dma_address(src.sg_wa.sg); 2030 op.src.u.dma.offset = 0; 2031 op.src.u.dma.length = sg_dma_len(src.sg_wa.sg); 2032 2033 op.dst.type = CCP_MEMTYPE_SYSTEM; 2034 op.dst.u.dma.address = sg_dma_address(dst.sg_wa.sg); 2035 op.dst.u.dma.offset = dst.sg_wa.sg_used; 2036 op.dst.u.dma.length = op.src.u.dma.length; 2037 2038 ret = cmd_q->ccp->vdata->perform->passthru(&op); 2039 if (ret) { 2040 cmd->engine_error = cmd_q->cmd_error; 2041 goto e_dst; 2042 } 2043 2044 dst.sg_wa.sg_used += src.sg_wa.sg->length; 2045 if (dst.sg_wa.sg_used == dst.sg_wa.sg->length) { 2046 dst.sg_wa.sg = sg_next(dst.sg_wa.sg); 2047 dst.sg_wa.sg_used = 0; 2048 } 2049 src.sg_wa.sg = sg_next(src.sg_wa.sg); 2050 } 2051 2052 e_dst: 2053 if (!in_place) 2054 ccp_free_data(&dst, cmd_q); 2055 2056 e_src: 2057 ccp_free_data(&src, cmd_q); 2058 2059 e_mask: 2060 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) 2061 ccp_dm_free(&mask); 2062 2063 return ret; 2064 } 2065 2066 static int ccp_run_passthru_nomap_cmd(struct ccp_cmd_queue *cmd_q, 2067 struct ccp_cmd *cmd) 2068 { 2069 struct ccp_passthru_nomap_engine *pt = &cmd->u.passthru_nomap; 2070 struct ccp_dm_workarea mask; 2071 struct ccp_op op; 2072 int ret; 2073 2074 if (!pt->final && (pt->src_len & (CCP_PASSTHRU_BLOCKSIZE - 1))) 2075 return -EINVAL; 2076 2077 if (!pt->src_dma || !pt->dst_dma) 2078 return -EINVAL; 2079 2080 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) { 2081 if (pt->mask_len != CCP_PASSTHRU_MASKSIZE) 2082 return -EINVAL; 2083 if (!pt->mask) 2084 return -EINVAL; 2085 } 2086 2087 BUILD_BUG_ON(CCP_PASSTHRU_SB_COUNT != 1); 2088 2089 memset(&op, 0, sizeof(op)); 2090 op.cmd_q = cmd_q; 2091 op.jobid = CCP_NEW_JOBID(cmd_q->ccp); 2092 2093 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) { 2094 /* Load the mask */ 2095 op.sb_key = cmd_q->sb_key; 2096 2097 mask.length = pt->mask_len; 2098 mask.dma.address = pt->mask; 2099 mask.dma.length = pt->mask_len; 2100 2101 ret = ccp_copy_to_sb(cmd_q, &mask, op.jobid, op.sb_key, 2102 CCP_PASSTHRU_BYTESWAP_NOOP); 2103 if (ret) { 2104 cmd->engine_error = cmd_q->cmd_error; 2105 return ret; 2106 } 2107 } 2108 2109 /* Send data to the CCP Passthru engine */ 2110 op.eom = 1; 2111 op.soc = 1; 2112 2113 op.src.type = CCP_MEMTYPE_SYSTEM; 2114 op.src.u.dma.address = pt->src_dma; 2115 op.src.u.dma.offset = 0; 2116 op.src.u.dma.length = pt->src_len; 2117 2118 op.dst.type = CCP_MEMTYPE_SYSTEM; 2119 op.dst.u.dma.address = pt->dst_dma; 2120 op.dst.u.dma.offset = 0; 2121 op.dst.u.dma.length = pt->src_len; 2122 2123 ret = cmd_q->ccp->vdata->perform->passthru(&op); 2124 if (ret) 2125 cmd->engine_error = cmd_q->cmd_error; 2126 2127 return ret; 2128 } 2129 2130 static int ccp_run_ecc_mm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) 2131 { 2132 struct ccp_ecc_engine *ecc = &cmd->u.ecc; 2133 struct ccp_dm_workarea src, dst; 2134 struct ccp_op op; 2135 int ret; 2136 u8 *save; 2137 2138 if (!ecc->u.mm.operand_1 || 2139 (ecc->u.mm.operand_1_len > CCP_ECC_MODULUS_BYTES)) 2140 return -EINVAL; 2141 2142 if (ecc->function != CCP_ECC_FUNCTION_MINV_384BIT) 2143 if (!ecc->u.mm.operand_2 || 2144 (ecc->u.mm.operand_2_len > CCP_ECC_MODULUS_BYTES)) 2145 return -EINVAL; 2146 2147 if (!ecc->u.mm.result || 2148 (ecc->u.mm.result_len < CCP_ECC_MODULUS_BYTES)) 2149 return -EINVAL; 2150 2151 memset(&op, 0, sizeof(op)); 2152 op.cmd_q = cmd_q; 2153 op.jobid = CCP_NEW_JOBID(cmd_q->ccp); 2154 2155 /* Concatenate the modulus and the operands. Both the modulus and 2156 * the operands must be in little endian format. Since the input 2157 * is in big endian format it must be converted and placed in a 2158 * fixed length buffer. 2159 */ 2160 ret = ccp_init_dm_workarea(&src, cmd_q, CCP_ECC_SRC_BUF_SIZE, 2161 DMA_TO_DEVICE); 2162 if (ret) 2163 return ret; 2164 2165 /* Save the workarea address since it is updated in order to perform 2166 * the concatenation 2167 */ 2168 save = src.address; 2169 2170 /* Copy the ECC modulus */ 2171 ret = ccp_reverse_set_dm_area(&src, 0, ecc->mod, 0, ecc->mod_len); 2172 if (ret) 2173 goto e_src; 2174 src.address += CCP_ECC_OPERAND_SIZE; 2175 2176 /* Copy the first operand */ 2177 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.mm.operand_1, 0, 2178 ecc->u.mm.operand_1_len); 2179 if (ret) 2180 goto e_src; 2181 src.address += CCP_ECC_OPERAND_SIZE; 2182 2183 if (ecc->function != CCP_ECC_FUNCTION_MINV_384BIT) { 2184 /* Copy the second operand */ 2185 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.mm.operand_2, 0, 2186 ecc->u.mm.operand_2_len); 2187 if (ret) 2188 goto e_src; 2189 src.address += CCP_ECC_OPERAND_SIZE; 2190 } 2191 2192 /* Restore the workarea address */ 2193 src.address = save; 2194 2195 /* Prepare the output area for the operation */ 2196 ret = ccp_init_dm_workarea(&dst, cmd_q, CCP_ECC_DST_BUF_SIZE, 2197 DMA_FROM_DEVICE); 2198 if (ret) 2199 goto e_src; 2200 2201 op.soc = 1; 2202 op.src.u.dma.address = src.dma.address; 2203 op.src.u.dma.offset = 0; 2204 op.src.u.dma.length = src.length; 2205 op.dst.u.dma.address = dst.dma.address; 2206 op.dst.u.dma.offset = 0; 2207 op.dst.u.dma.length = dst.length; 2208 2209 op.u.ecc.function = cmd->u.ecc.function; 2210 2211 ret = cmd_q->ccp->vdata->perform->ecc(&op); 2212 if (ret) { 2213 cmd->engine_error = cmd_q->cmd_error; 2214 goto e_dst; 2215 } 2216 2217 ecc->ecc_result = le16_to_cpup( 2218 (const __le16 *)(dst.address + CCP_ECC_RESULT_OFFSET)); 2219 if (!(ecc->ecc_result & CCP_ECC_RESULT_SUCCESS)) { 2220 ret = -EIO; 2221 goto e_dst; 2222 } 2223 2224 /* Save the ECC result */ 2225 ccp_reverse_get_dm_area(&dst, 0, ecc->u.mm.result, 0, 2226 CCP_ECC_MODULUS_BYTES); 2227 2228 e_dst: 2229 ccp_dm_free(&dst); 2230 2231 e_src: 2232 ccp_dm_free(&src); 2233 2234 return ret; 2235 } 2236 2237 static int ccp_run_ecc_pm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) 2238 { 2239 struct ccp_ecc_engine *ecc = &cmd->u.ecc; 2240 struct ccp_dm_workarea src, dst; 2241 struct ccp_op op; 2242 int ret; 2243 u8 *save; 2244 2245 if (!ecc->u.pm.point_1.x || 2246 (ecc->u.pm.point_1.x_len > CCP_ECC_MODULUS_BYTES) || 2247 !ecc->u.pm.point_1.y || 2248 (ecc->u.pm.point_1.y_len > CCP_ECC_MODULUS_BYTES)) 2249 return -EINVAL; 2250 2251 if (ecc->function == CCP_ECC_FUNCTION_PADD_384BIT) { 2252 if (!ecc->u.pm.point_2.x || 2253 (ecc->u.pm.point_2.x_len > CCP_ECC_MODULUS_BYTES) || 2254 !ecc->u.pm.point_2.y || 2255 (ecc->u.pm.point_2.y_len > CCP_ECC_MODULUS_BYTES)) 2256 return -EINVAL; 2257 } else { 2258 if (!ecc->u.pm.domain_a || 2259 (ecc->u.pm.domain_a_len > CCP_ECC_MODULUS_BYTES)) 2260 return -EINVAL; 2261 2262 if (ecc->function == CCP_ECC_FUNCTION_PMUL_384BIT) 2263 if (!ecc->u.pm.scalar || 2264 (ecc->u.pm.scalar_len > CCP_ECC_MODULUS_BYTES)) 2265 return -EINVAL; 2266 } 2267 2268 if (!ecc->u.pm.result.x || 2269 (ecc->u.pm.result.x_len < CCP_ECC_MODULUS_BYTES) || 2270 !ecc->u.pm.result.y || 2271 (ecc->u.pm.result.y_len < CCP_ECC_MODULUS_BYTES)) 2272 return -EINVAL; 2273 2274 memset(&op, 0, sizeof(op)); 2275 op.cmd_q = cmd_q; 2276 op.jobid = CCP_NEW_JOBID(cmd_q->ccp); 2277 2278 /* Concatenate the modulus and the operands. Both the modulus and 2279 * the operands must be in little endian format. Since the input 2280 * is in big endian format it must be converted and placed in a 2281 * fixed length buffer. 2282 */ 2283 ret = ccp_init_dm_workarea(&src, cmd_q, CCP_ECC_SRC_BUF_SIZE, 2284 DMA_TO_DEVICE); 2285 if (ret) 2286 return ret; 2287 2288 /* Save the workarea address since it is updated in order to perform 2289 * the concatenation 2290 */ 2291 save = src.address; 2292 2293 /* Copy the ECC modulus */ 2294 ret = ccp_reverse_set_dm_area(&src, 0, ecc->mod, 0, ecc->mod_len); 2295 if (ret) 2296 goto e_src; 2297 src.address += CCP_ECC_OPERAND_SIZE; 2298 2299 /* Copy the first point X and Y coordinate */ 2300 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_1.x, 0, 2301 ecc->u.pm.point_1.x_len); 2302 if (ret) 2303 goto e_src; 2304 src.address += CCP_ECC_OPERAND_SIZE; 2305 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_1.y, 0, 2306 ecc->u.pm.point_1.y_len); 2307 if (ret) 2308 goto e_src; 2309 src.address += CCP_ECC_OPERAND_SIZE; 2310 2311 /* Set the first point Z coordinate to 1 */ 2312 *src.address = 0x01; 2313 src.address += CCP_ECC_OPERAND_SIZE; 2314 2315 if (ecc->function == CCP_ECC_FUNCTION_PADD_384BIT) { 2316 /* Copy the second point X and Y coordinate */ 2317 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_2.x, 0, 2318 ecc->u.pm.point_2.x_len); 2319 if (ret) 2320 goto e_src; 2321 src.address += CCP_ECC_OPERAND_SIZE; 2322 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_2.y, 0, 2323 ecc->u.pm.point_2.y_len); 2324 if (ret) 2325 goto e_src; 2326 src.address += CCP_ECC_OPERAND_SIZE; 2327 2328 /* Set the second point Z coordinate to 1 */ 2329 *src.address = 0x01; 2330 src.address += CCP_ECC_OPERAND_SIZE; 2331 } else { 2332 /* Copy the Domain "a" parameter */ 2333 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.domain_a, 0, 2334 ecc->u.pm.domain_a_len); 2335 if (ret) 2336 goto e_src; 2337 src.address += CCP_ECC_OPERAND_SIZE; 2338 2339 if (ecc->function == CCP_ECC_FUNCTION_PMUL_384BIT) { 2340 /* Copy the scalar value */ 2341 ret = ccp_reverse_set_dm_area(&src, 0, 2342 ecc->u.pm.scalar, 0, 2343 ecc->u.pm.scalar_len); 2344 if (ret) 2345 goto e_src; 2346 src.address += CCP_ECC_OPERAND_SIZE; 2347 } 2348 } 2349 2350 /* Restore the workarea address */ 2351 src.address = save; 2352 2353 /* Prepare the output area for the operation */ 2354 ret = ccp_init_dm_workarea(&dst, cmd_q, CCP_ECC_DST_BUF_SIZE, 2355 DMA_FROM_DEVICE); 2356 if (ret) 2357 goto e_src; 2358 2359 op.soc = 1; 2360 op.src.u.dma.address = src.dma.address; 2361 op.src.u.dma.offset = 0; 2362 op.src.u.dma.length = src.length; 2363 op.dst.u.dma.address = dst.dma.address; 2364 op.dst.u.dma.offset = 0; 2365 op.dst.u.dma.length = dst.length; 2366 2367 op.u.ecc.function = cmd->u.ecc.function; 2368 2369 ret = cmd_q->ccp->vdata->perform->ecc(&op); 2370 if (ret) { 2371 cmd->engine_error = cmd_q->cmd_error; 2372 goto e_dst; 2373 } 2374 2375 ecc->ecc_result = le16_to_cpup( 2376 (const __le16 *)(dst.address + CCP_ECC_RESULT_OFFSET)); 2377 if (!(ecc->ecc_result & CCP_ECC_RESULT_SUCCESS)) { 2378 ret = -EIO; 2379 goto e_dst; 2380 } 2381 2382 /* Save the workarea address since it is updated as we walk through 2383 * to copy the point math result 2384 */ 2385 save = dst.address; 2386 2387 /* Save the ECC result X and Y coordinates */ 2388 ccp_reverse_get_dm_area(&dst, 0, ecc->u.pm.result.x, 0, 2389 CCP_ECC_MODULUS_BYTES); 2390 dst.address += CCP_ECC_OUTPUT_SIZE; 2391 ccp_reverse_get_dm_area(&dst, 0, ecc->u.pm.result.y, 0, 2392 CCP_ECC_MODULUS_BYTES); 2393 dst.address += CCP_ECC_OUTPUT_SIZE; 2394 2395 /* Restore the workarea address */ 2396 dst.address = save; 2397 2398 e_dst: 2399 ccp_dm_free(&dst); 2400 2401 e_src: 2402 ccp_dm_free(&src); 2403 2404 return ret; 2405 } 2406 2407 static int ccp_run_ecc_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) 2408 { 2409 struct ccp_ecc_engine *ecc = &cmd->u.ecc; 2410 2411 ecc->ecc_result = 0; 2412 2413 if (!ecc->mod || 2414 (ecc->mod_len > CCP_ECC_MODULUS_BYTES)) 2415 return -EINVAL; 2416 2417 switch (ecc->function) { 2418 case CCP_ECC_FUNCTION_MMUL_384BIT: 2419 case CCP_ECC_FUNCTION_MADD_384BIT: 2420 case CCP_ECC_FUNCTION_MINV_384BIT: 2421 return ccp_run_ecc_mm_cmd(cmd_q, cmd); 2422 2423 case CCP_ECC_FUNCTION_PADD_384BIT: 2424 case CCP_ECC_FUNCTION_PMUL_384BIT: 2425 case CCP_ECC_FUNCTION_PDBL_384BIT: 2426 return ccp_run_ecc_pm_cmd(cmd_q, cmd); 2427 2428 default: 2429 return -EINVAL; 2430 } 2431 } 2432 2433 int ccp_run_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) 2434 { 2435 int ret; 2436 2437 cmd->engine_error = 0; 2438 cmd_q->cmd_error = 0; 2439 cmd_q->int_rcvd = 0; 2440 cmd_q->free_slots = cmd_q->ccp->vdata->perform->get_free_slots(cmd_q); 2441 2442 switch (cmd->engine) { 2443 case CCP_ENGINE_AES: 2444 ret = ccp_run_aes_cmd(cmd_q, cmd); 2445 break; 2446 case CCP_ENGINE_XTS_AES_128: 2447 ret = ccp_run_xts_aes_cmd(cmd_q, cmd); 2448 break; 2449 case CCP_ENGINE_DES3: 2450 ret = ccp_run_des3_cmd(cmd_q, cmd); 2451 break; 2452 case CCP_ENGINE_SHA: 2453 ret = ccp_run_sha_cmd(cmd_q, cmd); 2454 break; 2455 case CCP_ENGINE_RSA: 2456 ret = ccp_run_rsa_cmd(cmd_q, cmd); 2457 break; 2458 case CCP_ENGINE_PASSTHRU: 2459 if (cmd->flags & CCP_CMD_PASSTHRU_NO_DMA_MAP) 2460 ret = ccp_run_passthru_nomap_cmd(cmd_q, cmd); 2461 else 2462 ret = ccp_run_passthru_cmd(cmd_q, cmd); 2463 break; 2464 case CCP_ENGINE_ECC: 2465 ret = ccp_run_ecc_cmd(cmd_q, cmd); 2466 break; 2467 default: 2468 ret = -EINVAL; 2469 } 2470 2471 return ret; 2472 } 2473