1 /* 2 * AMD Cryptographic Coprocessor (CCP) driver 3 * 4 * Copyright (C) 2013,2017 Advanced Micro Devices, Inc. 5 * 6 * Author: Tom Lendacky <thomas.lendacky@amd.com> 7 * Author: Gary R Hook <gary.hook@amd.com> 8 * 9 * This program is free software; you can redistribute it and/or modify 10 * it under the terms of the GNU General Public License version 2 as 11 * published by the Free Software Foundation. 12 */ 13 14 #include <linux/module.h> 15 #include <linux/kernel.h> 16 #include <linux/pci.h> 17 #include <linux/interrupt.h> 18 #include <crypto/scatterwalk.h> 19 #include <crypto/des.h> 20 #include <linux/ccp.h> 21 22 #include "ccp-dev.h" 23 24 /* SHA initial context values */ 25 static const __be32 ccp_sha1_init[SHA1_DIGEST_SIZE / sizeof(__be32)] = { 26 cpu_to_be32(SHA1_H0), cpu_to_be32(SHA1_H1), 27 cpu_to_be32(SHA1_H2), cpu_to_be32(SHA1_H3), 28 cpu_to_be32(SHA1_H4), 29 }; 30 31 static const __be32 ccp_sha224_init[SHA256_DIGEST_SIZE / sizeof(__be32)] = { 32 cpu_to_be32(SHA224_H0), cpu_to_be32(SHA224_H1), 33 cpu_to_be32(SHA224_H2), cpu_to_be32(SHA224_H3), 34 cpu_to_be32(SHA224_H4), cpu_to_be32(SHA224_H5), 35 cpu_to_be32(SHA224_H6), cpu_to_be32(SHA224_H7), 36 }; 37 38 static const __be32 ccp_sha256_init[SHA256_DIGEST_SIZE / sizeof(__be32)] = { 39 cpu_to_be32(SHA256_H0), cpu_to_be32(SHA256_H1), 40 cpu_to_be32(SHA256_H2), cpu_to_be32(SHA256_H3), 41 cpu_to_be32(SHA256_H4), cpu_to_be32(SHA256_H5), 42 cpu_to_be32(SHA256_H6), cpu_to_be32(SHA256_H7), 43 }; 44 45 static const __be64 ccp_sha384_init[SHA512_DIGEST_SIZE / sizeof(__be64)] = { 46 cpu_to_be64(SHA384_H0), cpu_to_be64(SHA384_H1), 47 cpu_to_be64(SHA384_H2), cpu_to_be64(SHA384_H3), 48 cpu_to_be64(SHA384_H4), cpu_to_be64(SHA384_H5), 49 cpu_to_be64(SHA384_H6), cpu_to_be64(SHA384_H7), 50 }; 51 52 static const __be64 ccp_sha512_init[SHA512_DIGEST_SIZE / sizeof(__be64)] = { 53 cpu_to_be64(SHA512_H0), cpu_to_be64(SHA512_H1), 54 cpu_to_be64(SHA512_H2), cpu_to_be64(SHA512_H3), 55 cpu_to_be64(SHA512_H4), cpu_to_be64(SHA512_H5), 56 cpu_to_be64(SHA512_H6), cpu_to_be64(SHA512_H7), 57 }; 58 59 #define CCP_NEW_JOBID(ccp) ((ccp->vdata->version == CCP_VERSION(3, 0)) ? \ 60 ccp_gen_jobid(ccp) : 0) 61 62 static u32 ccp_gen_jobid(struct ccp_device *ccp) 63 { 64 return atomic_inc_return(&ccp->current_id) & CCP_JOBID_MASK; 65 } 66 67 static void ccp_sg_free(struct ccp_sg_workarea *wa) 68 { 69 if (wa->dma_count) 70 dma_unmap_sg(wa->dma_dev, wa->dma_sg, wa->nents, wa->dma_dir); 71 72 wa->dma_count = 0; 73 } 74 75 static int ccp_init_sg_workarea(struct ccp_sg_workarea *wa, struct device *dev, 76 struct scatterlist *sg, u64 len, 77 enum dma_data_direction dma_dir) 78 { 79 memset(wa, 0, sizeof(*wa)); 80 81 wa->sg = sg; 82 if (!sg) 83 return 0; 84 85 wa->nents = sg_nents_for_len(sg, len); 86 if (wa->nents < 0) 87 return wa->nents; 88 89 wa->bytes_left = len; 90 wa->sg_used = 0; 91 92 if (len == 0) 93 return 0; 94 95 if (dma_dir == DMA_NONE) 96 return 0; 97 98 wa->dma_sg = sg; 99 wa->dma_dev = dev; 100 wa->dma_dir = dma_dir; 101 wa->dma_count = dma_map_sg(dev, sg, wa->nents, dma_dir); 102 if (!wa->dma_count) 103 return -ENOMEM; 104 105 return 0; 106 } 107 108 static void ccp_update_sg_workarea(struct ccp_sg_workarea *wa, unsigned int len) 109 { 110 unsigned int nbytes = min_t(u64, len, wa->bytes_left); 111 112 if (!wa->sg) 113 return; 114 115 wa->sg_used += nbytes; 116 wa->bytes_left -= nbytes; 117 if (wa->sg_used == wa->sg->length) { 118 wa->sg = sg_next(wa->sg); 119 wa->sg_used = 0; 120 } 121 } 122 123 static void ccp_dm_free(struct ccp_dm_workarea *wa) 124 { 125 if (wa->length <= CCP_DMAPOOL_MAX_SIZE) { 126 if (wa->address) 127 dma_pool_free(wa->dma_pool, wa->address, 128 wa->dma.address); 129 } else { 130 if (wa->dma.address) 131 dma_unmap_single(wa->dev, wa->dma.address, wa->length, 132 wa->dma.dir); 133 kfree(wa->address); 134 } 135 136 wa->address = NULL; 137 wa->dma.address = 0; 138 } 139 140 static int ccp_init_dm_workarea(struct ccp_dm_workarea *wa, 141 struct ccp_cmd_queue *cmd_q, 142 unsigned int len, 143 enum dma_data_direction dir) 144 { 145 memset(wa, 0, sizeof(*wa)); 146 147 if (!len) 148 return 0; 149 150 wa->dev = cmd_q->ccp->dev; 151 wa->length = len; 152 153 if (len <= CCP_DMAPOOL_MAX_SIZE) { 154 wa->dma_pool = cmd_q->dma_pool; 155 156 wa->address = dma_pool_alloc(wa->dma_pool, GFP_KERNEL, 157 &wa->dma.address); 158 if (!wa->address) 159 return -ENOMEM; 160 161 wa->dma.length = CCP_DMAPOOL_MAX_SIZE; 162 163 memset(wa->address, 0, CCP_DMAPOOL_MAX_SIZE); 164 } else { 165 wa->address = kzalloc(len, GFP_KERNEL); 166 if (!wa->address) 167 return -ENOMEM; 168 169 wa->dma.address = dma_map_single(wa->dev, wa->address, len, 170 dir); 171 if (!wa->dma.address) 172 return -ENOMEM; 173 174 wa->dma.length = len; 175 } 176 wa->dma.dir = dir; 177 178 return 0; 179 } 180 181 static void ccp_set_dm_area(struct ccp_dm_workarea *wa, unsigned int wa_offset, 182 struct scatterlist *sg, unsigned int sg_offset, 183 unsigned int len) 184 { 185 WARN_ON(!wa->address); 186 187 scatterwalk_map_and_copy(wa->address + wa_offset, sg, sg_offset, len, 188 0); 189 } 190 191 static void ccp_get_dm_area(struct ccp_dm_workarea *wa, unsigned int wa_offset, 192 struct scatterlist *sg, unsigned int sg_offset, 193 unsigned int len) 194 { 195 WARN_ON(!wa->address); 196 197 scatterwalk_map_and_copy(wa->address + wa_offset, sg, sg_offset, len, 198 1); 199 } 200 201 static int ccp_reverse_set_dm_area(struct ccp_dm_workarea *wa, 202 unsigned int wa_offset, 203 struct scatterlist *sg, 204 unsigned int sg_offset, 205 unsigned int len) 206 { 207 u8 *p, *q; 208 209 ccp_set_dm_area(wa, wa_offset, sg, sg_offset, len); 210 211 p = wa->address + wa_offset; 212 q = p + len - 1; 213 while (p < q) { 214 *p = *p ^ *q; 215 *q = *p ^ *q; 216 *p = *p ^ *q; 217 p++; 218 q--; 219 } 220 return 0; 221 } 222 223 static void ccp_reverse_get_dm_area(struct ccp_dm_workarea *wa, 224 unsigned int wa_offset, 225 struct scatterlist *sg, 226 unsigned int sg_offset, 227 unsigned int len) 228 { 229 u8 *p, *q; 230 231 p = wa->address + wa_offset; 232 q = p + len - 1; 233 while (p < q) { 234 *p = *p ^ *q; 235 *q = *p ^ *q; 236 *p = *p ^ *q; 237 p++; 238 q--; 239 } 240 241 ccp_get_dm_area(wa, wa_offset, sg, sg_offset, len); 242 } 243 244 static void ccp_free_data(struct ccp_data *data, struct ccp_cmd_queue *cmd_q) 245 { 246 ccp_dm_free(&data->dm_wa); 247 ccp_sg_free(&data->sg_wa); 248 } 249 250 static int ccp_init_data(struct ccp_data *data, struct ccp_cmd_queue *cmd_q, 251 struct scatterlist *sg, u64 sg_len, 252 unsigned int dm_len, 253 enum dma_data_direction dir) 254 { 255 int ret; 256 257 memset(data, 0, sizeof(*data)); 258 259 ret = ccp_init_sg_workarea(&data->sg_wa, cmd_q->ccp->dev, sg, sg_len, 260 dir); 261 if (ret) 262 goto e_err; 263 264 ret = ccp_init_dm_workarea(&data->dm_wa, cmd_q, dm_len, dir); 265 if (ret) 266 goto e_err; 267 268 return 0; 269 270 e_err: 271 ccp_free_data(data, cmd_q); 272 273 return ret; 274 } 275 276 static unsigned int ccp_queue_buf(struct ccp_data *data, unsigned int from) 277 { 278 struct ccp_sg_workarea *sg_wa = &data->sg_wa; 279 struct ccp_dm_workarea *dm_wa = &data->dm_wa; 280 unsigned int buf_count, nbytes; 281 282 /* Clear the buffer if setting it */ 283 if (!from) 284 memset(dm_wa->address, 0, dm_wa->length); 285 286 if (!sg_wa->sg) 287 return 0; 288 289 /* Perform the copy operation 290 * nbytes will always be <= UINT_MAX because dm_wa->length is 291 * an unsigned int 292 */ 293 nbytes = min_t(u64, sg_wa->bytes_left, dm_wa->length); 294 scatterwalk_map_and_copy(dm_wa->address, sg_wa->sg, sg_wa->sg_used, 295 nbytes, from); 296 297 /* Update the structures and generate the count */ 298 buf_count = 0; 299 while (sg_wa->bytes_left && (buf_count < dm_wa->length)) { 300 nbytes = min(sg_wa->sg->length - sg_wa->sg_used, 301 dm_wa->length - buf_count); 302 nbytes = min_t(u64, sg_wa->bytes_left, nbytes); 303 304 buf_count += nbytes; 305 ccp_update_sg_workarea(sg_wa, nbytes); 306 } 307 308 return buf_count; 309 } 310 311 static unsigned int ccp_fill_queue_buf(struct ccp_data *data) 312 { 313 return ccp_queue_buf(data, 0); 314 } 315 316 static unsigned int ccp_empty_queue_buf(struct ccp_data *data) 317 { 318 return ccp_queue_buf(data, 1); 319 } 320 321 static void ccp_prepare_data(struct ccp_data *src, struct ccp_data *dst, 322 struct ccp_op *op, unsigned int block_size, 323 bool blocksize_op) 324 { 325 unsigned int sg_src_len, sg_dst_len, op_len; 326 327 /* The CCP can only DMA from/to one address each per operation. This 328 * requires that we find the smallest DMA area between the source 329 * and destination. The resulting len values will always be <= UINT_MAX 330 * because the dma length is an unsigned int. 331 */ 332 sg_src_len = sg_dma_len(src->sg_wa.sg) - src->sg_wa.sg_used; 333 sg_src_len = min_t(u64, src->sg_wa.bytes_left, sg_src_len); 334 335 if (dst) { 336 sg_dst_len = sg_dma_len(dst->sg_wa.sg) - dst->sg_wa.sg_used; 337 sg_dst_len = min_t(u64, src->sg_wa.bytes_left, sg_dst_len); 338 op_len = min(sg_src_len, sg_dst_len); 339 } else { 340 op_len = sg_src_len; 341 } 342 343 /* The data operation length will be at least block_size in length 344 * or the smaller of available sg room remaining for the source or 345 * the destination 346 */ 347 op_len = max(op_len, block_size); 348 349 /* Unless we have to buffer data, there's no reason to wait */ 350 op->soc = 0; 351 352 if (sg_src_len < block_size) { 353 /* Not enough data in the sg element, so it 354 * needs to be buffered into a blocksize chunk 355 */ 356 int cp_len = ccp_fill_queue_buf(src); 357 358 op->soc = 1; 359 op->src.u.dma.address = src->dm_wa.dma.address; 360 op->src.u.dma.offset = 0; 361 op->src.u.dma.length = (blocksize_op) ? block_size : cp_len; 362 } else { 363 /* Enough data in the sg element, but we need to 364 * adjust for any previously copied data 365 */ 366 op->src.u.dma.address = sg_dma_address(src->sg_wa.sg); 367 op->src.u.dma.offset = src->sg_wa.sg_used; 368 op->src.u.dma.length = op_len & ~(block_size - 1); 369 370 ccp_update_sg_workarea(&src->sg_wa, op->src.u.dma.length); 371 } 372 373 if (dst) { 374 if (sg_dst_len < block_size) { 375 /* Not enough room in the sg element or we're on the 376 * last piece of data (when using padding), so the 377 * output needs to be buffered into a blocksize chunk 378 */ 379 op->soc = 1; 380 op->dst.u.dma.address = dst->dm_wa.dma.address; 381 op->dst.u.dma.offset = 0; 382 op->dst.u.dma.length = op->src.u.dma.length; 383 } else { 384 /* Enough room in the sg element, but we need to 385 * adjust for any previously used area 386 */ 387 op->dst.u.dma.address = sg_dma_address(dst->sg_wa.sg); 388 op->dst.u.dma.offset = dst->sg_wa.sg_used; 389 op->dst.u.dma.length = op->src.u.dma.length; 390 } 391 } 392 } 393 394 static void ccp_process_data(struct ccp_data *src, struct ccp_data *dst, 395 struct ccp_op *op) 396 { 397 op->init = 0; 398 399 if (dst) { 400 if (op->dst.u.dma.address == dst->dm_wa.dma.address) 401 ccp_empty_queue_buf(dst); 402 else 403 ccp_update_sg_workarea(&dst->sg_wa, 404 op->dst.u.dma.length); 405 } 406 } 407 408 static int ccp_copy_to_from_sb(struct ccp_cmd_queue *cmd_q, 409 struct ccp_dm_workarea *wa, u32 jobid, u32 sb, 410 u32 byte_swap, bool from) 411 { 412 struct ccp_op op; 413 414 memset(&op, 0, sizeof(op)); 415 416 op.cmd_q = cmd_q; 417 op.jobid = jobid; 418 op.eom = 1; 419 420 if (from) { 421 op.soc = 1; 422 op.src.type = CCP_MEMTYPE_SB; 423 op.src.u.sb = sb; 424 op.dst.type = CCP_MEMTYPE_SYSTEM; 425 op.dst.u.dma.address = wa->dma.address; 426 op.dst.u.dma.length = wa->length; 427 } else { 428 op.src.type = CCP_MEMTYPE_SYSTEM; 429 op.src.u.dma.address = wa->dma.address; 430 op.src.u.dma.length = wa->length; 431 op.dst.type = CCP_MEMTYPE_SB; 432 op.dst.u.sb = sb; 433 } 434 435 op.u.passthru.byte_swap = byte_swap; 436 437 return cmd_q->ccp->vdata->perform->passthru(&op); 438 } 439 440 static int ccp_copy_to_sb(struct ccp_cmd_queue *cmd_q, 441 struct ccp_dm_workarea *wa, u32 jobid, u32 sb, 442 u32 byte_swap) 443 { 444 return ccp_copy_to_from_sb(cmd_q, wa, jobid, sb, byte_swap, false); 445 } 446 447 static int ccp_copy_from_sb(struct ccp_cmd_queue *cmd_q, 448 struct ccp_dm_workarea *wa, u32 jobid, u32 sb, 449 u32 byte_swap) 450 { 451 return ccp_copy_to_from_sb(cmd_q, wa, jobid, sb, byte_swap, true); 452 } 453 454 static int ccp_run_aes_cmac_cmd(struct ccp_cmd_queue *cmd_q, 455 struct ccp_cmd *cmd) 456 { 457 struct ccp_aes_engine *aes = &cmd->u.aes; 458 struct ccp_dm_workarea key, ctx; 459 struct ccp_data src; 460 struct ccp_op op; 461 unsigned int dm_offset; 462 int ret; 463 464 if (!((aes->key_len == AES_KEYSIZE_128) || 465 (aes->key_len == AES_KEYSIZE_192) || 466 (aes->key_len == AES_KEYSIZE_256))) 467 return -EINVAL; 468 469 if (aes->src_len & (AES_BLOCK_SIZE - 1)) 470 return -EINVAL; 471 472 if (aes->iv_len != AES_BLOCK_SIZE) 473 return -EINVAL; 474 475 if (!aes->key || !aes->iv || !aes->src) 476 return -EINVAL; 477 478 if (aes->cmac_final) { 479 if (aes->cmac_key_len != AES_BLOCK_SIZE) 480 return -EINVAL; 481 482 if (!aes->cmac_key) 483 return -EINVAL; 484 } 485 486 BUILD_BUG_ON(CCP_AES_KEY_SB_COUNT != 1); 487 BUILD_BUG_ON(CCP_AES_CTX_SB_COUNT != 1); 488 489 ret = -EIO; 490 memset(&op, 0, sizeof(op)); 491 op.cmd_q = cmd_q; 492 op.jobid = CCP_NEW_JOBID(cmd_q->ccp); 493 op.sb_key = cmd_q->sb_key; 494 op.sb_ctx = cmd_q->sb_ctx; 495 op.init = 1; 496 op.u.aes.type = aes->type; 497 op.u.aes.mode = aes->mode; 498 op.u.aes.action = aes->action; 499 500 /* All supported key sizes fit in a single (32-byte) SB entry 501 * and must be in little endian format. Use the 256-bit byte 502 * swap passthru option to convert from big endian to little 503 * endian. 504 */ 505 ret = ccp_init_dm_workarea(&key, cmd_q, 506 CCP_AES_KEY_SB_COUNT * CCP_SB_BYTES, 507 DMA_TO_DEVICE); 508 if (ret) 509 return ret; 510 511 dm_offset = CCP_SB_BYTES - aes->key_len; 512 ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len); 513 ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key, 514 CCP_PASSTHRU_BYTESWAP_256BIT); 515 if (ret) { 516 cmd->engine_error = cmd_q->cmd_error; 517 goto e_key; 518 } 519 520 /* The AES context fits in a single (32-byte) SB entry and 521 * must be in little endian format. Use the 256-bit byte swap 522 * passthru option to convert from big endian to little endian. 523 */ 524 ret = ccp_init_dm_workarea(&ctx, cmd_q, 525 CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES, 526 DMA_BIDIRECTIONAL); 527 if (ret) 528 goto e_key; 529 530 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE; 531 ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len); 532 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, 533 CCP_PASSTHRU_BYTESWAP_256BIT); 534 if (ret) { 535 cmd->engine_error = cmd_q->cmd_error; 536 goto e_ctx; 537 } 538 539 /* Send data to the CCP AES engine */ 540 ret = ccp_init_data(&src, cmd_q, aes->src, aes->src_len, 541 AES_BLOCK_SIZE, DMA_TO_DEVICE); 542 if (ret) 543 goto e_ctx; 544 545 while (src.sg_wa.bytes_left) { 546 ccp_prepare_data(&src, NULL, &op, AES_BLOCK_SIZE, true); 547 if (aes->cmac_final && !src.sg_wa.bytes_left) { 548 op.eom = 1; 549 550 /* Push the K1/K2 key to the CCP now */ 551 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, 552 op.sb_ctx, 553 CCP_PASSTHRU_BYTESWAP_256BIT); 554 if (ret) { 555 cmd->engine_error = cmd_q->cmd_error; 556 goto e_src; 557 } 558 559 ccp_set_dm_area(&ctx, 0, aes->cmac_key, 0, 560 aes->cmac_key_len); 561 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, 562 CCP_PASSTHRU_BYTESWAP_256BIT); 563 if (ret) { 564 cmd->engine_error = cmd_q->cmd_error; 565 goto e_src; 566 } 567 } 568 569 ret = cmd_q->ccp->vdata->perform->aes(&op); 570 if (ret) { 571 cmd->engine_error = cmd_q->cmd_error; 572 goto e_src; 573 } 574 575 ccp_process_data(&src, NULL, &op); 576 } 577 578 /* Retrieve the AES context - convert from LE to BE using 579 * 32-byte (256-bit) byteswapping 580 */ 581 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, 582 CCP_PASSTHRU_BYTESWAP_256BIT); 583 if (ret) { 584 cmd->engine_error = cmd_q->cmd_error; 585 goto e_src; 586 } 587 588 /* ...but we only need AES_BLOCK_SIZE bytes */ 589 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE; 590 ccp_get_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len); 591 592 e_src: 593 ccp_free_data(&src, cmd_q); 594 595 e_ctx: 596 ccp_dm_free(&ctx); 597 598 e_key: 599 ccp_dm_free(&key); 600 601 return ret; 602 } 603 604 static int ccp_run_aes_gcm_cmd(struct ccp_cmd_queue *cmd_q, 605 struct ccp_cmd *cmd) 606 { 607 struct ccp_aes_engine *aes = &cmd->u.aes; 608 struct ccp_dm_workarea key, ctx, final_wa, tag; 609 struct ccp_data src, dst; 610 struct ccp_data aad; 611 struct ccp_op op; 612 613 unsigned long long *final; 614 unsigned int dm_offset; 615 unsigned int ilen; 616 bool in_place = true; /* Default value */ 617 int ret; 618 619 struct scatterlist *p_inp, sg_inp[2]; 620 struct scatterlist *p_tag, sg_tag[2]; 621 struct scatterlist *p_outp, sg_outp[2]; 622 struct scatterlist *p_aad; 623 624 if (!aes->iv) 625 return -EINVAL; 626 627 if (!((aes->key_len == AES_KEYSIZE_128) || 628 (aes->key_len == AES_KEYSIZE_192) || 629 (aes->key_len == AES_KEYSIZE_256))) 630 return -EINVAL; 631 632 if (!aes->key) /* Gotta have a key SGL */ 633 return -EINVAL; 634 635 /* First, decompose the source buffer into AAD & PT, 636 * and the destination buffer into AAD, CT & tag, or 637 * the input into CT & tag. 638 * It is expected that the input and output SGs will 639 * be valid, even if the AAD and input lengths are 0. 640 */ 641 p_aad = aes->src; 642 p_inp = scatterwalk_ffwd(sg_inp, aes->src, aes->aad_len); 643 p_outp = scatterwalk_ffwd(sg_outp, aes->dst, aes->aad_len); 644 if (aes->action == CCP_AES_ACTION_ENCRYPT) { 645 ilen = aes->src_len; 646 p_tag = scatterwalk_ffwd(sg_tag, p_outp, ilen); 647 } else { 648 /* Input length for decryption includes tag */ 649 ilen = aes->src_len - AES_BLOCK_SIZE; 650 p_tag = scatterwalk_ffwd(sg_tag, p_inp, ilen); 651 } 652 653 memset(&op, 0, sizeof(op)); 654 op.cmd_q = cmd_q; 655 op.jobid = CCP_NEW_JOBID(cmd_q->ccp); 656 op.sb_key = cmd_q->sb_key; /* Pre-allocated */ 657 op.sb_ctx = cmd_q->sb_ctx; /* Pre-allocated */ 658 op.init = 1; 659 op.u.aes.type = aes->type; 660 661 /* Copy the key to the LSB */ 662 ret = ccp_init_dm_workarea(&key, cmd_q, 663 CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES, 664 DMA_TO_DEVICE); 665 if (ret) 666 return ret; 667 668 dm_offset = CCP_SB_BYTES - aes->key_len; 669 ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len); 670 ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key, 671 CCP_PASSTHRU_BYTESWAP_256BIT); 672 if (ret) { 673 cmd->engine_error = cmd_q->cmd_error; 674 goto e_key; 675 } 676 677 /* Copy the context (IV) to the LSB. 678 * There is an assumption here that the IV is 96 bits in length, plus 679 * a nonce of 32 bits. If no IV is present, use a zeroed buffer. 680 */ 681 ret = ccp_init_dm_workarea(&ctx, cmd_q, 682 CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES, 683 DMA_BIDIRECTIONAL); 684 if (ret) 685 goto e_key; 686 687 dm_offset = CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES - aes->iv_len; 688 ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len); 689 690 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, 691 CCP_PASSTHRU_BYTESWAP_256BIT); 692 if (ret) { 693 cmd->engine_error = cmd_q->cmd_error; 694 goto e_ctx; 695 } 696 697 op.init = 1; 698 if (aes->aad_len > 0) { 699 /* Step 1: Run a GHASH over the Additional Authenticated Data */ 700 ret = ccp_init_data(&aad, cmd_q, p_aad, aes->aad_len, 701 AES_BLOCK_SIZE, 702 DMA_TO_DEVICE); 703 if (ret) 704 goto e_ctx; 705 706 op.u.aes.mode = CCP_AES_MODE_GHASH; 707 op.u.aes.action = CCP_AES_GHASHAAD; 708 709 while (aad.sg_wa.bytes_left) { 710 ccp_prepare_data(&aad, NULL, &op, AES_BLOCK_SIZE, true); 711 712 ret = cmd_q->ccp->vdata->perform->aes(&op); 713 if (ret) { 714 cmd->engine_error = cmd_q->cmd_error; 715 goto e_aad; 716 } 717 718 ccp_process_data(&aad, NULL, &op); 719 op.init = 0; 720 } 721 } 722 723 op.u.aes.mode = CCP_AES_MODE_GCTR; 724 op.u.aes.action = aes->action; 725 726 if (ilen > 0) { 727 /* Step 2: Run a GCTR over the plaintext */ 728 in_place = (sg_virt(p_inp) == sg_virt(p_outp)) ? true : false; 729 730 ret = ccp_init_data(&src, cmd_q, p_inp, ilen, 731 AES_BLOCK_SIZE, 732 in_place ? DMA_BIDIRECTIONAL 733 : DMA_TO_DEVICE); 734 if (ret) 735 goto e_ctx; 736 737 if (in_place) { 738 dst = src; 739 } else { 740 ret = ccp_init_data(&dst, cmd_q, p_outp, ilen, 741 AES_BLOCK_SIZE, DMA_FROM_DEVICE); 742 if (ret) 743 goto e_src; 744 } 745 746 op.soc = 0; 747 op.eom = 0; 748 op.init = 1; 749 while (src.sg_wa.bytes_left) { 750 ccp_prepare_data(&src, &dst, &op, AES_BLOCK_SIZE, true); 751 if (!src.sg_wa.bytes_left) { 752 unsigned int nbytes = aes->src_len 753 % AES_BLOCK_SIZE; 754 755 if (nbytes) { 756 op.eom = 1; 757 op.u.aes.size = (nbytes * 8) - 1; 758 } 759 } 760 761 ret = cmd_q->ccp->vdata->perform->aes(&op); 762 if (ret) { 763 cmd->engine_error = cmd_q->cmd_error; 764 goto e_dst; 765 } 766 767 ccp_process_data(&src, &dst, &op); 768 op.init = 0; 769 } 770 } 771 772 /* Step 3: Update the IV portion of the context with the original IV */ 773 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, 774 CCP_PASSTHRU_BYTESWAP_256BIT); 775 if (ret) { 776 cmd->engine_error = cmd_q->cmd_error; 777 goto e_dst; 778 } 779 780 ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len); 781 782 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, 783 CCP_PASSTHRU_BYTESWAP_256BIT); 784 if (ret) { 785 cmd->engine_error = cmd_q->cmd_error; 786 goto e_dst; 787 } 788 789 /* Step 4: Concatenate the lengths of the AAD and source, and 790 * hash that 16 byte buffer. 791 */ 792 ret = ccp_init_dm_workarea(&final_wa, cmd_q, AES_BLOCK_SIZE, 793 DMA_BIDIRECTIONAL); 794 if (ret) 795 goto e_dst; 796 final = (unsigned long long *) final_wa.address; 797 final[0] = cpu_to_be64(aes->aad_len * 8); 798 final[1] = cpu_to_be64(ilen * 8); 799 800 op.u.aes.mode = CCP_AES_MODE_GHASH; 801 op.u.aes.action = CCP_AES_GHASHFINAL; 802 op.src.type = CCP_MEMTYPE_SYSTEM; 803 op.src.u.dma.address = final_wa.dma.address; 804 op.src.u.dma.length = AES_BLOCK_SIZE; 805 op.dst.type = CCP_MEMTYPE_SYSTEM; 806 op.dst.u.dma.address = final_wa.dma.address; 807 op.dst.u.dma.length = AES_BLOCK_SIZE; 808 op.eom = 1; 809 op.u.aes.size = 0; 810 ret = cmd_q->ccp->vdata->perform->aes(&op); 811 if (ret) 812 goto e_dst; 813 814 if (aes->action == CCP_AES_ACTION_ENCRYPT) { 815 /* Put the ciphered tag after the ciphertext. */ 816 ccp_get_dm_area(&final_wa, 0, p_tag, 0, AES_BLOCK_SIZE); 817 } else { 818 /* Does this ciphered tag match the input? */ 819 ret = ccp_init_dm_workarea(&tag, cmd_q, AES_BLOCK_SIZE, 820 DMA_BIDIRECTIONAL); 821 if (ret) 822 goto e_tag; 823 ccp_set_dm_area(&tag, 0, p_tag, 0, AES_BLOCK_SIZE); 824 825 ret = memcmp(tag.address, final_wa.address, AES_BLOCK_SIZE); 826 ccp_dm_free(&tag); 827 } 828 829 e_tag: 830 ccp_dm_free(&final_wa); 831 832 e_dst: 833 if (aes->src_len && !in_place) 834 ccp_free_data(&dst, cmd_q); 835 836 e_src: 837 if (aes->src_len) 838 ccp_free_data(&src, cmd_q); 839 840 e_aad: 841 if (aes->aad_len) 842 ccp_free_data(&aad, cmd_q); 843 844 e_ctx: 845 ccp_dm_free(&ctx); 846 847 e_key: 848 ccp_dm_free(&key); 849 850 return ret; 851 } 852 853 static int ccp_run_aes_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) 854 { 855 struct ccp_aes_engine *aes = &cmd->u.aes; 856 struct ccp_dm_workarea key, ctx; 857 struct ccp_data src, dst; 858 struct ccp_op op; 859 unsigned int dm_offset; 860 bool in_place = false; 861 int ret; 862 863 if (aes->mode == CCP_AES_MODE_CMAC) 864 return ccp_run_aes_cmac_cmd(cmd_q, cmd); 865 866 if (aes->mode == CCP_AES_MODE_GCM) 867 return ccp_run_aes_gcm_cmd(cmd_q, cmd); 868 869 if (!((aes->key_len == AES_KEYSIZE_128) || 870 (aes->key_len == AES_KEYSIZE_192) || 871 (aes->key_len == AES_KEYSIZE_256))) 872 return -EINVAL; 873 874 if (((aes->mode == CCP_AES_MODE_ECB) || 875 (aes->mode == CCP_AES_MODE_CBC) || 876 (aes->mode == CCP_AES_MODE_CFB)) && 877 (aes->src_len & (AES_BLOCK_SIZE - 1))) 878 return -EINVAL; 879 880 if (!aes->key || !aes->src || !aes->dst) 881 return -EINVAL; 882 883 if (aes->mode != CCP_AES_MODE_ECB) { 884 if (aes->iv_len != AES_BLOCK_SIZE) 885 return -EINVAL; 886 887 if (!aes->iv) 888 return -EINVAL; 889 } 890 891 BUILD_BUG_ON(CCP_AES_KEY_SB_COUNT != 1); 892 BUILD_BUG_ON(CCP_AES_CTX_SB_COUNT != 1); 893 894 ret = -EIO; 895 memset(&op, 0, sizeof(op)); 896 op.cmd_q = cmd_q; 897 op.jobid = CCP_NEW_JOBID(cmd_q->ccp); 898 op.sb_key = cmd_q->sb_key; 899 op.sb_ctx = cmd_q->sb_ctx; 900 op.init = (aes->mode == CCP_AES_MODE_ECB) ? 0 : 1; 901 op.u.aes.type = aes->type; 902 op.u.aes.mode = aes->mode; 903 op.u.aes.action = aes->action; 904 905 /* All supported key sizes fit in a single (32-byte) SB entry 906 * and must be in little endian format. Use the 256-bit byte 907 * swap passthru option to convert from big endian to little 908 * endian. 909 */ 910 ret = ccp_init_dm_workarea(&key, cmd_q, 911 CCP_AES_KEY_SB_COUNT * CCP_SB_BYTES, 912 DMA_TO_DEVICE); 913 if (ret) 914 return ret; 915 916 dm_offset = CCP_SB_BYTES - aes->key_len; 917 ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len); 918 ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key, 919 CCP_PASSTHRU_BYTESWAP_256BIT); 920 if (ret) { 921 cmd->engine_error = cmd_q->cmd_error; 922 goto e_key; 923 } 924 925 /* The AES context fits in a single (32-byte) SB entry and 926 * must be in little endian format. Use the 256-bit byte swap 927 * passthru option to convert from big endian to little endian. 928 */ 929 ret = ccp_init_dm_workarea(&ctx, cmd_q, 930 CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES, 931 DMA_BIDIRECTIONAL); 932 if (ret) 933 goto e_key; 934 935 if (aes->mode != CCP_AES_MODE_ECB) { 936 /* Load the AES context - convert to LE */ 937 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE; 938 ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len); 939 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, 940 CCP_PASSTHRU_BYTESWAP_256BIT); 941 if (ret) { 942 cmd->engine_error = cmd_q->cmd_error; 943 goto e_ctx; 944 } 945 } 946 switch (aes->mode) { 947 case CCP_AES_MODE_CFB: /* CFB128 only */ 948 case CCP_AES_MODE_CTR: 949 op.u.aes.size = AES_BLOCK_SIZE * BITS_PER_BYTE - 1; 950 break; 951 default: 952 op.u.aes.size = 0; 953 } 954 955 /* Prepare the input and output data workareas. For in-place 956 * operations we need to set the dma direction to BIDIRECTIONAL 957 * and copy the src workarea to the dst workarea. 958 */ 959 if (sg_virt(aes->src) == sg_virt(aes->dst)) 960 in_place = true; 961 962 ret = ccp_init_data(&src, cmd_q, aes->src, aes->src_len, 963 AES_BLOCK_SIZE, 964 in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE); 965 if (ret) 966 goto e_ctx; 967 968 if (in_place) { 969 dst = src; 970 } else { 971 ret = ccp_init_data(&dst, cmd_q, aes->dst, aes->src_len, 972 AES_BLOCK_SIZE, DMA_FROM_DEVICE); 973 if (ret) 974 goto e_src; 975 } 976 977 /* Send data to the CCP AES engine */ 978 while (src.sg_wa.bytes_left) { 979 ccp_prepare_data(&src, &dst, &op, AES_BLOCK_SIZE, true); 980 if (!src.sg_wa.bytes_left) { 981 op.eom = 1; 982 983 /* Since we don't retrieve the AES context in ECB 984 * mode we have to wait for the operation to complete 985 * on the last piece of data 986 */ 987 if (aes->mode == CCP_AES_MODE_ECB) 988 op.soc = 1; 989 } 990 991 ret = cmd_q->ccp->vdata->perform->aes(&op); 992 if (ret) { 993 cmd->engine_error = cmd_q->cmd_error; 994 goto e_dst; 995 } 996 997 ccp_process_data(&src, &dst, &op); 998 } 999 1000 if (aes->mode != CCP_AES_MODE_ECB) { 1001 /* Retrieve the AES context - convert from LE to BE using 1002 * 32-byte (256-bit) byteswapping 1003 */ 1004 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, 1005 CCP_PASSTHRU_BYTESWAP_256BIT); 1006 if (ret) { 1007 cmd->engine_error = cmd_q->cmd_error; 1008 goto e_dst; 1009 } 1010 1011 /* ...but we only need AES_BLOCK_SIZE bytes */ 1012 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE; 1013 ccp_get_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len); 1014 } 1015 1016 e_dst: 1017 if (!in_place) 1018 ccp_free_data(&dst, cmd_q); 1019 1020 e_src: 1021 ccp_free_data(&src, cmd_q); 1022 1023 e_ctx: 1024 ccp_dm_free(&ctx); 1025 1026 e_key: 1027 ccp_dm_free(&key); 1028 1029 return ret; 1030 } 1031 1032 static int ccp_run_xts_aes_cmd(struct ccp_cmd_queue *cmd_q, 1033 struct ccp_cmd *cmd) 1034 { 1035 struct ccp_xts_aes_engine *xts = &cmd->u.xts; 1036 struct ccp_dm_workarea key, ctx; 1037 struct ccp_data src, dst; 1038 struct ccp_op op; 1039 unsigned int unit_size, dm_offset; 1040 bool in_place = false; 1041 int ret; 1042 1043 switch (xts->unit_size) { 1044 case CCP_XTS_AES_UNIT_SIZE_16: 1045 unit_size = 16; 1046 break; 1047 case CCP_XTS_AES_UNIT_SIZE_512: 1048 unit_size = 512; 1049 break; 1050 case CCP_XTS_AES_UNIT_SIZE_1024: 1051 unit_size = 1024; 1052 break; 1053 case CCP_XTS_AES_UNIT_SIZE_2048: 1054 unit_size = 2048; 1055 break; 1056 case CCP_XTS_AES_UNIT_SIZE_4096: 1057 unit_size = 4096; 1058 break; 1059 1060 default: 1061 return -EINVAL; 1062 } 1063 1064 if (xts->key_len != AES_KEYSIZE_128) 1065 return -EINVAL; 1066 1067 if (!xts->final && (xts->src_len & (AES_BLOCK_SIZE - 1))) 1068 return -EINVAL; 1069 1070 if (xts->iv_len != AES_BLOCK_SIZE) 1071 return -EINVAL; 1072 1073 if (!xts->key || !xts->iv || !xts->src || !xts->dst) 1074 return -EINVAL; 1075 1076 BUILD_BUG_ON(CCP_XTS_AES_KEY_SB_COUNT != 1); 1077 BUILD_BUG_ON(CCP_XTS_AES_CTX_SB_COUNT != 1); 1078 1079 ret = -EIO; 1080 memset(&op, 0, sizeof(op)); 1081 op.cmd_q = cmd_q; 1082 op.jobid = CCP_NEW_JOBID(cmd_q->ccp); 1083 op.sb_key = cmd_q->sb_key; 1084 op.sb_ctx = cmd_q->sb_ctx; 1085 op.init = 1; 1086 op.u.xts.action = xts->action; 1087 op.u.xts.unit_size = xts->unit_size; 1088 1089 /* All supported key sizes fit in a single (32-byte) SB entry 1090 * and must be in little endian format. Use the 256-bit byte 1091 * swap passthru option to convert from big endian to little 1092 * endian. 1093 */ 1094 ret = ccp_init_dm_workarea(&key, cmd_q, 1095 CCP_XTS_AES_KEY_SB_COUNT * CCP_SB_BYTES, 1096 DMA_TO_DEVICE); 1097 if (ret) 1098 return ret; 1099 1100 dm_offset = CCP_SB_BYTES - AES_KEYSIZE_128; 1101 ccp_set_dm_area(&key, dm_offset, xts->key, 0, xts->key_len); 1102 ccp_set_dm_area(&key, 0, xts->key, dm_offset, xts->key_len); 1103 ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key, 1104 CCP_PASSTHRU_BYTESWAP_256BIT); 1105 if (ret) { 1106 cmd->engine_error = cmd_q->cmd_error; 1107 goto e_key; 1108 } 1109 1110 /* The AES context fits in a single (32-byte) SB entry and 1111 * for XTS is already in little endian format so no byte swapping 1112 * is needed. 1113 */ 1114 ret = ccp_init_dm_workarea(&ctx, cmd_q, 1115 CCP_XTS_AES_CTX_SB_COUNT * CCP_SB_BYTES, 1116 DMA_BIDIRECTIONAL); 1117 if (ret) 1118 goto e_key; 1119 1120 ccp_set_dm_area(&ctx, 0, xts->iv, 0, xts->iv_len); 1121 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, 1122 CCP_PASSTHRU_BYTESWAP_NOOP); 1123 if (ret) { 1124 cmd->engine_error = cmd_q->cmd_error; 1125 goto e_ctx; 1126 } 1127 1128 /* Prepare the input and output data workareas. For in-place 1129 * operations we need to set the dma direction to BIDIRECTIONAL 1130 * and copy the src workarea to the dst workarea. 1131 */ 1132 if (sg_virt(xts->src) == sg_virt(xts->dst)) 1133 in_place = true; 1134 1135 ret = ccp_init_data(&src, cmd_q, xts->src, xts->src_len, 1136 unit_size, 1137 in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE); 1138 if (ret) 1139 goto e_ctx; 1140 1141 if (in_place) { 1142 dst = src; 1143 } else { 1144 ret = ccp_init_data(&dst, cmd_q, xts->dst, xts->src_len, 1145 unit_size, DMA_FROM_DEVICE); 1146 if (ret) 1147 goto e_src; 1148 } 1149 1150 /* Send data to the CCP AES engine */ 1151 while (src.sg_wa.bytes_left) { 1152 ccp_prepare_data(&src, &dst, &op, unit_size, true); 1153 if (!src.sg_wa.bytes_left) 1154 op.eom = 1; 1155 1156 ret = cmd_q->ccp->vdata->perform->xts_aes(&op); 1157 if (ret) { 1158 cmd->engine_error = cmd_q->cmd_error; 1159 goto e_dst; 1160 } 1161 1162 ccp_process_data(&src, &dst, &op); 1163 } 1164 1165 /* Retrieve the AES context - convert from LE to BE using 1166 * 32-byte (256-bit) byteswapping 1167 */ 1168 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, 1169 CCP_PASSTHRU_BYTESWAP_256BIT); 1170 if (ret) { 1171 cmd->engine_error = cmd_q->cmd_error; 1172 goto e_dst; 1173 } 1174 1175 /* ...but we only need AES_BLOCK_SIZE bytes */ 1176 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE; 1177 ccp_get_dm_area(&ctx, dm_offset, xts->iv, 0, xts->iv_len); 1178 1179 e_dst: 1180 if (!in_place) 1181 ccp_free_data(&dst, cmd_q); 1182 1183 e_src: 1184 ccp_free_data(&src, cmd_q); 1185 1186 e_ctx: 1187 ccp_dm_free(&ctx); 1188 1189 e_key: 1190 ccp_dm_free(&key); 1191 1192 return ret; 1193 } 1194 1195 static int ccp_run_des3_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) 1196 { 1197 struct ccp_des3_engine *des3 = &cmd->u.des3; 1198 1199 struct ccp_dm_workarea key, ctx; 1200 struct ccp_data src, dst; 1201 struct ccp_op op; 1202 unsigned int dm_offset; 1203 unsigned int len_singlekey; 1204 bool in_place = false; 1205 int ret; 1206 1207 /* Error checks */ 1208 if (!cmd_q->ccp->vdata->perform->des3) 1209 return -EINVAL; 1210 1211 if (des3->key_len != DES3_EDE_KEY_SIZE) 1212 return -EINVAL; 1213 1214 if (((des3->mode == CCP_DES3_MODE_ECB) || 1215 (des3->mode == CCP_DES3_MODE_CBC)) && 1216 (des3->src_len & (DES3_EDE_BLOCK_SIZE - 1))) 1217 return -EINVAL; 1218 1219 if (!des3->key || !des3->src || !des3->dst) 1220 return -EINVAL; 1221 1222 if (des3->mode != CCP_DES3_MODE_ECB) { 1223 if (des3->iv_len != DES3_EDE_BLOCK_SIZE) 1224 return -EINVAL; 1225 1226 if (!des3->iv) 1227 return -EINVAL; 1228 } 1229 1230 ret = -EIO; 1231 /* Zero out all the fields of the command desc */ 1232 memset(&op, 0, sizeof(op)); 1233 1234 /* Set up the Function field */ 1235 op.cmd_q = cmd_q; 1236 op.jobid = CCP_NEW_JOBID(cmd_q->ccp); 1237 op.sb_key = cmd_q->sb_key; 1238 1239 op.init = (des3->mode == CCP_DES3_MODE_ECB) ? 0 : 1; 1240 op.u.des3.type = des3->type; 1241 op.u.des3.mode = des3->mode; 1242 op.u.des3.action = des3->action; 1243 1244 /* 1245 * All supported key sizes fit in a single (32-byte) KSB entry and 1246 * (like AES) must be in little endian format. Use the 256-bit byte 1247 * swap passthru option to convert from big endian to little endian. 1248 */ 1249 ret = ccp_init_dm_workarea(&key, cmd_q, 1250 CCP_DES3_KEY_SB_COUNT * CCP_SB_BYTES, 1251 DMA_TO_DEVICE); 1252 if (ret) 1253 return ret; 1254 1255 /* 1256 * The contents of the key triplet are in the reverse order of what 1257 * is required by the engine. Copy the 3 pieces individually to put 1258 * them where they belong. 1259 */ 1260 dm_offset = CCP_SB_BYTES - des3->key_len; /* Basic offset */ 1261 1262 len_singlekey = des3->key_len / 3; 1263 ccp_set_dm_area(&key, dm_offset + 2 * len_singlekey, 1264 des3->key, 0, len_singlekey); 1265 ccp_set_dm_area(&key, dm_offset + len_singlekey, 1266 des3->key, len_singlekey, len_singlekey); 1267 ccp_set_dm_area(&key, dm_offset, 1268 des3->key, 2 * len_singlekey, len_singlekey); 1269 1270 /* Copy the key to the SB */ 1271 ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key, 1272 CCP_PASSTHRU_BYTESWAP_256BIT); 1273 if (ret) { 1274 cmd->engine_error = cmd_q->cmd_error; 1275 goto e_key; 1276 } 1277 1278 /* 1279 * The DES3 context fits in a single (32-byte) KSB entry and 1280 * must be in little endian format. Use the 256-bit byte swap 1281 * passthru option to convert from big endian to little endian. 1282 */ 1283 if (des3->mode != CCP_DES3_MODE_ECB) { 1284 u32 load_mode; 1285 1286 op.sb_ctx = cmd_q->sb_ctx; 1287 1288 ret = ccp_init_dm_workarea(&ctx, cmd_q, 1289 CCP_DES3_CTX_SB_COUNT * CCP_SB_BYTES, 1290 DMA_BIDIRECTIONAL); 1291 if (ret) 1292 goto e_key; 1293 1294 /* Load the context into the LSB */ 1295 dm_offset = CCP_SB_BYTES - des3->iv_len; 1296 ccp_set_dm_area(&ctx, dm_offset, des3->iv, 0, des3->iv_len); 1297 1298 if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0)) 1299 load_mode = CCP_PASSTHRU_BYTESWAP_NOOP; 1300 else 1301 load_mode = CCP_PASSTHRU_BYTESWAP_256BIT; 1302 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, 1303 load_mode); 1304 if (ret) { 1305 cmd->engine_error = cmd_q->cmd_error; 1306 goto e_ctx; 1307 } 1308 } 1309 1310 /* 1311 * Prepare the input and output data workareas. For in-place 1312 * operations we need to set the dma direction to BIDIRECTIONAL 1313 * and copy the src workarea to the dst workarea. 1314 */ 1315 if (sg_virt(des3->src) == sg_virt(des3->dst)) 1316 in_place = true; 1317 1318 ret = ccp_init_data(&src, cmd_q, des3->src, des3->src_len, 1319 DES3_EDE_BLOCK_SIZE, 1320 in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE); 1321 if (ret) 1322 goto e_ctx; 1323 1324 if (in_place) 1325 dst = src; 1326 else { 1327 ret = ccp_init_data(&dst, cmd_q, des3->dst, des3->src_len, 1328 DES3_EDE_BLOCK_SIZE, DMA_FROM_DEVICE); 1329 if (ret) 1330 goto e_src; 1331 } 1332 1333 /* Send data to the CCP DES3 engine */ 1334 while (src.sg_wa.bytes_left) { 1335 ccp_prepare_data(&src, &dst, &op, DES3_EDE_BLOCK_SIZE, true); 1336 if (!src.sg_wa.bytes_left) { 1337 op.eom = 1; 1338 1339 /* Since we don't retrieve the context in ECB mode 1340 * we have to wait for the operation to complete 1341 * on the last piece of data 1342 */ 1343 op.soc = 0; 1344 } 1345 1346 ret = cmd_q->ccp->vdata->perform->des3(&op); 1347 if (ret) { 1348 cmd->engine_error = cmd_q->cmd_error; 1349 goto e_dst; 1350 } 1351 1352 ccp_process_data(&src, &dst, &op); 1353 } 1354 1355 if (des3->mode != CCP_DES3_MODE_ECB) { 1356 /* Retrieve the context and make BE */ 1357 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, 1358 CCP_PASSTHRU_BYTESWAP_256BIT); 1359 if (ret) { 1360 cmd->engine_error = cmd_q->cmd_error; 1361 goto e_dst; 1362 } 1363 1364 /* ...but we only need the last DES3_EDE_BLOCK_SIZE bytes */ 1365 if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0)) 1366 dm_offset = CCP_SB_BYTES - des3->iv_len; 1367 else 1368 dm_offset = 0; 1369 ccp_get_dm_area(&ctx, dm_offset, des3->iv, 0, 1370 DES3_EDE_BLOCK_SIZE); 1371 } 1372 e_dst: 1373 if (!in_place) 1374 ccp_free_data(&dst, cmd_q); 1375 1376 e_src: 1377 ccp_free_data(&src, cmd_q); 1378 1379 e_ctx: 1380 if (des3->mode != CCP_DES3_MODE_ECB) 1381 ccp_dm_free(&ctx); 1382 1383 e_key: 1384 ccp_dm_free(&key); 1385 1386 return ret; 1387 } 1388 1389 static int ccp_run_sha_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) 1390 { 1391 struct ccp_sha_engine *sha = &cmd->u.sha; 1392 struct ccp_dm_workarea ctx; 1393 struct ccp_data src; 1394 struct ccp_op op; 1395 unsigned int ioffset, ooffset; 1396 unsigned int digest_size; 1397 int sb_count; 1398 const void *init; 1399 u64 block_size; 1400 int ctx_size; 1401 int ret; 1402 1403 switch (sha->type) { 1404 case CCP_SHA_TYPE_1: 1405 if (sha->ctx_len < SHA1_DIGEST_SIZE) 1406 return -EINVAL; 1407 block_size = SHA1_BLOCK_SIZE; 1408 break; 1409 case CCP_SHA_TYPE_224: 1410 if (sha->ctx_len < SHA224_DIGEST_SIZE) 1411 return -EINVAL; 1412 block_size = SHA224_BLOCK_SIZE; 1413 break; 1414 case CCP_SHA_TYPE_256: 1415 if (sha->ctx_len < SHA256_DIGEST_SIZE) 1416 return -EINVAL; 1417 block_size = SHA256_BLOCK_SIZE; 1418 break; 1419 case CCP_SHA_TYPE_384: 1420 if (cmd_q->ccp->vdata->version < CCP_VERSION(4, 0) 1421 || sha->ctx_len < SHA384_DIGEST_SIZE) 1422 return -EINVAL; 1423 block_size = SHA384_BLOCK_SIZE; 1424 break; 1425 case CCP_SHA_TYPE_512: 1426 if (cmd_q->ccp->vdata->version < CCP_VERSION(4, 0) 1427 || sha->ctx_len < SHA512_DIGEST_SIZE) 1428 return -EINVAL; 1429 block_size = SHA512_BLOCK_SIZE; 1430 break; 1431 default: 1432 return -EINVAL; 1433 } 1434 1435 if (!sha->ctx) 1436 return -EINVAL; 1437 1438 if (!sha->final && (sha->src_len & (block_size - 1))) 1439 return -EINVAL; 1440 1441 /* The version 3 device can't handle zero-length input */ 1442 if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0)) { 1443 1444 if (!sha->src_len) { 1445 unsigned int digest_len; 1446 const u8 *sha_zero; 1447 1448 /* Not final, just return */ 1449 if (!sha->final) 1450 return 0; 1451 1452 /* CCP can't do a zero length sha operation so the 1453 * caller must buffer the data. 1454 */ 1455 if (sha->msg_bits) 1456 return -EINVAL; 1457 1458 /* The CCP cannot perform zero-length sha operations 1459 * so the caller is required to buffer data for the 1460 * final operation. However, a sha operation for a 1461 * message with a total length of zero is valid so 1462 * known values are required to supply the result. 1463 */ 1464 switch (sha->type) { 1465 case CCP_SHA_TYPE_1: 1466 sha_zero = sha1_zero_message_hash; 1467 digest_len = SHA1_DIGEST_SIZE; 1468 break; 1469 case CCP_SHA_TYPE_224: 1470 sha_zero = sha224_zero_message_hash; 1471 digest_len = SHA224_DIGEST_SIZE; 1472 break; 1473 case CCP_SHA_TYPE_256: 1474 sha_zero = sha256_zero_message_hash; 1475 digest_len = SHA256_DIGEST_SIZE; 1476 break; 1477 default: 1478 return -EINVAL; 1479 } 1480 1481 scatterwalk_map_and_copy((void *)sha_zero, sha->ctx, 0, 1482 digest_len, 1); 1483 1484 return 0; 1485 } 1486 } 1487 1488 /* Set variables used throughout */ 1489 switch (sha->type) { 1490 case CCP_SHA_TYPE_1: 1491 digest_size = SHA1_DIGEST_SIZE; 1492 init = (void *) ccp_sha1_init; 1493 ctx_size = SHA1_DIGEST_SIZE; 1494 sb_count = 1; 1495 if (cmd_q->ccp->vdata->version != CCP_VERSION(3, 0)) 1496 ooffset = ioffset = CCP_SB_BYTES - SHA1_DIGEST_SIZE; 1497 else 1498 ooffset = ioffset = 0; 1499 break; 1500 case CCP_SHA_TYPE_224: 1501 digest_size = SHA224_DIGEST_SIZE; 1502 init = (void *) ccp_sha224_init; 1503 ctx_size = SHA256_DIGEST_SIZE; 1504 sb_count = 1; 1505 ioffset = 0; 1506 if (cmd_q->ccp->vdata->version != CCP_VERSION(3, 0)) 1507 ooffset = CCP_SB_BYTES - SHA224_DIGEST_SIZE; 1508 else 1509 ooffset = 0; 1510 break; 1511 case CCP_SHA_TYPE_256: 1512 digest_size = SHA256_DIGEST_SIZE; 1513 init = (void *) ccp_sha256_init; 1514 ctx_size = SHA256_DIGEST_SIZE; 1515 sb_count = 1; 1516 ooffset = ioffset = 0; 1517 break; 1518 case CCP_SHA_TYPE_384: 1519 digest_size = SHA384_DIGEST_SIZE; 1520 init = (void *) ccp_sha384_init; 1521 ctx_size = SHA512_DIGEST_SIZE; 1522 sb_count = 2; 1523 ioffset = 0; 1524 ooffset = 2 * CCP_SB_BYTES - SHA384_DIGEST_SIZE; 1525 break; 1526 case CCP_SHA_TYPE_512: 1527 digest_size = SHA512_DIGEST_SIZE; 1528 init = (void *) ccp_sha512_init; 1529 ctx_size = SHA512_DIGEST_SIZE; 1530 sb_count = 2; 1531 ooffset = ioffset = 0; 1532 break; 1533 default: 1534 ret = -EINVAL; 1535 goto e_data; 1536 } 1537 1538 /* For zero-length plaintext the src pointer is ignored; 1539 * otherwise both parts must be valid 1540 */ 1541 if (sha->src_len && !sha->src) 1542 return -EINVAL; 1543 1544 memset(&op, 0, sizeof(op)); 1545 op.cmd_q = cmd_q; 1546 op.jobid = CCP_NEW_JOBID(cmd_q->ccp); 1547 op.sb_ctx = cmd_q->sb_ctx; /* Pre-allocated */ 1548 op.u.sha.type = sha->type; 1549 op.u.sha.msg_bits = sha->msg_bits; 1550 1551 /* For SHA1/224/256 the context fits in a single (32-byte) SB entry; 1552 * SHA384/512 require 2 adjacent SB slots, with the right half in the 1553 * first slot, and the left half in the second. Each portion must then 1554 * be in little endian format: use the 256-bit byte swap option. 1555 */ 1556 ret = ccp_init_dm_workarea(&ctx, cmd_q, sb_count * CCP_SB_BYTES, 1557 DMA_BIDIRECTIONAL); 1558 if (ret) 1559 return ret; 1560 if (sha->first) { 1561 switch (sha->type) { 1562 case CCP_SHA_TYPE_1: 1563 case CCP_SHA_TYPE_224: 1564 case CCP_SHA_TYPE_256: 1565 memcpy(ctx.address + ioffset, init, ctx_size); 1566 break; 1567 case CCP_SHA_TYPE_384: 1568 case CCP_SHA_TYPE_512: 1569 memcpy(ctx.address + ctx_size / 2, init, 1570 ctx_size / 2); 1571 memcpy(ctx.address, init + ctx_size / 2, 1572 ctx_size / 2); 1573 break; 1574 default: 1575 ret = -EINVAL; 1576 goto e_ctx; 1577 } 1578 } else { 1579 /* Restore the context */ 1580 ccp_set_dm_area(&ctx, 0, sha->ctx, 0, 1581 sb_count * CCP_SB_BYTES); 1582 } 1583 1584 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, 1585 CCP_PASSTHRU_BYTESWAP_256BIT); 1586 if (ret) { 1587 cmd->engine_error = cmd_q->cmd_error; 1588 goto e_ctx; 1589 } 1590 1591 if (sha->src) { 1592 /* Send data to the CCP SHA engine; block_size is set above */ 1593 ret = ccp_init_data(&src, cmd_q, sha->src, sha->src_len, 1594 block_size, DMA_TO_DEVICE); 1595 if (ret) 1596 goto e_ctx; 1597 1598 while (src.sg_wa.bytes_left) { 1599 ccp_prepare_data(&src, NULL, &op, block_size, false); 1600 if (sha->final && !src.sg_wa.bytes_left) 1601 op.eom = 1; 1602 1603 ret = cmd_q->ccp->vdata->perform->sha(&op); 1604 if (ret) { 1605 cmd->engine_error = cmd_q->cmd_error; 1606 goto e_data; 1607 } 1608 1609 ccp_process_data(&src, NULL, &op); 1610 } 1611 } else { 1612 op.eom = 1; 1613 ret = cmd_q->ccp->vdata->perform->sha(&op); 1614 if (ret) { 1615 cmd->engine_error = cmd_q->cmd_error; 1616 goto e_data; 1617 } 1618 } 1619 1620 /* Retrieve the SHA context - convert from LE to BE using 1621 * 32-byte (256-bit) byteswapping to BE 1622 */ 1623 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, 1624 CCP_PASSTHRU_BYTESWAP_256BIT); 1625 if (ret) { 1626 cmd->engine_error = cmd_q->cmd_error; 1627 goto e_data; 1628 } 1629 1630 if (sha->final) { 1631 /* Finishing up, so get the digest */ 1632 switch (sha->type) { 1633 case CCP_SHA_TYPE_1: 1634 case CCP_SHA_TYPE_224: 1635 case CCP_SHA_TYPE_256: 1636 ccp_get_dm_area(&ctx, ooffset, 1637 sha->ctx, 0, 1638 digest_size); 1639 break; 1640 case CCP_SHA_TYPE_384: 1641 case CCP_SHA_TYPE_512: 1642 ccp_get_dm_area(&ctx, 0, 1643 sha->ctx, LSB_ITEM_SIZE - ooffset, 1644 LSB_ITEM_SIZE); 1645 ccp_get_dm_area(&ctx, LSB_ITEM_SIZE + ooffset, 1646 sha->ctx, 0, 1647 LSB_ITEM_SIZE - ooffset); 1648 break; 1649 default: 1650 ret = -EINVAL; 1651 goto e_ctx; 1652 } 1653 } else { 1654 /* Stash the context */ 1655 ccp_get_dm_area(&ctx, 0, sha->ctx, 0, 1656 sb_count * CCP_SB_BYTES); 1657 } 1658 1659 if (sha->final && sha->opad) { 1660 /* HMAC operation, recursively perform final SHA */ 1661 struct ccp_cmd hmac_cmd; 1662 struct scatterlist sg; 1663 u8 *hmac_buf; 1664 1665 if (sha->opad_len != block_size) { 1666 ret = -EINVAL; 1667 goto e_data; 1668 } 1669 1670 hmac_buf = kmalloc(block_size + digest_size, GFP_KERNEL); 1671 if (!hmac_buf) { 1672 ret = -ENOMEM; 1673 goto e_data; 1674 } 1675 sg_init_one(&sg, hmac_buf, block_size + digest_size); 1676 1677 scatterwalk_map_and_copy(hmac_buf, sha->opad, 0, block_size, 0); 1678 switch (sha->type) { 1679 case CCP_SHA_TYPE_1: 1680 case CCP_SHA_TYPE_224: 1681 case CCP_SHA_TYPE_256: 1682 memcpy(hmac_buf + block_size, 1683 ctx.address + ooffset, 1684 digest_size); 1685 break; 1686 case CCP_SHA_TYPE_384: 1687 case CCP_SHA_TYPE_512: 1688 memcpy(hmac_buf + block_size, 1689 ctx.address + LSB_ITEM_SIZE + ooffset, 1690 LSB_ITEM_SIZE); 1691 memcpy(hmac_buf + block_size + 1692 (LSB_ITEM_SIZE - ooffset), 1693 ctx.address, 1694 LSB_ITEM_SIZE); 1695 break; 1696 default: 1697 ret = -EINVAL; 1698 goto e_ctx; 1699 } 1700 1701 memset(&hmac_cmd, 0, sizeof(hmac_cmd)); 1702 hmac_cmd.engine = CCP_ENGINE_SHA; 1703 hmac_cmd.u.sha.type = sha->type; 1704 hmac_cmd.u.sha.ctx = sha->ctx; 1705 hmac_cmd.u.sha.ctx_len = sha->ctx_len; 1706 hmac_cmd.u.sha.src = &sg; 1707 hmac_cmd.u.sha.src_len = block_size + digest_size; 1708 hmac_cmd.u.sha.opad = NULL; 1709 hmac_cmd.u.sha.opad_len = 0; 1710 hmac_cmd.u.sha.first = 1; 1711 hmac_cmd.u.sha.final = 1; 1712 hmac_cmd.u.sha.msg_bits = (block_size + digest_size) << 3; 1713 1714 ret = ccp_run_sha_cmd(cmd_q, &hmac_cmd); 1715 if (ret) 1716 cmd->engine_error = hmac_cmd.engine_error; 1717 1718 kfree(hmac_buf); 1719 } 1720 1721 e_data: 1722 if (sha->src) 1723 ccp_free_data(&src, cmd_q); 1724 1725 e_ctx: 1726 ccp_dm_free(&ctx); 1727 1728 return ret; 1729 } 1730 1731 static int ccp_run_rsa_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) 1732 { 1733 struct ccp_rsa_engine *rsa = &cmd->u.rsa; 1734 struct ccp_dm_workarea exp, src, dst; 1735 struct ccp_op op; 1736 unsigned int sb_count, i_len, o_len; 1737 int ret; 1738 1739 /* Check against the maximum allowable size, in bits */ 1740 if (rsa->key_size > cmd_q->ccp->vdata->rsamax) 1741 return -EINVAL; 1742 1743 if (!rsa->exp || !rsa->mod || !rsa->src || !rsa->dst) 1744 return -EINVAL; 1745 1746 memset(&op, 0, sizeof(op)); 1747 op.cmd_q = cmd_q; 1748 op.jobid = CCP_NEW_JOBID(cmd_q->ccp); 1749 1750 /* The RSA modulus must precede the message being acted upon, so 1751 * it must be copied to a DMA area where the message and the 1752 * modulus can be concatenated. Therefore the input buffer 1753 * length required is twice the output buffer length (which 1754 * must be a multiple of 256-bits). Compute o_len, i_len in bytes. 1755 * Buffer sizes must be a multiple of 32 bytes; rounding up may be 1756 * required. 1757 */ 1758 o_len = 32 * ((rsa->key_size + 255) / 256); 1759 i_len = o_len * 2; 1760 1761 if (cmd_q->ccp->vdata->version < CCP_VERSION(5, 0)) { 1762 /* sb_count is the number of storage block slots required 1763 * for the modulus. 1764 */ 1765 sb_count = o_len / CCP_SB_BYTES; 1766 op.sb_key = cmd_q->ccp->vdata->perform->sballoc(cmd_q, 1767 sb_count); 1768 if (!op.sb_key) 1769 return -EIO; 1770 } else { 1771 /* A version 5 device allows a modulus size that will not fit 1772 * in the LSB, so the command will transfer it from memory. 1773 * Set the sb key to the default, even though it's not used. 1774 */ 1775 op.sb_key = cmd_q->sb_key; 1776 } 1777 1778 /* The RSA exponent must be in little endian format. Reverse its 1779 * byte order. 1780 */ 1781 ret = ccp_init_dm_workarea(&exp, cmd_q, o_len, DMA_TO_DEVICE); 1782 if (ret) 1783 goto e_sb; 1784 1785 ret = ccp_reverse_set_dm_area(&exp, 0, rsa->exp, 0, rsa->exp_len); 1786 if (ret) 1787 goto e_exp; 1788 1789 if (cmd_q->ccp->vdata->version < CCP_VERSION(5, 0)) { 1790 /* Copy the exponent to the local storage block, using 1791 * as many 32-byte blocks as were allocated above. It's 1792 * already little endian, so no further change is required. 1793 */ 1794 ret = ccp_copy_to_sb(cmd_q, &exp, op.jobid, op.sb_key, 1795 CCP_PASSTHRU_BYTESWAP_NOOP); 1796 if (ret) { 1797 cmd->engine_error = cmd_q->cmd_error; 1798 goto e_exp; 1799 } 1800 } else { 1801 /* The exponent can be retrieved from memory via DMA. */ 1802 op.exp.u.dma.address = exp.dma.address; 1803 op.exp.u.dma.offset = 0; 1804 } 1805 1806 /* Concatenate the modulus and the message. Both the modulus and 1807 * the operands must be in little endian format. Since the input 1808 * is in big endian format it must be converted. 1809 */ 1810 ret = ccp_init_dm_workarea(&src, cmd_q, i_len, DMA_TO_DEVICE); 1811 if (ret) 1812 goto e_exp; 1813 1814 ret = ccp_reverse_set_dm_area(&src, 0, rsa->mod, 0, rsa->mod_len); 1815 if (ret) 1816 goto e_src; 1817 ret = ccp_reverse_set_dm_area(&src, o_len, rsa->src, 0, rsa->src_len); 1818 if (ret) 1819 goto e_src; 1820 1821 /* Prepare the output area for the operation */ 1822 ret = ccp_init_dm_workarea(&dst, cmd_q, o_len, DMA_FROM_DEVICE); 1823 if (ret) 1824 goto e_src; 1825 1826 op.soc = 1; 1827 op.src.u.dma.address = src.dma.address; 1828 op.src.u.dma.offset = 0; 1829 op.src.u.dma.length = i_len; 1830 op.dst.u.dma.address = dst.dma.address; 1831 op.dst.u.dma.offset = 0; 1832 op.dst.u.dma.length = o_len; 1833 1834 op.u.rsa.mod_size = rsa->key_size; 1835 op.u.rsa.input_len = i_len; 1836 1837 ret = cmd_q->ccp->vdata->perform->rsa(&op); 1838 if (ret) { 1839 cmd->engine_error = cmd_q->cmd_error; 1840 goto e_dst; 1841 } 1842 1843 ccp_reverse_get_dm_area(&dst, 0, rsa->dst, 0, rsa->mod_len); 1844 1845 e_dst: 1846 ccp_dm_free(&dst); 1847 1848 e_src: 1849 ccp_dm_free(&src); 1850 1851 e_exp: 1852 ccp_dm_free(&exp); 1853 1854 e_sb: 1855 if (cmd_q->ccp->vdata->version < CCP_VERSION(5, 0)) 1856 cmd_q->ccp->vdata->perform->sbfree(cmd_q, op.sb_key, sb_count); 1857 1858 return ret; 1859 } 1860 1861 static int ccp_run_passthru_cmd(struct ccp_cmd_queue *cmd_q, 1862 struct ccp_cmd *cmd) 1863 { 1864 struct ccp_passthru_engine *pt = &cmd->u.passthru; 1865 struct ccp_dm_workarea mask; 1866 struct ccp_data src, dst; 1867 struct ccp_op op; 1868 bool in_place = false; 1869 unsigned int i; 1870 int ret = 0; 1871 1872 if (!pt->final && (pt->src_len & (CCP_PASSTHRU_BLOCKSIZE - 1))) 1873 return -EINVAL; 1874 1875 if (!pt->src || !pt->dst) 1876 return -EINVAL; 1877 1878 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) { 1879 if (pt->mask_len != CCP_PASSTHRU_MASKSIZE) 1880 return -EINVAL; 1881 if (!pt->mask) 1882 return -EINVAL; 1883 } 1884 1885 BUILD_BUG_ON(CCP_PASSTHRU_SB_COUNT != 1); 1886 1887 memset(&op, 0, sizeof(op)); 1888 op.cmd_q = cmd_q; 1889 op.jobid = CCP_NEW_JOBID(cmd_q->ccp); 1890 1891 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) { 1892 /* Load the mask */ 1893 op.sb_key = cmd_q->sb_key; 1894 1895 ret = ccp_init_dm_workarea(&mask, cmd_q, 1896 CCP_PASSTHRU_SB_COUNT * 1897 CCP_SB_BYTES, 1898 DMA_TO_DEVICE); 1899 if (ret) 1900 return ret; 1901 1902 ccp_set_dm_area(&mask, 0, pt->mask, 0, pt->mask_len); 1903 ret = ccp_copy_to_sb(cmd_q, &mask, op.jobid, op.sb_key, 1904 CCP_PASSTHRU_BYTESWAP_NOOP); 1905 if (ret) { 1906 cmd->engine_error = cmd_q->cmd_error; 1907 goto e_mask; 1908 } 1909 } 1910 1911 /* Prepare the input and output data workareas. For in-place 1912 * operations we need to set the dma direction to BIDIRECTIONAL 1913 * and copy the src workarea to the dst workarea. 1914 */ 1915 if (sg_virt(pt->src) == sg_virt(pt->dst)) 1916 in_place = true; 1917 1918 ret = ccp_init_data(&src, cmd_q, pt->src, pt->src_len, 1919 CCP_PASSTHRU_MASKSIZE, 1920 in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE); 1921 if (ret) 1922 goto e_mask; 1923 1924 if (in_place) { 1925 dst = src; 1926 } else { 1927 ret = ccp_init_data(&dst, cmd_q, pt->dst, pt->src_len, 1928 CCP_PASSTHRU_MASKSIZE, DMA_FROM_DEVICE); 1929 if (ret) 1930 goto e_src; 1931 } 1932 1933 /* Send data to the CCP Passthru engine 1934 * Because the CCP engine works on a single source and destination 1935 * dma address at a time, each entry in the source scatterlist 1936 * (after the dma_map_sg call) must be less than or equal to the 1937 * (remaining) length in the destination scatterlist entry and the 1938 * length must be a multiple of CCP_PASSTHRU_BLOCKSIZE 1939 */ 1940 dst.sg_wa.sg_used = 0; 1941 for (i = 1; i <= src.sg_wa.dma_count; i++) { 1942 if (!dst.sg_wa.sg || 1943 (dst.sg_wa.sg->length < src.sg_wa.sg->length)) { 1944 ret = -EINVAL; 1945 goto e_dst; 1946 } 1947 1948 if (i == src.sg_wa.dma_count) { 1949 op.eom = 1; 1950 op.soc = 1; 1951 } 1952 1953 op.src.type = CCP_MEMTYPE_SYSTEM; 1954 op.src.u.dma.address = sg_dma_address(src.sg_wa.sg); 1955 op.src.u.dma.offset = 0; 1956 op.src.u.dma.length = sg_dma_len(src.sg_wa.sg); 1957 1958 op.dst.type = CCP_MEMTYPE_SYSTEM; 1959 op.dst.u.dma.address = sg_dma_address(dst.sg_wa.sg); 1960 op.dst.u.dma.offset = dst.sg_wa.sg_used; 1961 op.dst.u.dma.length = op.src.u.dma.length; 1962 1963 ret = cmd_q->ccp->vdata->perform->passthru(&op); 1964 if (ret) { 1965 cmd->engine_error = cmd_q->cmd_error; 1966 goto e_dst; 1967 } 1968 1969 dst.sg_wa.sg_used += src.sg_wa.sg->length; 1970 if (dst.sg_wa.sg_used == dst.sg_wa.sg->length) { 1971 dst.sg_wa.sg = sg_next(dst.sg_wa.sg); 1972 dst.sg_wa.sg_used = 0; 1973 } 1974 src.sg_wa.sg = sg_next(src.sg_wa.sg); 1975 } 1976 1977 e_dst: 1978 if (!in_place) 1979 ccp_free_data(&dst, cmd_q); 1980 1981 e_src: 1982 ccp_free_data(&src, cmd_q); 1983 1984 e_mask: 1985 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) 1986 ccp_dm_free(&mask); 1987 1988 return ret; 1989 } 1990 1991 static int ccp_run_passthru_nomap_cmd(struct ccp_cmd_queue *cmd_q, 1992 struct ccp_cmd *cmd) 1993 { 1994 struct ccp_passthru_nomap_engine *pt = &cmd->u.passthru_nomap; 1995 struct ccp_dm_workarea mask; 1996 struct ccp_op op; 1997 int ret; 1998 1999 if (!pt->final && (pt->src_len & (CCP_PASSTHRU_BLOCKSIZE - 1))) 2000 return -EINVAL; 2001 2002 if (!pt->src_dma || !pt->dst_dma) 2003 return -EINVAL; 2004 2005 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) { 2006 if (pt->mask_len != CCP_PASSTHRU_MASKSIZE) 2007 return -EINVAL; 2008 if (!pt->mask) 2009 return -EINVAL; 2010 } 2011 2012 BUILD_BUG_ON(CCP_PASSTHRU_SB_COUNT != 1); 2013 2014 memset(&op, 0, sizeof(op)); 2015 op.cmd_q = cmd_q; 2016 op.jobid = CCP_NEW_JOBID(cmd_q->ccp); 2017 2018 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) { 2019 /* Load the mask */ 2020 op.sb_key = cmd_q->sb_key; 2021 2022 mask.length = pt->mask_len; 2023 mask.dma.address = pt->mask; 2024 mask.dma.length = pt->mask_len; 2025 2026 ret = ccp_copy_to_sb(cmd_q, &mask, op.jobid, op.sb_key, 2027 CCP_PASSTHRU_BYTESWAP_NOOP); 2028 if (ret) { 2029 cmd->engine_error = cmd_q->cmd_error; 2030 return ret; 2031 } 2032 } 2033 2034 /* Send data to the CCP Passthru engine */ 2035 op.eom = 1; 2036 op.soc = 1; 2037 2038 op.src.type = CCP_MEMTYPE_SYSTEM; 2039 op.src.u.dma.address = pt->src_dma; 2040 op.src.u.dma.offset = 0; 2041 op.src.u.dma.length = pt->src_len; 2042 2043 op.dst.type = CCP_MEMTYPE_SYSTEM; 2044 op.dst.u.dma.address = pt->dst_dma; 2045 op.dst.u.dma.offset = 0; 2046 op.dst.u.dma.length = pt->src_len; 2047 2048 ret = cmd_q->ccp->vdata->perform->passthru(&op); 2049 if (ret) 2050 cmd->engine_error = cmd_q->cmd_error; 2051 2052 return ret; 2053 } 2054 2055 static int ccp_run_ecc_mm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) 2056 { 2057 struct ccp_ecc_engine *ecc = &cmd->u.ecc; 2058 struct ccp_dm_workarea src, dst; 2059 struct ccp_op op; 2060 int ret; 2061 u8 *save; 2062 2063 if (!ecc->u.mm.operand_1 || 2064 (ecc->u.mm.operand_1_len > CCP_ECC_MODULUS_BYTES)) 2065 return -EINVAL; 2066 2067 if (ecc->function != CCP_ECC_FUNCTION_MINV_384BIT) 2068 if (!ecc->u.mm.operand_2 || 2069 (ecc->u.mm.operand_2_len > CCP_ECC_MODULUS_BYTES)) 2070 return -EINVAL; 2071 2072 if (!ecc->u.mm.result || 2073 (ecc->u.mm.result_len < CCP_ECC_MODULUS_BYTES)) 2074 return -EINVAL; 2075 2076 memset(&op, 0, sizeof(op)); 2077 op.cmd_q = cmd_q; 2078 op.jobid = CCP_NEW_JOBID(cmd_q->ccp); 2079 2080 /* Concatenate the modulus and the operands. Both the modulus and 2081 * the operands must be in little endian format. Since the input 2082 * is in big endian format it must be converted and placed in a 2083 * fixed length buffer. 2084 */ 2085 ret = ccp_init_dm_workarea(&src, cmd_q, CCP_ECC_SRC_BUF_SIZE, 2086 DMA_TO_DEVICE); 2087 if (ret) 2088 return ret; 2089 2090 /* Save the workarea address since it is updated in order to perform 2091 * the concatenation 2092 */ 2093 save = src.address; 2094 2095 /* Copy the ECC modulus */ 2096 ret = ccp_reverse_set_dm_area(&src, 0, ecc->mod, 0, ecc->mod_len); 2097 if (ret) 2098 goto e_src; 2099 src.address += CCP_ECC_OPERAND_SIZE; 2100 2101 /* Copy the first operand */ 2102 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.mm.operand_1, 0, 2103 ecc->u.mm.operand_1_len); 2104 if (ret) 2105 goto e_src; 2106 src.address += CCP_ECC_OPERAND_SIZE; 2107 2108 if (ecc->function != CCP_ECC_FUNCTION_MINV_384BIT) { 2109 /* Copy the second operand */ 2110 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.mm.operand_2, 0, 2111 ecc->u.mm.operand_2_len); 2112 if (ret) 2113 goto e_src; 2114 src.address += CCP_ECC_OPERAND_SIZE; 2115 } 2116 2117 /* Restore the workarea address */ 2118 src.address = save; 2119 2120 /* Prepare the output area for the operation */ 2121 ret = ccp_init_dm_workarea(&dst, cmd_q, CCP_ECC_DST_BUF_SIZE, 2122 DMA_FROM_DEVICE); 2123 if (ret) 2124 goto e_src; 2125 2126 op.soc = 1; 2127 op.src.u.dma.address = src.dma.address; 2128 op.src.u.dma.offset = 0; 2129 op.src.u.dma.length = src.length; 2130 op.dst.u.dma.address = dst.dma.address; 2131 op.dst.u.dma.offset = 0; 2132 op.dst.u.dma.length = dst.length; 2133 2134 op.u.ecc.function = cmd->u.ecc.function; 2135 2136 ret = cmd_q->ccp->vdata->perform->ecc(&op); 2137 if (ret) { 2138 cmd->engine_error = cmd_q->cmd_error; 2139 goto e_dst; 2140 } 2141 2142 ecc->ecc_result = le16_to_cpup( 2143 (const __le16 *)(dst.address + CCP_ECC_RESULT_OFFSET)); 2144 if (!(ecc->ecc_result & CCP_ECC_RESULT_SUCCESS)) { 2145 ret = -EIO; 2146 goto e_dst; 2147 } 2148 2149 /* Save the ECC result */ 2150 ccp_reverse_get_dm_area(&dst, 0, ecc->u.mm.result, 0, 2151 CCP_ECC_MODULUS_BYTES); 2152 2153 e_dst: 2154 ccp_dm_free(&dst); 2155 2156 e_src: 2157 ccp_dm_free(&src); 2158 2159 return ret; 2160 } 2161 2162 static int ccp_run_ecc_pm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) 2163 { 2164 struct ccp_ecc_engine *ecc = &cmd->u.ecc; 2165 struct ccp_dm_workarea src, dst; 2166 struct ccp_op op; 2167 int ret; 2168 u8 *save; 2169 2170 if (!ecc->u.pm.point_1.x || 2171 (ecc->u.pm.point_1.x_len > CCP_ECC_MODULUS_BYTES) || 2172 !ecc->u.pm.point_1.y || 2173 (ecc->u.pm.point_1.y_len > CCP_ECC_MODULUS_BYTES)) 2174 return -EINVAL; 2175 2176 if (ecc->function == CCP_ECC_FUNCTION_PADD_384BIT) { 2177 if (!ecc->u.pm.point_2.x || 2178 (ecc->u.pm.point_2.x_len > CCP_ECC_MODULUS_BYTES) || 2179 !ecc->u.pm.point_2.y || 2180 (ecc->u.pm.point_2.y_len > CCP_ECC_MODULUS_BYTES)) 2181 return -EINVAL; 2182 } else { 2183 if (!ecc->u.pm.domain_a || 2184 (ecc->u.pm.domain_a_len > CCP_ECC_MODULUS_BYTES)) 2185 return -EINVAL; 2186 2187 if (ecc->function == CCP_ECC_FUNCTION_PMUL_384BIT) 2188 if (!ecc->u.pm.scalar || 2189 (ecc->u.pm.scalar_len > CCP_ECC_MODULUS_BYTES)) 2190 return -EINVAL; 2191 } 2192 2193 if (!ecc->u.pm.result.x || 2194 (ecc->u.pm.result.x_len < CCP_ECC_MODULUS_BYTES) || 2195 !ecc->u.pm.result.y || 2196 (ecc->u.pm.result.y_len < CCP_ECC_MODULUS_BYTES)) 2197 return -EINVAL; 2198 2199 memset(&op, 0, sizeof(op)); 2200 op.cmd_q = cmd_q; 2201 op.jobid = CCP_NEW_JOBID(cmd_q->ccp); 2202 2203 /* Concatenate the modulus and the operands. Both the modulus and 2204 * the operands must be in little endian format. Since the input 2205 * is in big endian format it must be converted and placed in a 2206 * fixed length buffer. 2207 */ 2208 ret = ccp_init_dm_workarea(&src, cmd_q, CCP_ECC_SRC_BUF_SIZE, 2209 DMA_TO_DEVICE); 2210 if (ret) 2211 return ret; 2212 2213 /* Save the workarea address since it is updated in order to perform 2214 * the concatenation 2215 */ 2216 save = src.address; 2217 2218 /* Copy the ECC modulus */ 2219 ret = ccp_reverse_set_dm_area(&src, 0, ecc->mod, 0, ecc->mod_len); 2220 if (ret) 2221 goto e_src; 2222 src.address += CCP_ECC_OPERAND_SIZE; 2223 2224 /* Copy the first point X and Y coordinate */ 2225 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_1.x, 0, 2226 ecc->u.pm.point_1.x_len); 2227 if (ret) 2228 goto e_src; 2229 src.address += CCP_ECC_OPERAND_SIZE; 2230 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_1.y, 0, 2231 ecc->u.pm.point_1.y_len); 2232 if (ret) 2233 goto e_src; 2234 src.address += CCP_ECC_OPERAND_SIZE; 2235 2236 /* Set the first point Z coordinate to 1 */ 2237 *src.address = 0x01; 2238 src.address += CCP_ECC_OPERAND_SIZE; 2239 2240 if (ecc->function == CCP_ECC_FUNCTION_PADD_384BIT) { 2241 /* Copy the second point X and Y coordinate */ 2242 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_2.x, 0, 2243 ecc->u.pm.point_2.x_len); 2244 if (ret) 2245 goto e_src; 2246 src.address += CCP_ECC_OPERAND_SIZE; 2247 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_2.y, 0, 2248 ecc->u.pm.point_2.y_len); 2249 if (ret) 2250 goto e_src; 2251 src.address += CCP_ECC_OPERAND_SIZE; 2252 2253 /* Set the second point Z coordinate to 1 */ 2254 *src.address = 0x01; 2255 src.address += CCP_ECC_OPERAND_SIZE; 2256 } else { 2257 /* Copy the Domain "a" parameter */ 2258 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.domain_a, 0, 2259 ecc->u.pm.domain_a_len); 2260 if (ret) 2261 goto e_src; 2262 src.address += CCP_ECC_OPERAND_SIZE; 2263 2264 if (ecc->function == CCP_ECC_FUNCTION_PMUL_384BIT) { 2265 /* Copy the scalar value */ 2266 ret = ccp_reverse_set_dm_area(&src, 0, 2267 ecc->u.pm.scalar, 0, 2268 ecc->u.pm.scalar_len); 2269 if (ret) 2270 goto e_src; 2271 src.address += CCP_ECC_OPERAND_SIZE; 2272 } 2273 } 2274 2275 /* Restore the workarea address */ 2276 src.address = save; 2277 2278 /* Prepare the output area for the operation */ 2279 ret = ccp_init_dm_workarea(&dst, cmd_q, CCP_ECC_DST_BUF_SIZE, 2280 DMA_FROM_DEVICE); 2281 if (ret) 2282 goto e_src; 2283 2284 op.soc = 1; 2285 op.src.u.dma.address = src.dma.address; 2286 op.src.u.dma.offset = 0; 2287 op.src.u.dma.length = src.length; 2288 op.dst.u.dma.address = dst.dma.address; 2289 op.dst.u.dma.offset = 0; 2290 op.dst.u.dma.length = dst.length; 2291 2292 op.u.ecc.function = cmd->u.ecc.function; 2293 2294 ret = cmd_q->ccp->vdata->perform->ecc(&op); 2295 if (ret) { 2296 cmd->engine_error = cmd_q->cmd_error; 2297 goto e_dst; 2298 } 2299 2300 ecc->ecc_result = le16_to_cpup( 2301 (const __le16 *)(dst.address + CCP_ECC_RESULT_OFFSET)); 2302 if (!(ecc->ecc_result & CCP_ECC_RESULT_SUCCESS)) { 2303 ret = -EIO; 2304 goto e_dst; 2305 } 2306 2307 /* Save the workarea address since it is updated as we walk through 2308 * to copy the point math result 2309 */ 2310 save = dst.address; 2311 2312 /* Save the ECC result X and Y coordinates */ 2313 ccp_reverse_get_dm_area(&dst, 0, ecc->u.pm.result.x, 0, 2314 CCP_ECC_MODULUS_BYTES); 2315 dst.address += CCP_ECC_OUTPUT_SIZE; 2316 ccp_reverse_get_dm_area(&dst, 0, ecc->u.pm.result.y, 0, 2317 CCP_ECC_MODULUS_BYTES); 2318 dst.address += CCP_ECC_OUTPUT_SIZE; 2319 2320 /* Restore the workarea address */ 2321 dst.address = save; 2322 2323 e_dst: 2324 ccp_dm_free(&dst); 2325 2326 e_src: 2327 ccp_dm_free(&src); 2328 2329 return ret; 2330 } 2331 2332 static int ccp_run_ecc_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) 2333 { 2334 struct ccp_ecc_engine *ecc = &cmd->u.ecc; 2335 2336 ecc->ecc_result = 0; 2337 2338 if (!ecc->mod || 2339 (ecc->mod_len > CCP_ECC_MODULUS_BYTES)) 2340 return -EINVAL; 2341 2342 switch (ecc->function) { 2343 case CCP_ECC_FUNCTION_MMUL_384BIT: 2344 case CCP_ECC_FUNCTION_MADD_384BIT: 2345 case CCP_ECC_FUNCTION_MINV_384BIT: 2346 return ccp_run_ecc_mm_cmd(cmd_q, cmd); 2347 2348 case CCP_ECC_FUNCTION_PADD_384BIT: 2349 case CCP_ECC_FUNCTION_PMUL_384BIT: 2350 case CCP_ECC_FUNCTION_PDBL_384BIT: 2351 return ccp_run_ecc_pm_cmd(cmd_q, cmd); 2352 2353 default: 2354 return -EINVAL; 2355 } 2356 } 2357 2358 int ccp_run_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) 2359 { 2360 int ret; 2361 2362 cmd->engine_error = 0; 2363 cmd_q->cmd_error = 0; 2364 cmd_q->int_rcvd = 0; 2365 cmd_q->free_slots = cmd_q->ccp->vdata->perform->get_free_slots(cmd_q); 2366 2367 switch (cmd->engine) { 2368 case CCP_ENGINE_AES: 2369 ret = ccp_run_aes_cmd(cmd_q, cmd); 2370 break; 2371 case CCP_ENGINE_XTS_AES_128: 2372 ret = ccp_run_xts_aes_cmd(cmd_q, cmd); 2373 break; 2374 case CCP_ENGINE_DES3: 2375 ret = ccp_run_des3_cmd(cmd_q, cmd); 2376 break; 2377 case CCP_ENGINE_SHA: 2378 ret = ccp_run_sha_cmd(cmd_q, cmd); 2379 break; 2380 case CCP_ENGINE_RSA: 2381 ret = ccp_run_rsa_cmd(cmd_q, cmd); 2382 break; 2383 case CCP_ENGINE_PASSTHRU: 2384 if (cmd->flags & CCP_CMD_PASSTHRU_NO_DMA_MAP) 2385 ret = ccp_run_passthru_nomap_cmd(cmd_q, cmd); 2386 else 2387 ret = ccp_run_passthru_cmd(cmd_q, cmd); 2388 break; 2389 case CCP_ENGINE_ECC: 2390 ret = ccp_run_ecc_cmd(cmd_q, cmd); 2391 break; 2392 default: 2393 ret = -EINVAL; 2394 } 2395 2396 return ret; 2397 } 2398