1 /* 2 * AMD Cryptographic Coprocessor (CCP) driver 3 * 4 * Copyright (C) 2013,2016 Advanced Micro Devices, Inc. 5 * 6 * Author: Tom Lendacky <thomas.lendacky@amd.com> 7 * Author: Gary R Hook <gary.hook@amd.com> 8 * 9 * This program is free software; you can redistribute it and/or modify 10 * it under the terms of the GNU General Public License version 2 as 11 * published by the Free Software Foundation. 12 */ 13 14 #include <linux/module.h> 15 #include <linux/kernel.h> 16 #include <linux/pci.h> 17 #include <linux/interrupt.h> 18 #include <crypto/scatterwalk.h> 19 #include <crypto/des.h> 20 #include <linux/ccp.h> 21 22 #include "ccp-dev.h" 23 24 /* SHA initial context values */ 25 static const __be32 ccp_sha1_init[SHA1_DIGEST_SIZE / sizeof(__be32)] = { 26 cpu_to_be32(SHA1_H0), cpu_to_be32(SHA1_H1), 27 cpu_to_be32(SHA1_H2), cpu_to_be32(SHA1_H3), 28 cpu_to_be32(SHA1_H4), 29 }; 30 31 static const __be32 ccp_sha224_init[SHA256_DIGEST_SIZE / sizeof(__be32)] = { 32 cpu_to_be32(SHA224_H0), cpu_to_be32(SHA224_H1), 33 cpu_to_be32(SHA224_H2), cpu_to_be32(SHA224_H3), 34 cpu_to_be32(SHA224_H4), cpu_to_be32(SHA224_H5), 35 cpu_to_be32(SHA224_H6), cpu_to_be32(SHA224_H7), 36 }; 37 38 static const __be32 ccp_sha256_init[SHA256_DIGEST_SIZE / sizeof(__be32)] = { 39 cpu_to_be32(SHA256_H0), cpu_to_be32(SHA256_H1), 40 cpu_to_be32(SHA256_H2), cpu_to_be32(SHA256_H3), 41 cpu_to_be32(SHA256_H4), cpu_to_be32(SHA256_H5), 42 cpu_to_be32(SHA256_H6), cpu_to_be32(SHA256_H7), 43 }; 44 45 static const __be64 ccp_sha384_init[SHA512_DIGEST_SIZE / sizeof(__be64)] = { 46 cpu_to_be64(SHA384_H0), cpu_to_be64(SHA384_H1), 47 cpu_to_be64(SHA384_H2), cpu_to_be64(SHA384_H3), 48 cpu_to_be64(SHA384_H4), cpu_to_be64(SHA384_H5), 49 cpu_to_be64(SHA384_H6), cpu_to_be64(SHA384_H7), 50 }; 51 52 static const __be64 ccp_sha512_init[SHA512_DIGEST_SIZE / sizeof(__be64)] = { 53 cpu_to_be64(SHA512_H0), cpu_to_be64(SHA512_H1), 54 cpu_to_be64(SHA512_H2), cpu_to_be64(SHA512_H3), 55 cpu_to_be64(SHA512_H4), cpu_to_be64(SHA512_H5), 56 cpu_to_be64(SHA512_H6), cpu_to_be64(SHA512_H7), 57 }; 58 59 #define CCP_NEW_JOBID(ccp) ((ccp->vdata->version == CCP_VERSION(3, 0)) ? \ 60 ccp_gen_jobid(ccp) : 0) 61 62 static u32 ccp_gen_jobid(struct ccp_device *ccp) 63 { 64 return atomic_inc_return(&ccp->current_id) & CCP_JOBID_MASK; 65 } 66 67 static void ccp_sg_free(struct ccp_sg_workarea *wa) 68 { 69 if (wa->dma_count) 70 dma_unmap_sg(wa->dma_dev, wa->dma_sg, wa->nents, wa->dma_dir); 71 72 wa->dma_count = 0; 73 } 74 75 static int ccp_init_sg_workarea(struct ccp_sg_workarea *wa, struct device *dev, 76 struct scatterlist *sg, u64 len, 77 enum dma_data_direction dma_dir) 78 { 79 memset(wa, 0, sizeof(*wa)); 80 81 wa->sg = sg; 82 if (!sg) 83 return 0; 84 85 wa->nents = sg_nents_for_len(sg, len); 86 if (wa->nents < 0) 87 return wa->nents; 88 89 wa->bytes_left = len; 90 wa->sg_used = 0; 91 92 if (len == 0) 93 return 0; 94 95 if (dma_dir == DMA_NONE) 96 return 0; 97 98 wa->dma_sg = sg; 99 wa->dma_dev = dev; 100 wa->dma_dir = dma_dir; 101 wa->dma_count = dma_map_sg(dev, sg, wa->nents, dma_dir); 102 if (!wa->dma_count) 103 return -ENOMEM; 104 105 return 0; 106 } 107 108 static void ccp_update_sg_workarea(struct ccp_sg_workarea *wa, unsigned int len) 109 { 110 unsigned int nbytes = min_t(u64, len, wa->bytes_left); 111 112 if (!wa->sg) 113 return; 114 115 wa->sg_used += nbytes; 116 wa->bytes_left -= nbytes; 117 if (wa->sg_used == wa->sg->length) { 118 wa->sg = sg_next(wa->sg); 119 wa->sg_used = 0; 120 } 121 } 122 123 static void ccp_dm_free(struct ccp_dm_workarea *wa) 124 { 125 if (wa->length <= CCP_DMAPOOL_MAX_SIZE) { 126 if (wa->address) 127 dma_pool_free(wa->dma_pool, wa->address, 128 wa->dma.address); 129 } else { 130 if (wa->dma.address) 131 dma_unmap_single(wa->dev, wa->dma.address, wa->length, 132 wa->dma.dir); 133 kfree(wa->address); 134 } 135 136 wa->address = NULL; 137 wa->dma.address = 0; 138 } 139 140 static int ccp_init_dm_workarea(struct ccp_dm_workarea *wa, 141 struct ccp_cmd_queue *cmd_q, 142 unsigned int len, 143 enum dma_data_direction dir) 144 { 145 memset(wa, 0, sizeof(*wa)); 146 147 if (!len) 148 return 0; 149 150 wa->dev = cmd_q->ccp->dev; 151 wa->length = len; 152 153 if (len <= CCP_DMAPOOL_MAX_SIZE) { 154 wa->dma_pool = cmd_q->dma_pool; 155 156 wa->address = dma_pool_alloc(wa->dma_pool, GFP_KERNEL, 157 &wa->dma.address); 158 if (!wa->address) 159 return -ENOMEM; 160 161 wa->dma.length = CCP_DMAPOOL_MAX_SIZE; 162 163 memset(wa->address, 0, CCP_DMAPOOL_MAX_SIZE); 164 } else { 165 wa->address = kzalloc(len, GFP_KERNEL); 166 if (!wa->address) 167 return -ENOMEM; 168 169 wa->dma.address = dma_map_single(wa->dev, wa->address, len, 170 dir); 171 if (!wa->dma.address) 172 return -ENOMEM; 173 174 wa->dma.length = len; 175 } 176 wa->dma.dir = dir; 177 178 return 0; 179 } 180 181 static void ccp_set_dm_area(struct ccp_dm_workarea *wa, unsigned int wa_offset, 182 struct scatterlist *sg, unsigned int sg_offset, 183 unsigned int len) 184 { 185 WARN_ON(!wa->address); 186 187 scatterwalk_map_and_copy(wa->address + wa_offset, sg, sg_offset, len, 188 0); 189 } 190 191 static void ccp_get_dm_area(struct ccp_dm_workarea *wa, unsigned int wa_offset, 192 struct scatterlist *sg, unsigned int sg_offset, 193 unsigned int len) 194 { 195 WARN_ON(!wa->address); 196 197 scatterwalk_map_and_copy(wa->address + wa_offset, sg, sg_offset, len, 198 1); 199 } 200 201 static int ccp_reverse_set_dm_area(struct ccp_dm_workarea *wa, 202 unsigned int wa_offset, 203 struct scatterlist *sg, 204 unsigned int sg_offset, 205 unsigned int len) 206 { 207 u8 *p, *q; 208 209 ccp_set_dm_area(wa, wa_offset, sg, sg_offset, len); 210 211 p = wa->address + wa_offset; 212 q = p + len - 1; 213 while (p < q) { 214 *p = *p ^ *q; 215 *q = *p ^ *q; 216 *p = *p ^ *q; 217 p++; 218 q--; 219 } 220 return 0; 221 } 222 223 static void ccp_reverse_get_dm_area(struct ccp_dm_workarea *wa, 224 unsigned int wa_offset, 225 struct scatterlist *sg, 226 unsigned int sg_offset, 227 unsigned int len) 228 { 229 u8 *p, *q; 230 231 p = wa->address + wa_offset; 232 q = p + len - 1; 233 while (p < q) { 234 *p = *p ^ *q; 235 *q = *p ^ *q; 236 *p = *p ^ *q; 237 p++; 238 q--; 239 } 240 241 ccp_get_dm_area(wa, wa_offset, sg, sg_offset, len); 242 } 243 244 static void ccp_free_data(struct ccp_data *data, struct ccp_cmd_queue *cmd_q) 245 { 246 ccp_dm_free(&data->dm_wa); 247 ccp_sg_free(&data->sg_wa); 248 } 249 250 static int ccp_init_data(struct ccp_data *data, struct ccp_cmd_queue *cmd_q, 251 struct scatterlist *sg, u64 sg_len, 252 unsigned int dm_len, 253 enum dma_data_direction dir) 254 { 255 int ret; 256 257 memset(data, 0, sizeof(*data)); 258 259 ret = ccp_init_sg_workarea(&data->sg_wa, cmd_q->ccp->dev, sg, sg_len, 260 dir); 261 if (ret) 262 goto e_err; 263 264 ret = ccp_init_dm_workarea(&data->dm_wa, cmd_q, dm_len, dir); 265 if (ret) 266 goto e_err; 267 268 return 0; 269 270 e_err: 271 ccp_free_data(data, cmd_q); 272 273 return ret; 274 } 275 276 static unsigned int ccp_queue_buf(struct ccp_data *data, unsigned int from) 277 { 278 struct ccp_sg_workarea *sg_wa = &data->sg_wa; 279 struct ccp_dm_workarea *dm_wa = &data->dm_wa; 280 unsigned int buf_count, nbytes; 281 282 /* Clear the buffer if setting it */ 283 if (!from) 284 memset(dm_wa->address, 0, dm_wa->length); 285 286 if (!sg_wa->sg) 287 return 0; 288 289 /* Perform the copy operation 290 * nbytes will always be <= UINT_MAX because dm_wa->length is 291 * an unsigned int 292 */ 293 nbytes = min_t(u64, sg_wa->bytes_left, dm_wa->length); 294 scatterwalk_map_and_copy(dm_wa->address, sg_wa->sg, sg_wa->sg_used, 295 nbytes, from); 296 297 /* Update the structures and generate the count */ 298 buf_count = 0; 299 while (sg_wa->bytes_left && (buf_count < dm_wa->length)) { 300 nbytes = min(sg_wa->sg->length - sg_wa->sg_used, 301 dm_wa->length - buf_count); 302 nbytes = min_t(u64, sg_wa->bytes_left, nbytes); 303 304 buf_count += nbytes; 305 ccp_update_sg_workarea(sg_wa, nbytes); 306 } 307 308 return buf_count; 309 } 310 311 static unsigned int ccp_fill_queue_buf(struct ccp_data *data) 312 { 313 return ccp_queue_buf(data, 0); 314 } 315 316 static unsigned int ccp_empty_queue_buf(struct ccp_data *data) 317 { 318 return ccp_queue_buf(data, 1); 319 } 320 321 static void ccp_prepare_data(struct ccp_data *src, struct ccp_data *dst, 322 struct ccp_op *op, unsigned int block_size, 323 bool blocksize_op) 324 { 325 unsigned int sg_src_len, sg_dst_len, op_len; 326 327 /* The CCP can only DMA from/to one address each per operation. This 328 * requires that we find the smallest DMA area between the source 329 * and destination. The resulting len values will always be <= UINT_MAX 330 * because the dma length is an unsigned int. 331 */ 332 sg_src_len = sg_dma_len(src->sg_wa.sg) - src->sg_wa.sg_used; 333 sg_src_len = min_t(u64, src->sg_wa.bytes_left, sg_src_len); 334 335 if (dst) { 336 sg_dst_len = sg_dma_len(dst->sg_wa.sg) - dst->sg_wa.sg_used; 337 sg_dst_len = min_t(u64, src->sg_wa.bytes_left, sg_dst_len); 338 op_len = min(sg_src_len, sg_dst_len); 339 } else { 340 op_len = sg_src_len; 341 } 342 343 /* The data operation length will be at least block_size in length 344 * or the smaller of available sg room remaining for the source or 345 * the destination 346 */ 347 op_len = max(op_len, block_size); 348 349 /* Unless we have to buffer data, there's no reason to wait */ 350 op->soc = 0; 351 352 if (sg_src_len < block_size) { 353 /* Not enough data in the sg element, so it 354 * needs to be buffered into a blocksize chunk 355 */ 356 int cp_len = ccp_fill_queue_buf(src); 357 358 op->soc = 1; 359 op->src.u.dma.address = src->dm_wa.dma.address; 360 op->src.u.dma.offset = 0; 361 op->src.u.dma.length = (blocksize_op) ? block_size : cp_len; 362 } else { 363 /* Enough data in the sg element, but we need to 364 * adjust for any previously copied data 365 */ 366 op->src.u.dma.address = sg_dma_address(src->sg_wa.sg); 367 op->src.u.dma.offset = src->sg_wa.sg_used; 368 op->src.u.dma.length = op_len & ~(block_size - 1); 369 370 ccp_update_sg_workarea(&src->sg_wa, op->src.u.dma.length); 371 } 372 373 if (dst) { 374 if (sg_dst_len < block_size) { 375 /* Not enough room in the sg element or we're on the 376 * last piece of data (when using padding), so the 377 * output needs to be buffered into a blocksize chunk 378 */ 379 op->soc = 1; 380 op->dst.u.dma.address = dst->dm_wa.dma.address; 381 op->dst.u.dma.offset = 0; 382 op->dst.u.dma.length = op->src.u.dma.length; 383 } else { 384 /* Enough room in the sg element, but we need to 385 * adjust for any previously used area 386 */ 387 op->dst.u.dma.address = sg_dma_address(dst->sg_wa.sg); 388 op->dst.u.dma.offset = dst->sg_wa.sg_used; 389 op->dst.u.dma.length = op->src.u.dma.length; 390 } 391 } 392 } 393 394 static void ccp_process_data(struct ccp_data *src, struct ccp_data *dst, 395 struct ccp_op *op) 396 { 397 op->init = 0; 398 399 if (dst) { 400 if (op->dst.u.dma.address == dst->dm_wa.dma.address) 401 ccp_empty_queue_buf(dst); 402 else 403 ccp_update_sg_workarea(&dst->sg_wa, 404 op->dst.u.dma.length); 405 } 406 } 407 408 static int ccp_copy_to_from_sb(struct ccp_cmd_queue *cmd_q, 409 struct ccp_dm_workarea *wa, u32 jobid, u32 sb, 410 u32 byte_swap, bool from) 411 { 412 struct ccp_op op; 413 414 memset(&op, 0, sizeof(op)); 415 416 op.cmd_q = cmd_q; 417 op.jobid = jobid; 418 op.eom = 1; 419 420 if (from) { 421 op.soc = 1; 422 op.src.type = CCP_MEMTYPE_SB; 423 op.src.u.sb = sb; 424 op.dst.type = CCP_MEMTYPE_SYSTEM; 425 op.dst.u.dma.address = wa->dma.address; 426 op.dst.u.dma.length = wa->length; 427 } else { 428 op.src.type = CCP_MEMTYPE_SYSTEM; 429 op.src.u.dma.address = wa->dma.address; 430 op.src.u.dma.length = wa->length; 431 op.dst.type = CCP_MEMTYPE_SB; 432 op.dst.u.sb = sb; 433 } 434 435 op.u.passthru.byte_swap = byte_swap; 436 437 return cmd_q->ccp->vdata->perform->passthru(&op); 438 } 439 440 static int ccp_copy_to_sb(struct ccp_cmd_queue *cmd_q, 441 struct ccp_dm_workarea *wa, u32 jobid, u32 sb, 442 u32 byte_swap) 443 { 444 return ccp_copy_to_from_sb(cmd_q, wa, jobid, sb, byte_swap, false); 445 } 446 447 static int ccp_copy_from_sb(struct ccp_cmd_queue *cmd_q, 448 struct ccp_dm_workarea *wa, u32 jobid, u32 sb, 449 u32 byte_swap) 450 { 451 return ccp_copy_to_from_sb(cmd_q, wa, jobid, sb, byte_swap, true); 452 } 453 454 static int ccp_run_aes_cmac_cmd(struct ccp_cmd_queue *cmd_q, 455 struct ccp_cmd *cmd) 456 { 457 struct ccp_aes_engine *aes = &cmd->u.aes; 458 struct ccp_dm_workarea key, ctx; 459 struct ccp_data src; 460 struct ccp_op op; 461 unsigned int dm_offset; 462 int ret; 463 464 if (!((aes->key_len == AES_KEYSIZE_128) || 465 (aes->key_len == AES_KEYSIZE_192) || 466 (aes->key_len == AES_KEYSIZE_256))) 467 return -EINVAL; 468 469 if (aes->src_len & (AES_BLOCK_SIZE - 1)) 470 return -EINVAL; 471 472 if (aes->iv_len != AES_BLOCK_SIZE) 473 return -EINVAL; 474 475 if (!aes->key || !aes->iv || !aes->src) 476 return -EINVAL; 477 478 if (aes->cmac_final) { 479 if (aes->cmac_key_len != AES_BLOCK_SIZE) 480 return -EINVAL; 481 482 if (!aes->cmac_key) 483 return -EINVAL; 484 } 485 486 BUILD_BUG_ON(CCP_AES_KEY_SB_COUNT != 1); 487 BUILD_BUG_ON(CCP_AES_CTX_SB_COUNT != 1); 488 489 ret = -EIO; 490 memset(&op, 0, sizeof(op)); 491 op.cmd_q = cmd_q; 492 op.jobid = CCP_NEW_JOBID(cmd_q->ccp); 493 op.sb_key = cmd_q->sb_key; 494 op.sb_ctx = cmd_q->sb_ctx; 495 op.init = 1; 496 op.u.aes.type = aes->type; 497 op.u.aes.mode = aes->mode; 498 op.u.aes.action = aes->action; 499 500 /* All supported key sizes fit in a single (32-byte) SB entry 501 * and must be in little endian format. Use the 256-bit byte 502 * swap passthru option to convert from big endian to little 503 * endian. 504 */ 505 ret = ccp_init_dm_workarea(&key, cmd_q, 506 CCP_AES_KEY_SB_COUNT * CCP_SB_BYTES, 507 DMA_TO_DEVICE); 508 if (ret) 509 return ret; 510 511 dm_offset = CCP_SB_BYTES - aes->key_len; 512 ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len); 513 ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key, 514 CCP_PASSTHRU_BYTESWAP_256BIT); 515 if (ret) { 516 cmd->engine_error = cmd_q->cmd_error; 517 goto e_key; 518 } 519 520 /* The AES context fits in a single (32-byte) SB entry and 521 * must be in little endian format. Use the 256-bit byte swap 522 * passthru option to convert from big endian to little endian. 523 */ 524 ret = ccp_init_dm_workarea(&ctx, cmd_q, 525 CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES, 526 DMA_BIDIRECTIONAL); 527 if (ret) 528 goto e_key; 529 530 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE; 531 ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len); 532 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, 533 CCP_PASSTHRU_BYTESWAP_256BIT); 534 if (ret) { 535 cmd->engine_error = cmd_q->cmd_error; 536 goto e_ctx; 537 } 538 539 /* Send data to the CCP AES engine */ 540 ret = ccp_init_data(&src, cmd_q, aes->src, aes->src_len, 541 AES_BLOCK_SIZE, DMA_TO_DEVICE); 542 if (ret) 543 goto e_ctx; 544 545 while (src.sg_wa.bytes_left) { 546 ccp_prepare_data(&src, NULL, &op, AES_BLOCK_SIZE, true); 547 if (aes->cmac_final && !src.sg_wa.bytes_left) { 548 op.eom = 1; 549 550 /* Push the K1/K2 key to the CCP now */ 551 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, 552 op.sb_ctx, 553 CCP_PASSTHRU_BYTESWAP_256BIT); 554 if (ret) { 555 cmd->engine_error = cmd_q->cmd_error; 556 goto e_src; 557 } 558 559 ccp_set_dm_area(&ctx, 0, aes->cmac_key, 0, 560 aes->cmac_key_len); 561 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, 562 CCP_PASSTHRU_BYTESWAP_256BIT); 563 if (ret) { 564 cmd->engine_error = cmd_q->cmd_error; 565 goto e_src; 566 } 567 } 568 569 ret = cmd_q->ccp->vdata->perform->aes(&op); 570 if (ret) { 571 cmd->engine_error = cmd_q->cmd_error; 572 goto e_src; 573 } 574 575 ccp_process_data(&src, NULL, &op); 576 } 577 578 /* Retrieve the AES context - convert from LE to BE using 579 * 32-byte (256-bit) byteswapping 580 */ 581 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, 582 CCP_PASSTHRU_BYTESWAP_256BIT); 583 if (ret) { 584 cmd->engine_error = cmd_q->cmd_error; 585 goto e_src; 586 } 587 588 /* ...but we only need AES_BLOCK_SIZE bytes */ 589 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE; 590 ccp_get_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len); 591 592 e_src: 593 ccp_free_data(&src, cmd_q); 594 595 e_ctx: 596 ccp_dm_free(&ctx); 597 598 e_key: 599 ccp_dm_free(&key); 600 601 return ret; 602 } 603 604 static int ccp_run_aes_gcm_cmd(struct ccp_cmd_queue *cmd_q, 605 struct ccp_cmd *cmd) 606 { 607 struct ccp_aes_engine *aes = &cmd->u.aes; 608 struct ccp_dm_workarea key, ctx, final_wa, tag; 609 struct ccp_data src, dst; 610 struct ccp_data aad; 611 struct ccp_op op; 612 613 unsigned long long *final; 614 unsigned int dm_offset; 615 unsigned int ilen; 616 bool in_place = true; /* Default value */ 617 int ret; 618 619 struct scatterlist *p_inp, sg_inp[2]; 620 struct scatterlist *p_tag, sg_tag[2]; 621 struct scatterlist *p_outp, sg_outp[2]; 622 struct scatterlist *p_aad; 623 624 if (!aes->iv) 625 return -EINVAL; 626 627 if (!((aes->key_len == AES_KEYSIZE_128) || 628 (aes->key_len == AES_KEYSIZE_192) || 629 (aes->key_len == AES_KEYSIZE_256))) 630 return -EINVAL; 631 632 if (!aes->key) /* Gotta have a key SGL */ 633 return -EINVAL; 634 635 /* First, decompose the source buffer into AAD & PT, 636 * and the destination buffer into AAD, CT & tag, or 637 * the input into CT & tag. 638 * It is expected that the input and output SGs will 639 * be valid, even if the AAD and input lengths are 0. 640 */ 641 p_aad = aes->src; 642 p_inp = scatterwalk_ffwd(sg_inp, aes->src, aes->aad_len); 643 p_outp = scatterwalk_ffwd(sg_outp, aes->dst, aes->aad_len); 644 if (aes->action == CCP_AES_ACTION_ENCRYPT) { 645 ilen = aes->src_len; 646 p_tag = scatterwalk_ffwd(sg_tag, p_outp, ilen); 647 } else { 648 /* Input length for decryption includes tag */ 649 ilen = aes->src_len - AES_BLOCK_SIZE; 650 p_tag = scatterwalk_ffwd(sg_tag, p_inp, ilen); 651 } 652 653 memset(&op, 0, sizeof(op)); 654 op.cmd_q = cmd_q; 655 op.jobid = CCP_NEW_JOBID(cmd_q->ccp); 656 op.sb_key = cmd_q->sb_key; /* Pre-allocated */ 657 op.sb_ctx = cmd_q->sb_ctx; /* Pre-allocated */ 658 op.init = 1; 659 op.u.aes.type = aes->type; 660 661 /* Copy the key to the LSB */ 662 ret = ccp_init_dm_workarea(&key, cmd_q, 663 CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES, 664 DMA_TO_DEVICE); 665 if (ret) 666 return ret; 667 668 dm_offset = CCP_SB_BYTES - aes->key_len; 669 ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len); 670 ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key, 671 CCP_PASSTHRU_BYTESWAP_256BIT); 672 if (ret) { 673 cmd->engine_error = cmd_q->cmd_error; 674 goto e_key; 675 } 676 677 /* Copy the context (IV) to the LSB. 678 * There is an assumption here that the IV is 96 bits in length, plus 679 * a nonce of 32 bits. If no IV is present, use a zeroed buffer. 680 */ 681 ret = ccp_init_dm_workarea(&ctx, cmd_q, 682 CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES, 683 DMA_BIDIRECTIONAL); 684 if (ret) 685 goto e_key; 686 687 dm_offset = CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES - aes->iv_len; 688 ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len); 689 690 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, 691 CCP_PASSTHRU_BYTESWAP_256BIT); 692 if (ret) { 693 cmd->engine_error = cmd_q->cmd_error; 694 goto e_ctx; 695 } 696 697 op.init = 1; 698 if (aes->aad_len > 0) { 699 /* Step 1: Run a GHASH over the Additional Authenticated Data */ 700 ret = ccp_init_data(&aad, cmd_q, p_aad, aes->aad_len, 701 AES_BLOCK_SIZE, 702 DMA_TO_DEVICE); 703 if (ret) 704 goto e_ctx; 705 706 op.u.aes.mode = CCP_AES_MODE_GHASH; 707 op.u.aes.action = CCP_AES_GHASHAAD; 708 709 while (aad.sg_wa.bytes_left) { 710 ccp_prepare_data(&aad, NULL, &op, AES_BLOCK_SIZE, true); 711 712 ret = cmd_q->ccp->vdata->perform->aes(&op); 713 if (ret) { 714 cmd->engine_error = cmd_q->cmd_error; 715 goto e_aad; 716 } 717 718 ccp_process_data(&aad, NULL, &op); 719 op.init = 0; 720 } 721 } 722 723 op.u.aes.mode = CCP_AES_MODE_GCTR; 724 op.u.aes.action = aes->action; 725 726 if (ilen > 0) { 727 /* Step 2: Run a GCTR over the plaintext */ 728 in_place = (sg_virt(p_inp) == sg_virt(p_outp)) ? true : false; 729 730 ret = ccp_init_data(&src, cmd_q, p_inp, ilen, 731 AES_BLOCK_SIZE, 732 in_place ? DMA_BIDIRECTIONAL 733 : DMA_TO_DEVICE); 734 if (ret) 735 goto e_ctx; 736 737 if (in_place) { 738 dst = src; 739 } else { 740 ret = ccp_init_data(&dst, cmd_q, p_outp, ilen, 741 AES_BLOCK_SIZE, DMA_FROM_DEVICE); 742 if (ret) 743 goto e_src; 744 } 745 746 op.soc = 0; 747 op.eom = 0; 748 op.init = 1; 749 while (src.sg_wa.bytes_left) { 750 ccp_prepare_data(&src, &dst, &op, AES_BLOCK_SIZE, true); 751 if (!src.sg_wa.bytes_left) { 752 unsigned int nbytes = aes->src_len 753 % AES_BLOCK_SIZE; 754 755 if (nbytes) { 756 op.eom = 1; 757 op.u.aes.size = (nbytes * 8) - 1; 758 } 759 } 760 761 ret = cmd_q->ccp->vdata->perform->aes(&op); 762 if (ret) { 763 cmd->engine_error = cmd_q->cmd_error; 764 goto e_dst; 765 } 766 767 ccp_process_data(&src, &dst, &op); 768 op.init = 0; 769 } 770 } 771 772 /* Step 3: Update the IV portion of the context with the original IV */ 773 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, 774 CCP_PASSTHRU_BYTESWAP_256BIT); 775 if (ret) { 776 cmd->engine_error = cmd_q->cmd_error; 777 goto e_dst; 778 } 779 780 ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len); 781 782 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, 783 CCP_PASSTHRU_BYTESWAP_256BIT); 784 if (ret) { 785 cmd->engine_error = cmd_q->cmd_error; 786 goto e_dst; 787 } 788 789 /* Step 4: Concatenate the lengths of the AAD and source, and 790 * hash that 16 byte buffer. 791 */ 792 ret = ccp_init_dm_workarea(&final_wa, cmd_q, AES_BLOCK_SIZE, 793 DMA_BIDIRECTIONAL); 794 if (ret) 795 goto e_dst; 796 final = (unsigned long long *) final_wa.address; 797 final[0] = cpu_to_be64(aes->aad_len * 8); 798 final[1] = cpu_to_be64(ilen * 8); 799 800 op.u.aes.mode = CCP_AES_MODE_GHASH; 801 op.u.aes.action = CCP_AES_GHASHFINAL; 802 op.src.type = CCP_MEMTYPE_SYSTEM; 803 op.src.u.dma.address = final_wa.dma.address; 804 op.src.u.dma.length = AES_BLOCK_SIZE; 805 op.dst.type = CCP_MEMTYPE_SYSTEM; 806 op.dst.u.dma.address = final_wa.dma.address; 807 op.dst.u.dma.length = AES_BLOCK_SIZE; 808 op.eom = 1; 809 op.u.aes.size = 0; 810 ret = cmd_q->ccp->vdata->perform->aes(&op); 811 if (ret) 812 goto e_dst; 813 814 if (aes->action == CCP_AES_ACTION_ENCRYPT) { 815 /* Put the ciphered tag after the ciphertext. */ 816 ccp_get_dm_area(&final_wa, 0, p_tag, 0, AES_BLOCK_SIZE); 817 } else { 818 /* Does this ciphered tag match the input? */ 819 ret = ccp_init_dm_workarea(&tag, cmd_q, AES_BLOCK_SIZE, 820 DMA_BIDIRECTIONAL); 821 if (ret) 822 goto e_tag; 823 ccp_set_dm_area(&tag, 0, p_tag, 0, AES_BLOCK_SIZE); 824 825 ret = memcmp(tag.address, final_wa.address, AES_BLOCK_SIZE); 826 ccp_dm_free(&tag); 827 } 828 829 e_tag: 830 ccp_dm_free(&final_wa); 831 832 e_dst: 833 if (aes->src_len && !in_place) 834 ccp_free_data(&dst, cmd_q); 835 836 e_src: 837 if (aes->src_len) 838 ccp_free_data(&src, cmd_q); 839 840 e_aad: 841 if (aes->aad_len) 842 ccp_free_data(&aad, cmd_q); 843 844 e_ctx: 845 ccp_dm_free(&ctx); 846 847 e_key: 848 ccp_dm_free(&key); 849 850 return ret; 851 } 852 853 static int ccp_run_aes_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) 854 { 855 struct ccp_aes_engine *aes = &cmd->u.aes; 856 struct ccp_dm_workarea key, ctx; 857 struct ccp_data src, dst; 858 struct ccp_op op; 859 unsigned int dm_offset; 860 bool in_place = false; 861 int ret; 862 863 if (aes->mode == CCP_AES_MODE_CMAC) 864 return ccp_run_aes_cmac_cmd(cmd_q, cmd); 865 866 if (aes->mode == CCP_AES_MODE_GCM) 867 return ccp_run_aes_gcm_cmd(cmd_q, cmd); 868 869 if (!((aes->key_len == AES_KEYSIZE_128) || 870 (aes->key_len == AES_KEYSIZE_192) || 871 (aes->key_len == AES_KEYSIZE_256))) 872 return -EINVAL; 873 874 if (((aes->mode == CCP_AES_MODE_ECB) || 875 (aes->mode == CCP_AES_MODE_CBC) || 876 (aes->mode == CCP_AES_MODE_CFB)) && 877 (aes->src_len & (AES_BLOCK_SIZE - 1))) 878 return -EINVAL; 879 880 if (!aes->key || !aes->src || !aes->dst) 881 return -EINVAL; 882 883 if (aes->mode != CCP_AES_MODE_ECB) { 884 if (aes->iv_len != AES_BLOCK_SIZE) 885 return -EINVAL; 886 887 if (!aes->iv) 888 return -EINVAL; 889 } 890 891 BUILD_BUG_ON(CCP_AES_KEY_SB_COUNT != 1); 892 BUILD_BUG_ON(CCP_AES_CTX_SB_COUNT != 1); 893 894 ret = -EIO; 895 memset(&op, 0, sizeof(op)); 896 op.cmd_q = cmd_q; 897 op.jobid = CCP_NEW_JOBID(cmd_q->ccp); 898 op.sb_key = cmd_q->sb_key; 899 op.sb_ctx = cmd_q->sb_ctx; 900 op.init = (aes->mode == CCP_AES_MODE_ECB) ? 0 : 1; 901 op.u.aes.type = aes->type; 902 op.u.aes.mode = aes->mode; 903 op.u.aes.action = aes->action; 904 905 /* All supported key sizes fit in a single (32-byte) SB entry 906 * and must be in little endian format. Use the 256-bit byte 907 * swap passthru option to convert from big endian to little 908 * endian. 909 */ 910 ret = ccp_init_dm_workarea(&key, cmd_q, 911 CCP_AES_KEY_SB_COUNT * CCP_SB_BYTES, 912 DMA_TO_DEVICE); 913 if (ret) 914 return ret; 915 916 dm_offset = CCP_SB_BYTES - aes->key_len; 917 ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len); 918 ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key, 919 CCP_PASSTHRU_BYTESWAP_256BIT); 920 if (ret) { 921 cmd->engine_error = cmd_q->cmd_error; 922 goto e_key; 923 } 924 925 /* The AES context fits in a single (32-byte) SB entry and 926 * must be in little endian format. Use the 256-bit byte swap 927 * passthru option to convert from big endian to little endian. 928 */ 929 ret = ccp_init_dm_workarea(&ctx, cmd_q, 930 CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES, 931 DMA_BIDIRECTIONAL); 932 if (ret) 933 goto e_key; 934 935 if (aes->mode != CCP_AES_MODE_ECB) { 936 /* Load the AES context - convert to LE */ 937 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE; 938 ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len); 939 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, 940 CCP_PASSTHRU_BYTESWAP_256BIT); 941 if (ret) { 942 cmd->engine_error = cmd_q->cmd_error; 943 goto e_ctx; 944 } 945 } 946 switch (aes->mode) { 947 case CCP_AES_MODE_CFB: /* CFB128 only */ 948 case CCP_AES_MODE_CTR: 949 op.u.aes.size = AES_BLOCK_SIZE * BITS_PER_BYTE - 1; 950 break; 951 default: 952 op.u.aes.size = 0; 953 } 954 955 /* Prepare the input and output data workareas. For in-place 956 * operations we need to set the dma direction to BIDIRECTIONAL 957 * and copy the src workarea to the dst workarea. 958 */ 959 if (sg_virt(aes->src) == sg_virt(aes->dst)) 960 in_place = true; 961 962 ret = ccp_init_data(&src, cmd_q, aes->src, aes->src_len, 963 AES_BLOCK_SIZE, 964 in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE); 965 if (ret) 966 goto e_ctx; 967 968 if (in_place) { 969 dst = src; 970 } else { 971 ret = ccp_init_data(&dst, cmd_q, aes->dst, aes->src_len, 972 AES_BLOCK_SIZE, DMA_FROM_DEVICE); 973 if (ret) 974 goto e_src; 975 } 976 977 /* Send data to the CCP AES engine */ 978 while (src.sg_wa.bytes_left) { 979 ccp_prepare_data(&src, &dst, &op, AES_BLOCK_SIZE, true); 980 if (!src.sg_wa.bytes_left) { 981 op.eom = 1; 982 983 /* Since we don't retrieve the AES context in ECB 984 * mode we have to wait for the operation to complete 985 * on the last piece of data 986 */ 987 if (aes->mode == CCP_AES_MODE_ECB) 988 op.soc = 1; 989 } 990 991 ret = cmd_q->ccp->vdata->perform->aes(&op); 992 if (ret) { 993 cmd->engine_error = cmd_q->cmd_error; 994 goto e_dst; 995 } 996 997 ccp_process_data(&src, &dst, &op); 998 } 999 1000 if (aes->mode != CCP_AES_MODE_ECB) { 1001 /* Retrieve the AES context - convert from LE to BE using 1002 * 32-byte (256-bit) byteswapping 1003 */ 1004 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, 1005 CCP_PASSTHRU_BYTESWAP_256BIT); 1006 if (ret) { 1007 cmd->engine_error = cmd_q->cmd_error; 1008 goto e_dst; 1009 } 1010 1011 /* ...but we only need AES_BLOCK_SIZE bytes */ 1012 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE; 1013 ccp_get_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len); 1014 } 1015 1016 e_dst: 1017 if (!in_place) 1018 ccp_free_data(&dst, cmd_q); 1019 1020 e_src: 1021 ccp_free_data(&src, cmd_q); 1022 1023 e_ctx: 1024 ccp_dm_free(&ctx); 1025 1026 e_key: 1027 ccp_dm_free(&key); 1028 1029 return ret; 1030 } 1031 1032 static int ccp_run_xts_aes_cmd(struct ccp_cmd_queue *cmd_q, 1033 struct ccp_cmd *cmd) 1034 { 1035 struct ccp_xts_aes_engine *xts = &cmd->u.xts; 1036 struct ccp_dm_workarea key, ctx; 1037 struct ccp_data src, dst; 1038 struct ccp_op op; 1039 unsigned int unit_size, dm_offset; 1040 bool in_place = false; 1041 int ret; 1042 1043 switch (xts->unit_size) { 1044 case CCP_XTS_AES_UNIT_SIZE_16: 1045 unit_size = 16; 1046 break; 1047 case CCP_XTS_AES_UNIT_SIZE_512: 1048 unit_size = 512; 1049 break; 1050 case CCP_XTS_AES_UNIT_SIZE_1024: 1051 unit_size = 1024; 1052 break; 1053 case CCP_XTS_AES_UNIT_SIZE_2048: 1054 unit_size = 2048; 1055 break; 1056 case CCP_XTS_AES_UNIT_SIZE_4096: 1057 unit_size = 4096; 1058 break; 1059 1060 default: 1061 return -EINVAL; 1062 } 1063 1064 if (xts->key_len != AES_KEYSIZE_128) 1065 return -EINVAL; 1066 1067 if (!xts->final && (xts->src_len & (AES_BLOCK_SIZE - 1))) 1068 return -EINVAL; 1069 1070 if (xts->iv_len != AES_BLOCK_SIZE) 1071 return -EINVAL; 1072 1073 if (!xts->key || !xts->iv || !xts->src || !xts->dst) 1074 return -EINVAL; 1075 1076 BUILD_BUG_ON(CCP_XTS_AES_KEY_SB_COUNT != 1); 1077 BUILD_BUG_ON(CCP_XTS_AES_CTX_SB_COUNT != 1); 1078 1079 ret = -EIO; 1080 memset(&op, 0, sizeof(op)); 1081 op.cmd_q = cmd_q; 1082 op.jobid = CCP_NEW_JOBID(cmd_q->ccp); 1083 op.sb_key = cmd_q->sb_key; 1084 op.sb_ctx = cmd_q->sb_ctx; 1085 op.init = 1; 1086 op.u.xts.action = xts->action; 1087 op.u.xts.unit_size = xts->unit_size; 1088 1089 /* All supported key sizes fit in a single (32-byte) SB entry 1090 * and must be in little endian format. Use the 256-bit byte 1091 * swap passthru option to convert from big endian to little 1092 * endian. 1093 */ 1094 ret = ccp_init_dm_workarea(&key, cmd_q, 1095 CCP_XTS_AES_KEY_SB_COUNT * CCP_SB_BYTES, 1096 DMA_TO_DEVICE); 1097 if (ret) 1098 return ret; 1099 1100 dm_offset = CCP_SB_BYTES - AES_KEYSIZE_128; 1101 ccp_set_dm_area(&key, dm_offset, xts->key, 0, xts->key_len); 1102 ccp_set_dm_area(&key, 0, xts->key, dm_offset, xts->key_len); 1103 ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key, 1104 CCP_PASSTHRU_BYTESWAP_256BIT); 1105 if (ret) { 1106 cmd->engine_error = cmd_q->cmd_error; 1107 goto e_key; 1108 } 1109 1110 /* The AES context fits in a single (32-byte) SB entry and 1111 * for XTS is already in little endian format so no byte swapping 1112 * is needed. 1113 */ 1114 ret = ccp_init_dm_workarea(&ctx, cmd_q, 1115 CCP_XTS_AES_CTX_SB_COUNT * CCP_SB_BYTES, 1116 DMA_BIDIRECTIONAL); 1117 if (ret) 1118 goto e_key; 1119 1120 ccp_set_dm_area(&ctx, 0, xts->iv, 0, xts->iv_len); 1121 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, 1122 CCP_PASSTHRU_BYTESWAP_NOOP); 1123 if (ret) { 1124 cmd->engine_error = cmd_q->cmd_error; 1125 goto e_ctx; 1126 } 1127 1128 /* Prepare the input and output data workareas. For in-place 1129 * operations we need to set the dma direction to BIDIRECTIONAL 1130 * and copy the src workarea to the dst workarea. 1131 */ 1132 if (sg_virt(xts->src) == sg_virt(xts->dst)) 1133 in_place = true; 1134 1135 ret = ccp_init_data(&src, cmd_q, xts->src, xts->src_len, 1136 unit_size, 1137 in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE); 1138 if (ret) 1139 goto e_ctx; 1140 1141 if (in_place) { 1142 dst = src; 1143 } else { 1144 ret = ccp_init_data(&dst, cmd_q, xts->dst, xts->src_len, 1145 unit_size, DMA_FROM_DEVICE); 1146 if (ret) 1147 goto e_src; 1148 } 1149 1150 /* Send data to the CCP AES engine */ 1151 while (src.sg_wa.bytes_left) { 1152 ccp_prepare_data(&src, &dst, &op, unit_size, true); 1153 if (!src.sg_wa.bytes_left) 1154 op.eom = 1; 1155 1156 ret = cmd_q->ccp->vdata->perform->xts_aes(&op); 1157 if (ret) { 1158 cmd->engine_error = cmd_q->cmd_error; 1159 goto e_dst; 1160 } 1161 1162 ccp_process_data(&src, &dst, &op); 1163 } 1164 1165 /* Retrieve the AES context - convert from LE to BE using 1166 * 32-byte (256-bit) byteswapping 1167 */ 1168 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, 1169 CCP_PASSTHRU_BYTESWAP_256BIT); 1170 if (ret) { 1171 cmd->engine_error = cmd_q->cmd_error; 1172 goto e_dst; 1173 } 1174 1175 /* ...but we only need AES_BLOCK_SIZE bytes */ 1176 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE; 1177 ccp_get_dm_area(&ctx, dm_offset, xts->iv, 0, xts->iv_len); 1178 1179 e_dst: 1180 if (!in_place) 1181 ccp_free_data(&dst, cmd_q); 1182 1183 e_src: 1184 ccp_free_data(&src, cmd_q); 1185 1186 e_ctx: 1187 ccp_dm_free(&ctx); 1188 1189 e_key: 1190 ccp_dm_free(&key); 1191 1192 return ret; 1193 } 1194 1195 static int ccp_run_des3_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) 1196 { 1197 struct ccp_des3_engine *des3 = &cmd->u.des3; 1198 1199 struct ccp_dm_workarea key, ctx; 1200 struct ccp_data src, dst; 1201 struct ccp_op op; 1202 unsigned int dm_offset; 1203 unsigned int len_singlekey; 1204 bool in_place = false; 1205 int ret; 1206 1207 /* Error checks */ 1208 if (!cmd_q->ccp->vdata->perform->des3) 1209 return -EINVAL; 1210 1211 if (des3->key_len != DES3_EDE_KEY_SIZE) 1212 return -EINVAL; 1213 1214 if (((des3->mode == CCP_DES3_MODE_ECB) || 1215 (des3->mode == CCP_DES3_MODE_CBC)) && 1216 (des3->src_len & (DES3_EDE_BLOCK_SIZE - 1))) 1217 return -EINVAL; 1218 1219 if (!des3->key || !des3->src || !des3->dst) 1220 return -EINVAL; 1221 1222 if (des3->mode != CCP_DES3_MODE_ECB) { 1223 if (des3->iv_len != DES3_EDE_BLOCK_SIZE) 1224 return -EINVAL; 1225 1226 if (!des3->iv) 1227 return -EINVAL; 1228 } 1229 1230 ret = -EIO; 1231 /* Zero out all the fields of the command desc */ 1232 memset(&op, 0, sizeof(op)); 1233 1234 /* Set up the Function field */ 1235 op.cmd_q = cmd_q; 1236 op.jobid = CCP_NEW_JOBID(cmd_q->ccp); 1237 op.sb_key = cmd_q->sb_key; 1238 1239 op.init = (des3->mode == CCP_DES3_MODE_ECB) ? 0 : 1; 1240 op.u.des3.type = des3->type; 1241 op.u.des3.mode = des3->mode; 1242 op.u.des3.action = des3->action; 1243 1244 /* 1245 * All supported key sizes fit in a single (32-byte) KSB entry and 1246 * (like AES) must be in little endian format. Use the 256-bit byte 1247 * swap passthru option to convert from big endian to little endian. 1248 */ 1249 ret = ccp_init_dm_workarea(&key, cmd_q, 1250 CCP_DES3_KEY_SB_COUNT * CCP_SB_BYTES, 1251 DMA_TO_DEVICE); 1252 if (ret) 1253 return ret; 1254 1255 /* 1256 * The contents of the key triplet are in the reverse order of what 1257 * is required by the engine. Copy the 3 pieces individually to put 1258 * them where they belong. 1259 */ 1260 dm_offset = CCP_SB_BYTES - des3->key_len; /* Basic offset */ 1261 1262 len_singlekey = des3->key_len / 3; 1263 ccp_set_dm_area(&key, dm_offset + 2 * len_singlekey, 1264 des3->key, 0, len_singlekey); 1265 ccp_set_dm_area(&key, dm_offset + len_singlekey, 1266 des3->key, len_singlekey, len_singlekey); 1267 ccp_set_dm_area(&key, dm_offset, 1268 des3->key, 2 * len_singlekey, len_singlekey); 1269 1270 /* Copy the key to the SB */ 1271 ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key, 1272 CCP_PASSTHRU_BYTESWAP_256BIT); 1273 if (ret) { 1274 cmd->engine_error = cmd_q->cmd_error; 1275 goto e_key; 1276 } 1277 1278 /* 1279 * The DES3 context fits in a single (32-byte) KSB entry and 1280 * must be in little endian format. Use the 256-bit byte swap 1281 * passthru option to convert from big endian to little endian. 1282 */ 1283 if (des3->mode != CCP_DES3_MODE_ECB) { 1284 u32 load_mode; 1285 1286 op.sb_ctx = cmd_q->sb_ctx; 1287 1288 ret = ccp_init_dm_workarea(&ctx, cmd_q, 1289 CCP_DES3_CTX_SB_COUNT * CCP_SB_BYTES, 1290 DMA_BIDIRECTIONAL); 1291 if (ret) 1292 goto e_key; 1293 1294 /* Load the context into the LSB */ 1295 dm_offset = CCP_SB_BYTES - des3->iv_len; 1296 ccp_set_dm_area(&ctx, dm_offset, des3->iv, 0, des3->iv_len); 1297 1298 if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0)) 1299 load_mode = CCP_PASSTHRU_BYTESWAP_NOOP; 1300 else 1301 load_mode = CCP_PASSTHRU_BYTESWAP_256BIT; 1302 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, 1303 load_mode); 1304 if (ret) { 1305 cmd->engine_error = cmd_q->cmd_error; 1306 goto e_ctx; 1307 } 1308 } 1309 1310 /* 1311 * Prepare the input and output data workareas. For in-place 1312 * operations we need to set the dma direction to BIDIRECTIONAL 1313 * and copy the src workarea to the dst workarea. 1314 */ 1315 if (sg_virt(des3->src) == sg_virt(des3->dst)) 1316 in_place = true; 1317 1318 ret = ccp_init_data(&src, cmd_q, des3->src, des3->src_len, 1319 DES3_EDE_BLOCK_SIZE, 1320 in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE); 1321 if (ret) 1322 goto e_ctx; 1323 1324 if (in_place) 1325 dst = src; 1326 else { 1327 ret = ccp_init_data(&dst, cmd_q, des3->dst, des3->src_len, 1328 DES3_EDE_BLOCK_SIZE, DMA_FROM_DEVICE); 1329 if (ret) 1330 goto e_src; 1331 } 1332 1333 /* Send data to the CCP DES3 engine */ 1334 while (src.sg_wa.bytes_left) { 1335 ccp_prepare_data(&src, &dst, &op, DES3_EDE_BLOCK_SIZE, true); 1336 if (!src.sg_wa.bytes_left) { 1337 op.eom = 1; 1338 1339 /* Since we don't retrieve the context in ECB mode 1340 * we have to wait for the operation to complete 1341 * on the last piece of data 1342 */ 1343 op.soc = 0; 1344 } 1345 1346 ret = cmd_q->ccp->vdata->perform->des3(&op); 1347 if (ret) { 1348 cmd->engine_error = cmd_q->cmd_error; 1349 goto e_dst; 1350 } 1351 1352 ccp_process_data(&src, &dst, &op); 1353 } 1354 1355 if (des3->mode != CCP_DES3_MODE_ECB) { 1356 /* Retrieve the context and make BE */ 1357 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, 1358 CCP_PASSTHRU_BYTESWAP_256BIT); 1359 if (ret) { 1360 cmd->engine_error = cmd_q->cmd_error; 1361 goto e_dst; 1362 } 1363 1364 /* ...but we only need the last DES3_EDE_BLOCK_SIZE bytes */ 1365 if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0)) 1366 dm_offset = CCP_SB_BYTES - des3->iv_len; 1367 else 1368 dm_offset = 0; 1369 ccp_get_dm_area(&ctx, dm_offset, des3->iv, 0, 1370 DES3_EDE_BLOCK_SIZE); 1371 } 1372 e_dst: 1373 if (!in_place) 1374 ccp_free_data(&dst, cmd_q); 1375 1376 e_src: 1377 ccp_free_data(&src, cmd_q); 1378 1379 e_ctx: 1380 if (des3->mode != CCP_DES3_MODE_ECB) 1381 ccp_dm_free(&ctx); 1382 1383 e_key: 1384 ccp_dm_free(&key); 1385 1386 return ret; 1387 } 1388 1389 static int ccp_run_sha_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) 1390 { 1391 struct ccp_sha_engine *sha = &cmd->u.sha; 1392 struct ccp_dm_workarea ctx; 1393 struct ccp_data src; 1394 struct ccp_op op; 1395 unsigned int ioffset, ooffset; 1396 unsigned int digest_size; 1397 int sb_count; 1398 const void *init; 1399 u64 block_size; 1400 int ctx_size; 1401 int ret; 1402 1403 switch (sha->type) { 1404 case CCP_SHA_TYPE_1: 1405 if (sha->ctx_len < SHA1_DIGEST_SIZE) 1406 return -EINVAL; 1407 block_size = SHA1_BLOCK_SIZE; 1408 break; 1409 case CCP_SHA_TYPE_224: 1410 if (sha->ctx_len < SHA224_DIGEST_SIZE) 1411 return -EINVAL; 1412 block_size = SHA224_BLOCK_SIZE; 1413 break; 1414 case CCP_SHA_TYPE_256: 1415 if (sha->ctx_len < SHA256_DIGEST_SIZE) 1416 return -EINVAL; 1417 block_size = SHA256_BLOCK_SIZE; 1418 break; 1419 case CCP_SHA_TYPE_384: 1420 if (cmd_q->ccp->vdata->version < CCP_VERSION(4, 0) 1421 || sha->ctx_len < SHA384_DIGEST_SIZE) 1422 return -EINVAL; 1423 block_size = SHA384_BLOCK_SIZE; 1424 break; 1425 case CCP_SHA_TYPE_512: 1426 if (cmd_q->ccp->vdata->version < CCP_VERSION(4, 0) 1427 || sha->ctx_len < SHA512_DIGEST_SIZE) 1428 return -EINVAL; 1429 block_size = SHA512_BLOCK_SIZE; 1430 break; 1431 default: 1432 return -EINVAL; 1433 } 1434 1435 if (!sha->ctx) 1436 return -EINVAL; 1437 1438 if (!sha->final && (sha->src_len & (block_size - 1))) 1439 return -EINVAL; 1440 1441 /* The version 3 device can't handle zero-length input */ 1442 if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0)) { 1443 1444 if (!sha->src_len) { 1445 unsigned int digest_len; 1446 const u8 *sha_zero; 1447 1448 /* Not final, just return */ 1449 if (!sha->final) 1450 return 0; 1451 1452 /* CCP can't do a zero length sha operation so the 1453 * caller must buffer the data. 1454 */ 1455 if (sha->msg_bits) 1456 return -EINVAL; 1457 1458 /* The CCP cannot perform zero-length sha operations 1459 * so the caller is required to buffer data for the 1460 * final operation. However, a sha operation for a 1461 * message with a total length of zero is valid so 1462 * known values are required to supply the result. 1463 */ 1464 switch (sha->type) { 1465 case CCP_SHA_TYPE_1: 1466 sha_zero = sha1_zero_message_hash; 1467 digest_len = SHA1_DIGEST_SIZE; 1468 break; 1469 case CCP_SHA_TYPE_224: 1470 sha_zero = sha224_zero_message_hash; 1471 digest_len = SHA224_DIGEST_SIZE; 1472 break; 1473 case CCP_SHA_TYPE_256: 1474 sha_zero = sha256_zero_message_hash; 1475 digest_len = SHA256_DIGEST_SIZE; 1476 break; 1477 default: 1478 return -EINVAL; 1479 } 1480 1481 scatterwalk_map_and_copy((void *)sha_zero, sha->ctx, 0, 1482 digest_len, 1); 1483 1484 return 0; 1485 } 1486 } 1487 1488 /* Set variables used throughout */ 1489 switch (sha->type) { 1490 case CCP_SHA_TYPE_1: 1491 digest_size = SHA1_DIGEST_SIZE; 1492 init = (void *) ccp_sha1_init; 1493 ctx_size = SHA1_DIGEST_SIZE; 1494 sb_count = 1; 1495 if (cmd_q->ccp->vdata->version != CCP_VERSION(3, 0)) 1496 ooffset = ioffset = CCP_SB_BYTES - SHA1_DIGEST_SIZE; 1497 else 1498 ooffset = ioffset = 0; 1499 break; 1500 case CCP_SHA_TYPE_224: 1501 digest_size = SHA224_DIGEST_SIZE; 1502 init = (void *) ccp_sha224_init; 1503 ctx_size = SHA256_DIGEST_SIZE; 1504 sb_count = 1; 1505 ioffset = 0; 1506 if (cmd_q->ccp->vdata->version != CCP_VERSION(3, 0)) 1507 ooffset = CCP_SB_BYTES - SHA224_DIGEST_SIZE; 1508 else 1509 ooffset = 0; 1510 break; 1511 case CCP_SHA_TYPE_256: 1512 digest_size = SHA256_DIGEST_SIZE; 1513 init = (void *) ccp_sha256_init; 1514 ctx_size = SHA256_DIGEST_SIZE; 1515 sb_count = 1; 1516 ooffset = ioffset = 0; 1517 break; 1518 case CCP_SHA_TYPE_384: 1519 digest_size = SHA384_DIGEST_SIZE; 1520 init = (void *) ccp_sha384_init; 1521 ctx_size = SHA512_DIGEST_SIZE; 1522 sb_count = 2; 1523 ioffset = 0; 1524 ooffset = 2 * CCP_SB_BYTES - SHA384_DIGEST_SIZE; 1525 break; 1526 case CCP_SHA_TYPE_512: 1527 digest_size = SHA512_DIGEST_SIZE; 1528 init = (void *) ccp_sha512_init; 1529 ctx_size = SHA512_DIGEST_SIZE; 1530 sb_count = 2; 1531 ooffset = ioffset = 0; 1532 break; 1533 default: 1534 ret = -EINVAL; 1535 goto e_data; 1536 } 1537 1538 /* For zero-length plaintext the src pointer is ignored; 1539 * otherwise both parts must be valid 1540 */ 1541 if (sha->src_len && !sha->src) 1542 return -EINVAL; 1543 1544 memset(&op, 0, sizeof(op)); 1545 op.cmd_q = cmd_q; 1546 op.jobid = CCP_NEW_JOBID(cmd_q->ccp); 1547 op.sb_ctx = cmd_q->sb_ctx; /* Pre-allocated */ 1548 op.u.sha.type = sha->type; 1549 op.u.sha.msg_bits = sha->msg_bits; 1550 1551 /* For SHA1/224/256 the context fits in a single (32-byte) SB entry; 1552 * SHA384/512 require 2 adjacent SB slots, with the right half in the 1553 * first slot, and the left half in the second. Each portion must then 1554 * be in little endian format: use the 256-bit byte swap option. 1555 */ 1556 ret = ccp_init_dm_workarea(&ctx, cmd_q, sb_count * CCP_SB_BYTES, 1557 DMA_BIDIRECTIONAL); 1558 if (ret) 1559 return ret; 1560 if (sha->first) { 1561 switch (sha->type) { 1562 case CCP_SHA_TYPE_1: 1563 case CCP_SHA_TYPE_224: 1564 case CCP_SHA_TYPE_256: 1565 memcpy(ctx.address + ioffset, init, ctx_size); 1566 break; 1567 case CCP_SHA_TYPE_384: 1568 case CCP_SHA_TYPE_512: 1569 memcpy(ctx.address + ctx_size / 2, init, 1570 ctx_size / 2); 1571 memcpy(ctx.address, init + ctx_size / 2, 1572 ctx_size / 2); 1573 break; 1574 default: 1575 ret = -EINVAL; 1576 goto e_ctx; 1577 } 1578 } else { 1579 /* Restore the context */ 1580 ccp_set_dm_area(&ctx, 0, sha->ctx, 0, 1581 sb_count * CCP_SB_BYTES); 1582 } 1583 1584 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, 1585 CCP_PASSTHRU_BYTESWAP_256BIT); 1586 if (ret) { 1587 cmd->engine_error = cmd_q->cmd_error; 1588 goto e_ctx; 1589 } 1590 1591 if (sha->src) { 1592 /* Send data to the CCP SHA engine; block_size is set above */ 1593 ret = ccp_init_data(&src, cmd_q, sha->src, sha->src_len, 1594 block_size, DMA_TO_DEVICE); 1595 if (ret) 1596 goto e_ctx; 1597 1598 while (src.sg_wa.bytes_left) { 1599 ccp_prepare_data(&src, NULL, &op, block_size, false); 1600 if (sha->final && !src.sg_wa.bytes_left) 1601 op.eom = 1; 1602 1603 ret = cmd_q->ccp->vdata->perform->sha(&op); 1604 if (ret) { 1605 cmd->engine_error = cmd_q->cmd_error; 1606 goto e_data; 1607 } 1608 1609 ccp_process_data(&src, NULL, &op); 1610 } 1611 } else { 1612 op.eom = 1; 1613 ret = cmd_q->ccp->vdata->perform->sha(&op); 1614 if (ret) { 1615 cmd->engine_error = cmd_q->cmd_error; 1616 goto e_data; 1617 } 1618 } 1619 1620 /* Retrieve the SHA context - convert from LE to BE using 1621 * 32-byte (256-bit) byteswapping to BE 1622 */ 1623 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, 1624 CCP_PASSTHRU_BYTESWAP_256BIT); 1625 if (ret) { 1626 cmd->engine_error = cmd_q->cmd_error; 1627 goto e_data; 1628 } 1629 1630 if (sha->final) { 1631 /* Finishing up, so get the digest */ 1632 switch (sha->type) { 1633 case CCP_SHA_TYPE_1: 1634 case CCP_SHA_TYPE_224: 1635 case CCP_SHA_TYPE_256: 1636 ccp_get_dm_area(&ctx, ooffset, 1637 sha->ctx, 0, 1638 digest_size); 1639 break; 1640 case CCP_SHA_TYPE_384: 1641 case CCP_SHA_TYPE_512: 1642 ccp_get_dm_area(&ctx, 0, 1643 sha->ctx, LSB_ITEM_SIZE - ooffset, 1644 LSB_ITEM_SIZE); 1645 ccp_get_dm_area(&ctx, LSB_ITEM_SIZE + ooffset, 1646 sha->ctx, 0, 1647 LSB_ITEM_SIZE - ooffset); 1648 break; 1649 default: 1650 ret = -EINVAL; 1651 goto e_ctx; 1652 } 1653 } else { 1654 /* Stash the context */ 1655 ccp_get_dm_area(&ctx, 0, sha->ctx, 0, 1656 sb_count * CCP_SB_BYTES); 1657 } 1658 1659 if (sha->final && sha->opad) { 1660 /* HMAC operation, recursively perform final SHA */ 1661 struct ccp_cmd hmac_cmd; 1662 struct scatterlist sg; 1663 u8 *hmac_buf; 1664 1665 if (sha->opad_len != block_size) { 1666 ret = -EINVAL; 1667 goto e_data; 1668 } 1669 1670 hmac_buf = kmalloc(block_size + digest_size, GFP_KERNEL); 1671 if (!hmac_buf) { 1672 ret = -ENOMEM; 1673 goto e_data; 1674 } 1675 sg_init_one(&sg, hmac_buf, block_size + digest_size); 1676 1677 scatterwalk_map_and_copy(hmac_buf, sha->opad, 0, block_size, 0); 1678 switch (sha->type) { 1679 case CCP_SHA_TYPE_1: 1680 case CCP_SHA_TYPE_224: 1681 case CCP_SHA_TYPE_256: 1682 memcpy(hmac_buf + block_size, 1683 ctx.address + ooffset, 1684 digest_size); 1685 break; 1686 case CCP_SHA_TYPE_384: 1687 case CCP_SHA_TYPE_512: 1688 memcpy(hmac_buf + block_size, 1689 ctx.address + LSB_ITEM_SIZE + ooffset, 1690 LSB_ITEM_SIZE); 1691 memcpy(hmac_buf + block_size + 1692 (LSB_ITEM_SIZE - ooffset), 1693 ctx.address, 1694 LSB_ITEM_SIZE); 1695 break; 1696 default: 1697 ret = -EINVAL; 1698 goto e_ctx; 1699 } 1700 1701 memset(&hmac_cmd, 0, sizeof(hmac_cmd)); 1702 hmac_cmd.engine = CCP_ENGINE_SHA; 1703 hmac_cmd.u.sha.type = sha->type; 1704 hmac_cmd.u.sha.ctx = sha->ctx; 1705 hmac_cmd.u.sha.ctx_len = sha->ctx_len; 1706 hmac_cmd.u.sha.src = &sg; 1707 hmac_cmd.u.sha.src_len = block_size + digest_size; 1708 hmac_cmd.u.sha.opad = NULL; 1709 hmac_cmd.u.sha.opad_len = 0; 1710 hmac_cmd.u.sha.first = 1; 1711 hmac_cmd.u.sha.final = 1; 1712 hmac_cmd.u.sha.msg_bits = (block_size + digest_size) << 3; 1713 1714 ret = ccp_run_sha_cmd(cmd_q, &hmac_cmd); 1715 if (ret) 1716 cmd->engine_error = hmac_cmd.engine_error; 1717 1718 kfree(hmac_buf); 1719 } 1720 1721 e_data: 1722 if (sha->src) 1723 ccp_free_data(&src, cmd_q); 1724 1725 e_ctx: 1726 ccp_dm_free(&ctx); 1727 1728 return ret; 1729 } 1730 1731 static int ccp_run_rsa_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) 1732 { 1733 struct ccp_rsa_engine *rsa = &cmd->u.rsa; 1734 struct ccp_dm_workarea exp, src; 1735 struct ccp_data dst; 1736 struct ccp_op op; 1737 unsigned int sb_count, i_len, o_len; 1738 int ret; 1739 1740 if (rsa->key_size > CCP_RSA_MAX_WIDTH) 1741 return -EINVAL; 1742 1743 if (!rsa->exp || !rsa->mod || !rsa->src || !rsa->dst) 1744 return -EINVAL; 1745 1746 /* The RSA modulus must precede the message being acted upon, so 1747 * it must be copied to a DMA area where the message and the 1748 * modulus can be concatenated. Therefore the input buffer 1749 * length required is twice the output buffer length (which 1750 * must be a multiple of 256-bits). 1751 */ 1752 o_len = ((rsa->key_size + 255) / 256) * 32; 1753 i_len = o_len * 2; 1754 1755 sb_count = o_len / CCP_SB_BYTES; 1756 1757 memset(&op, 0, sizeof(op)); 1758 op.cmd_q = cmd_q; 1759 op.jobid = ccp_gen_jobid(cmd_q->ccp); 1760 op.sb_key = cmd_q->ccp->vdata->perform->sballoc(cmd_q, sb_count); 1761 1762 if (!op.sb_key) 1763 return -EIO; 1764 1765 /* The RSA exponent may span multiple (32-byte) SB entries and must 1766 * be in little endian format. Reverse copy each 32-byte chunk 1767 * of the exponent (En chunk to E0 chunk, E(n-1) chunk to E1 chunk) 1768 * and each byte within that chunk and do not perform any byte swap 1769 * operations on the passthru operation. 1770 */ 1771 ret = ccp_init_dm_workarea(&exp, cmd_q, o_len, DMA_TO_DEVICE); 1772 if (ret) 1773 goto e_sb; 1774 1775 ret = ccp_reverse_set_dm_area(&exp, 0, rsa->exp, 0, rsa->exp_len); 1776 if (ret) 1777 goto e_exp; 1778 ret = ccp_copy_to_sb(cmd_q, &exp, op.jobid, op.sb_key, 1779 CCP_PASSTHRU_BYTESWAP_NOOP); 1780 if (ret) { 1781 cmd->engine_error = cmd_q->cmd_error; 1782 goto e_exp; 1783 } 1784 1785 /* Concatenate the modulus and the message. Both the modulus and 1786 * the operands must be in little endian format. Since the input 1787 * is in big endian format it must be converted. 1788 */ 1789 ret = ccp_init_dm_workarea(&src, cmd_q, i_len, DMA_TO_DEVICE); 1790 if (ret) 1791 goto e_exp; 1792 1793 ret = ccp_reverse_set_dm_area(&src, 0, rsa->mod, 0, rsa->mod_len); 1794 if (ret) 1795 goto e_src; 1796 ret = ccp_reverse_set_dm_area(&src, o_len, rsa->src, 0, rsa->src_len); 1797 if (ret) 1798 goto e_src; 1799 1800 /* Prepare the output area for the operation */ 1801 ret = ccp_init_data(&dst, cmd_q, rsa->dst, rsa->mod_len, 1802 o_len, DMA_FROM_DEVICE); 1803 if (ret) 1804 goto e_src; 1805 1806 op.soc = 1; 1807 op.src.u.dma.address = src.dma.address; 1808 op.src.u.dma.offset = 0; 1809 op.src.u.dma.length = i_len; 1810 op.dst.u.dma.address = dst.dm_wa.dma.address; 1811 op.dst.u.dma.offset = 0; 1812 op.dst.u.dma.length = o_len; 1813 1814 op.u.rsa.mod_size = rsa->key_size; 1815 op.u.rsa.input_len = i_len; 1816 1817 ret = cmd_q->ccp->vdata->perform->rsa(&op); 1818 if (ret) { 1819 cmd->engine_error = cmd_q->cmd_error; 1820 goto e_dst; 1821 } 1822 1823 ccp_reverse_get_dm_area(&dst.dm_wa, 0, rsa->dst, 0, rsa->mod_len); 1824 1825 e_dst: 1826 ccp_free_data(&dst, cmd_q); 1827 1828 e_src: 1829 ccp_dm_free(&src); 1830 1831 e_exp: 1832 ccp_dm_free(&exp); 1833 1834 e_sb: 1835 cmd_q->ccp->vdata->perform->sbfree(cmd_q, op.sb_key, sb_count); 1836 1837 return ret; 1838 } 1839 1840 static int ccp_run_passthru_cmd(struct ccp_cmd_queue *cmd_q, 1841 struct ccp_cmd *cmd) 1842 { 1843 struct ccp_passthru_engine *pt = &cmd->u.passthru; 1844 struct ccp_dm_workarea mask; 1845 struct ccp_data src, dst; 1846 struct ccp_op op; 1847 bool in_place = false; 1848 unsigned int i; 1849 int ret = 0; 1850 1851 if (!pt->final && (pt->src_len & (CCP_PASSTHRU_BLOCKSIZE - 1))) 1852 return -EINVAL; 1853 1854 if (!pt->src || !pt->dst) 1855 return -EINVAL; 1856 1857 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) { 1858 if (pt->mask_len != CCP_PASSTHRU_MASKSIZE) 1859 return -EINVAL; 1860 if (!pt->mask) 1861 return -EINVAL; 1862 } 1863 1864 BUILD_BUG_ON(CCP_PASSTHRU_SB_COUNT != 1); 1865 1866 memset(&op, 0, sizeof(op)); 1867 op.cmd_q = cmd_q; 1868 op.jobid = CCP_NEW_JOBID(cmd_q->ccp); 1869 1870 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) { 1871 /* Load the mask */ 1872 op.sb_key = cmd_q->sb_key; 1873 1874 ret = ccp_init_dm_workarea(&mask, cmd_q, 1875 CCP_PASSTHRU_SB_COUNT * 1876 CCP_SB_BYTES, 1877 DMA_TO_DEVICE); 1878 if (ret) 1879 return ret; 1880 1881 ccp_set_dm_area(&mask, 0, pt->mask, 0, pt->mask_len); 1882 ret = ccp_copy_to_sb(cmd_q, &mask, op.jobid, op.sb_key, 1883 CCP_PASSTHRU_BYTESWAP_NOOP); 1884 if (ret) { 1885 cmd->engine_error = cmd_q->cmd_error; 1886 goto e_mask; 1887 } 1888 } 1889 1890 /* Prepare the input and output data workareas. For in-place 1891 * operations we need to set the dma direction to BIDIRECTIONAL 1892 * and copy the src workarea to the dst workarea. 1893 */ 1894 if (sg_virt(pt->src) == sg_virt(pt->dst)) 1895 in_place = true; 1896 1897 ret = ccp_init_data(&src, cmd_q, pt->src, pt->src_len, 1898 CCP_PASSTHRU_MASKSIZE, 1899 in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE); 1900 if (ret) 1901 goto e_mask; 1902 1903 if (in_place) { 1904 dst = src; 1905 } else { 1906 ret = ccp_init_data(&dst, cmd_q, pt->dst, pt->src_len, 1907 CCP_PASSTHRU_MASKSIZE, DMA_FROM_DEVICE); 1908 if (ret) 1909 goto e_src; 1910 } 1911 1912 /* Send data to the CCP Passthru engine 1913 * Because the CCP engine works on a single source and destination 1914 * dma address at a time, each entry in the source scatterlist 1915 * (after the dma_map_sg call) must be less than or equal to the 1916 * (remaining) length in the destination scatterlist entry and the 1917 * length must be a multiple of CCP_PASSTHRU_BLOCKSIZE 1918 */ 1919 dst.sg_wa.sg_used = 0; 1920 for (i = 1; i <= src.sg_wa.dma_count; i++) { 1921 if (!dst.sg_wa.sg || 1922 (dst.sg_wa.sg->length < src.sg_wa.sg->length)) { 1923 ret = -EINVAL; 1924 goto e_dst; 1925 } 1926 1927 if (i == src.sg_wa.dma_count) { 1928 op.eom = 1; 1929 op.soc = 1; 1930 } 1931 1932 op.src.type = CCP_MEMTYPE_SYSTEM; 1933 op.src.u.dma.address = sg_dma_address(src.sg_wa.sg); 1934 op.src.u.dma.offset = 0; 1935 op.src.u.dma.length = sg_dma_len(src.sg_wa.sg); 1936 1937 op.dst.type = CCP_MEMTYPE_SYSTEM; 1938 op.dst.u.dma.address = sg_dma_address(dst.sg_wa.sg); 1939 op.dst.u.dma.offset = dst.sg_wa.sg_used; 1940 op.dst.u.dma.length = op.src.u.dma.length; 1941 1942 ret = cmd_q->ccp->vdata->perform->passthru(&op); 1943 if (ret) { 1944 cmd->engine_error = cmd_q->cmd_error; 1945 goto e_dst; 1946 } 1947 1948 dst.sg_wa.sg_used += src.sg_wa.sg->length; 1949 if (dst.sg_wa.sg_used == dst.sg_wa.sg->length) { 1950 dst.sg_wa.sg = sg_next(dst.sg_wa.sg); 1951 dst.sg_wa.sg_used = 0; 1952 } 1953 src.sg_wa.sg = sg_next(src.sg_wa.sg); 1954 } 1955 1956 e_dst: 1957 if (!in_place) 1958 ccp_free_data(&dst, cmd_q); 1959 1960 e_src: 1961 ccp_free_data(&src, cmd_q); 1962 1963 e_mask: 1964 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) 1965 ccp_dm_free(&mask); 1966 1967 return ret; 1968 } 1969 1970 static int ccp_run_passthru_nomap_cmd(struct ccp_cmd_queue *cmd_q, 1971 struct ccp_cmd *cmd) 1972 { 1973 struct ccp_passthru_nomap_engine *pt = &cmd->u.passthru_nomap; 1974 struct ccp_dm_workarea mask; 1975 struct ccp_op op; 1976 int ret; 1977 1978 if (!pt->final && (pt->src_len & (CCP_PASSTHRU_BLOCKSIZE - 1))) 1979 return -EINVAL; 1980 1981 if (!pt->src_dma || !pt->dst_dma) 1982 return -EINVAL; 1983 1984 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) { 1985 if (pt->mask_len != CCP_PASSTHRU_MASKSIZE) 1986 return -EINVAL; 1987 if (!pt->mask) 1988 return -EINVAL; 1989 } 1990 1991 BUILD_BUG_ON(CCP_PASSTHRU_SB_COUNT != 1); 1992 1993 memset(&op, 0, sizeof(op)); 1994 op.cmd_q = cmd_q; 1995 op.jobid = ccp_gen_jobid(cmd_q->ccp); 1996 1997 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) { 1998 /* Load the mask */ 1999 op.sb_key = cmd_q->sb_key; 2000 2001 mask.length = pt->mask_len; 2002 mask.dma.address = pt->mask; 2003 mask.dma.length = pt->mask_len; 2004 2005 ret = ccp_copy_to_sb(cmd_q, &mask, op.jobid, op.sb_key, 2006 CCP_PASSTHRU_BYTESWAP_NOOP); 2007 if (ret) { 2008 cmd->engine_error = cmd_q->cmd_error; 2009 return ret; 2010 } 2011 } 2012 2013 /* Send data to the CCP Passthru engine */ 2014 op.eom = 1; 2015 op.soc = 1; 2016 2017 op.src.type = CCP_MEMTYPE_SYSTEM; 2018 op.src.u.dma.address = pt->src_dma; 2019 op.src.u.dma.offset = 0; 2020 op.src.u.dma.length = pt->src_len; 2021 2022 op.dst.type = CCP_MEMTYPE_SYSTEM; 2023 op.dst.u.dma.address = pt->dst_dma; 2024 op.dst.u.dma.offset = 0; 2025 op.dst.u.dma.length = pt->src_len; 2026 2027 ret = cmd_q->ccp->vdata->perform->passthru(&op); 2028 if (ret) 2029 cmd->engine_error = cmd_q->cmd_error; 2030 2031 return ret; 2032 } 2033 2034 static int ccp_run_ecc_mm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) 2035 { 2036 struct ccp_ecc_engine *ecc = &cmd->u.ecc; 2037 struct ccp_dm_workarea src, dst; 2038 struct ccp_op op; 2039 int ret; 2040 u8 *save; 2041 2042 if (!ecc->u.mm.operand_1 || 2043 (ecc->u.mm.operand_1_len > CCP_ECC_MODULUS_BYTES)) 2044 return -EINVAL; 2045 2046 if (ecc->function != CCP_ECC_FUNCTION_MINV_384BIT) 2047 if (!ecc->u.mm.operand_2 || 2048 (ecc->u.mm.operand_2_len > CCP_ECC_MODULUS_BYTES)) 2049 return -EINVAL; 2050 2051 if (!ecc->u.mm.result || 2052 (ecc->u.mm.result_len < CCP_ECC_MODULUS_BYTES)) 2053 return -EINVAL; 2054 2055 memset(&op, 0, sizeof(op)); 2056 op.cmd_q = cmd_q; 2057 op.jobid = CCP_NEW_JOBID(cmd_q->ccp); 2058 2059 /* Concatenate the modulus and the operands. Both the modulus and 2060 * the operands must be in little endian format. Since the input 2061 * is in big endian format it must be converted and placed in a 2062 * fixed length buffer. 2063 */ 2064 ret = ccp_init_dm_workarea(&src, cmd_q, CCP_ECC_SRC_BUF_SIZE, 2065 DMA_TO_DEVICE); 2066 if (ret) 2067 return ret; 2068 2069 /* Save the workarea address since it is updated in order to perform 2070 * the concatenation 2071 */ 2072 save = src.address; 2073 2074 /* Copy the ECC modulus */ 2075 ret = ccp_reverse_set_dm_area(&src, 0, ecc->mod, 0, ecc->mod_len); 2076 if (ret) 2077 goto e_src; 2078 src.address += CCP_ECC_OPERAND_SIZE; 2079 2080 /* Copy the first operand */ 2081 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.mm.operand_1, 0, 2082 ecc->u.mm.operand_1_len); 2083 if (ret) 2084 goto e_src; 2085 src.address += CCP_ECC_OPERAND_SIZE; 2086 2087 if (ecc->function != CCP_ECC_FUNCTION_MINV_384BIT) { 2088 /* Copy the second operand */ 2089 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.mm.operand_2, 0, 2090 ecc->u.mm.operand_2_len); 2091 if (ret) 2092 goto e_src; 2093 src.address += CCP_ECC_OPERAND_SIZE; 2094 } 2095 2096 /* Restore the workarea address */ 2097 src.address = save; 2098 2099 /* Prepare the output area for the operation */ 2100 ret = ccp_init_dm_workarea(&dst, cmd_q, CCP_ECC_DST_BUF_SIZE, 2101 DMA_FROM_DEVICE); 2102 if (ret) 2103 goto e_src; 2104 2105 op.soc = 1; 2106 op.src.u.dma.address = src.dma.address; 2107 op.src.u.dma.offset = 0; 2108 op.src.u.dma.length = src.length; 2109 op.dst.u.dma.address = dst.dma.address; 2110 op.dst.u.dma.offset = 0; 2111 op.dst.u.dma.length = dst.length; 2112 2113 op.u.ecc.function = cmd->u.ecc.function; 2114 2115 ret = cmd_q->ccp->vdata->perform->ecc(&op); 2116 if (ret) { 2117 cmd->engine_error = cmd_q->cmd_error; 2118 goto e_dst; 2119 } 2120 2121 ecc->ecc_result = le16_to_cpup( 2122 (const __le16 *)(dst.address + CCP_ECC_RESULT_OFFSET)); 2123 if (!(ecc->ecc_result & CCP_ECC_RESULT_SUCCESS)) { 2124 ret = -EIO; 2125 goto e_dst; 2126 } 2127 2128 /* Save the ECC result */ 2129 ccp_reverse_get_dm_area(&dst, 0, ecc->u.mm.result, 0, 2130 CCP_ECC_MODULUS_BYTES); 2131 2132 e_dst: 2133 ccp_dm_free(&dst); 2134 2135 e_src: 2136 ccp_dm_free(&src); 2137 2138 return ret; 2139 } 2140 2141 static int ccp_run_ecc_pm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) 2142 { 2143 struct ccp_ecc_engine *ecc = &cmd->u.ecc; 2144 struct ccp_dm_workarea src, dst; 2145 struct ccp_op op; 2146 int ret; 2147 u8 *save; 2148 2149 if (!ecc->u.pm.point_1.x || 2150 (ecc->u.pm.point_1.x_len > CCP_ECC_MODULUS_BYTES) || 2151 !ecc->u.pm.point_1.y || 2152 (ecc->u.pm.point_1.y_len > CCP_ECC_MODULUS_BYTES)) 2153 return -EINVAL; 2154 2155 if (ecc->function == CCP_ECC_FUNCTION_PADD_384BIT) { 2156 if (!ecc->u.pm.point_2.x || 2157 (ecc->u.pm.point_2.x_len > CCP_ECC_MODULUS_BYTES) || 2158 !ecc->u.pm.point_2.y || 2159 (ecc->u.pm.point_2.y_len > CCP_ECC_MODULUS_BYTES)) 2160 return -EINVAL; 2161 } else { 2162 if (!ecc->u.pm.domain_a || 2163 (ecc->u.pm.domain_a_len > CCP_ECC_MODULUS_BYTES)) 2164 return -EINVAL; 2165 2166 if (ecc->function == CCP_ECC_FUNCTION_PMUL_384BIT) 2167 if (!ecc->u.pm.scalar || 2168 (ecc->u.pm.scalar_len > CCP_ECC_MODULUS_BYTES)) 2169 return -EINVAL; 2170 } 2171 2172 if (!ecc->u.pm.result.x || 2173 (ecc->u.pm.result.x_len < CCP_ECC_MODULUS_BYTES) || 2174 !ecc->u.pm.result.y || 2175 (ecc->u.pm.result.y_len < CCP_ECC_MODULUS_BYTES)) 2176 return -EINVAL; 2177 2178 memset(&op, 0, sizeof(op)); 2179 op.cmd_q = cmd_q; 2180 op.jobid = CCP_NEW_JOBID(cmd_q->ccp); 2181 2182 /* Concatenate the modulus and the operands. Both the modulus and 2183 * the operands must be in little endian format. Since the input 2184 * is in big endian format it must be converted and placed in a 2185 * fixed length buffer. 2186 */ 2187 ret = ccp_init_dm_workarea(&src, cmd_q, CCP_ECC_SRC_BUF_SIZE, 2188 DMA_TO_DEVICE); 2189 if (ret) 2190 return ret; 2191 2192 /* Save the workarea address since it is updated in order to perform 2193 * the concatenation 2194 */ 2195 save = src.address; 2196 2197 /* Copy the ECC modulus */ 2198 ret = ccp_reverse_set_dm_area(&src, 0, ecc->mod, 0, ecc->mod_len); 2199 if (ret) 2200 goto e_src; 2201 src.address += CCP_ECC_OPERAND_SIZE; 2202 2203 /* Copy the first point X and Y coordinate */ 2204 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_1.x, 0, 2205 ecc->u.pm.point_1.x_len); 2206 if (ret) 2207 goto e_src; 2208 src.address += CCP_ECC_OPERAND_SIZE; 2209 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_1.y, 0, 2210 ecc->u.pm.point_1.y_len); 2211 if (ret) 2212 goto e_src; 2213 src.address += CCP_ECC_OPERAND_SIZE; 2214 2215 /* Set the first point Z coordinate to 1 */ 2216 *src.address = 0x01; 2217 src.address += CCP_ECC_OPERAND_SIZE; 2218 2219 if (ecc->function == CCP_ECC_FUNCTION_PADD_384BIT) { 2220 /* Copy the second point X and Y coordinate */ 2221 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_2.x, 0, 2222 ecc->u.pm.point_2.x_len); 2223 if (ret) 2224 goto e_src; 2225 src.address += CCP_ECC_OPERAND_SIZE; 2226 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_2.y, 0, 2227 ecc->u.pm.point_2.y_len); 2228 if (ret) 2229 goto e_src; 2230 src.address += CCP_ECC_OPERAND_SIZE; 2231 2232 /* Set the second point Z coordinate to 1 */ 2233 *src.address = 0x01; 2234 src.address += CCP_ECC_OPERAND_SIZE; 2235 } else { 2236 /* Copy the Domain "a" parameter */ 2237 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.domain_a, 0, 2238 ecc->u.pm.domain_a_len); 2239 if (ret) 2240 goto e_src; 2241 src.address += CCP_ECC_OPERAND_SIZE; 2242 2243 if (ecc->function == CCP_ECC_FUNCTION_PMUL_384BIT) { 2244 /* Copy the scalar value */ 2245 ret = ccp_reverse_set_dm_area(&src, 0, 2246 ecc->u.pm.scalar, 0, 2247 ecc->u.pm.scalar_len); 2248 if (ret) 2249 goto e_src; 2250 src.address += CCP_ECC_OPERAND_SIZE; 2251 } 2252 } 2253 2254 /* Restore the workarea address */ 2255 src.address = save; 2256 2257 /* Prepare the output area for the operation */ 2258 ret = ccp_init_dm_workarea(&dst, cmd_q, CCP_ECC_DST_BUF_SIZE, 2259 DMA_FROM_DEVICE); 2260 if (ret) 2261 goto e_src; 2262 2263 op.soc = 1; 2264 op.src.u.dma.address = src.dma.address; 2265 op.src.u.dma.offset = 0; 2266 op.src.u.dma.length = src.length; 2267 op.dst.u.dma.address = dst.dma.address; 2268 op.dst.u.dma.offset = 0; 2269 op.dst.u.dma.length = dst.length; 2270 2271 op.u.ecc.function = cmd->u.ecc.function; 2272 2273 ret = cmd_q->ccp->vdata->perform->ecc(&op); 2274 if (ret) { 2275 cmd->engine_error = cmd_q->cmd_error; 2276 goto e_dst; 2277 } 2278 2279 ecc->ecc_result = le16_to_cpup( 2280 (const __le16 *)(dst.address + CCP_ECC_RESULT_OFFSET)); 2281 if (!(ecc->ecc_result & CCP_ECC_RESULT_SUCCESS)) { 2282 ret = -EIO; 2283 goto e_dst; 2284 } 2285 2286 /* Save the workarea address since it is updated as we walk through 2287 * to copy the point math result 2288 */ 2289 save = dst.address; 2290 2291 /* Save the ECC result X and Y coordinates */ 2292 ccp_reverse_get_dm_area(&dst, 0, ecc->u.pm.result.x, 0, 2293 CCP_ECC_MODULUS_BYTES); 2294 dst.address += CCP_ECC_OUTPUT_SIZE; 2295 ccp_reverse_get_dm_area(&dst, 0, ecc->u.pm.result.y, 0, 2296 CCP_ECC_MODULUS_BYTES); 2297 dst.address += CCP_ECC_OUTPUT_SIZE; 2298 2299 /* Restore the workarea address */ 2300 dst.address = save; 2301 2302 e_dst: 2303 ccp_dm_free(&dst); 2304 2305 e_src: 2306 ccp_dm_free(&src); 2307 2308 return ret; 2309 } 2310 2311 static int ccp_run_ecc_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) 2312 { 2313 struct ccp_ecc_engine *ecc = &cmd->u.ecc; 2314 2315 ecc->ecc_result = 0; 2316 2317 if (!ecc->mod || 2318 (ecc->mod_len > CCP_ECC_MODULUS_BYTES)) 2319 return -EINVAL; 2320 2321 switch (ecc->function) { 2322 case CCP_ECC_FUNCTION_MMUL_384BIT: 2323 case CCP_ECC_FUNCTION_MADD_384BIT: 2324 case CCP_ECC_FUNCTION_MINV_384BIT: 2325 return ccp_run_ecc_mm_cmd(cmd_q, cmd); 2326 2327 case CCP_ECC_FUNCTION_PADD_384BIT: 2328 case CCP_ECC_FUNCTION_PMUL_384BIT: 2329 case CCP_ECC_FUNCTION_PDBL_384BIT: 2330 return ccp_run_ecc_pm_cmd(cmd_q, cmd); 2331 2332 default: 2333 return -EINVAL; 2334 } 2335 } 2336 2337 int ccp_run_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) 2338 { 2339 int ret; 2340 2341 cmd->engine_error = 0; 2342 cmd_q->cmd_error = 0; 2343 cmd_q->int_rcvd = 0; 2344 cmd_q->free_slots = cmd_q->ccp->vdata->perform->get_free_slots(cmd_q); 2345 2346 switch (cmd->engine) { 2347 case CCP_ENGINE_AES: 2348 ret = ccp_run_aes_cmd(cmd_q, cmd); 2349 break; 2350 case CCP_ENGINE_XTS_AES_128: 2351 ret = ccp_run_xts_aes_cmd(cmd_q, cmd); 2352 break; 2353 case CCP_ENGINE_DES3: 2354 ret = ccp_run_des3_cmd(cmd_q, cmd); 2355 break; 2356 case CCP_ENGINE_SHA: 2357 ret = ccp_run_sha_cmd(cmd_q, cmd); 2358 break; 2359 case CCP_ENGINE_RSA: 2360 ret = ccp_run_rsa_cmd(cmd_q, cmd); 2361 break; 2362 case CCP_ENGINE_PASSTHRU: 2363 if (cmd->flags & CCP_CMD_PASSTHRU_NO_DMA_MAP) 2364 ret = ccp_run_passthru_nomap_cmd(cmd_q, cmd); 2365 else 2366 ret = ccp_run_passthru_cmd(cmd_q, cmd); 2367 break; 2368 case CCP_ENGINE_ECC: 2369 ret = ccp_run_ecc_cmd(cmd_q, cmd); 2370 break; 2371 default: 2372 ret = -EINVAL; 2373 } 2374 2375 return ret; 2376 } 2377