1 /* 2 * AMD Cryptographic Coprocessor (CCP) driver 3 * 4 * Copyright (C) 2013,2016 Advanced Micro Devices, Inc. 5 * 6 * Author: Tom Lendacky <thomas.lendacky@amd.com> 7 * Author: Gary R Hook <gary.hook@amd.com> 8 * 9 * This program is free software; you can redistribute it and/or modify 10 * it under the terms of the GNU General Public License version 2 as 11 * published by the Free Software Foundation. 12 */ 13 14 #include <linux/module.h> 15 #include <linux/kernel.h> 16 #include <linux/pci.h> 17 #include <linux/interrupt.h> 18 #include <crypto/scatterwalk.h> 19 #include <crypto/des.h> 20 #include <linux/ccp.h> 21 22 #include "ccp-dev.h" 23 24 /* SHA initial context values */ 25 static const __be32 ccp_sha1_init[SHA1_DIGEST_SIZE / sizeof(__be32)] = { 26 cpu_to_be32(SHA1_H0), cpu_to_be32(SHA1_H1), 27 cpu_to_be32(SHA1_H2), cpu_to_be32(SHA1_H3), 28 cpu_to_be32(SHA1_H4), 29 }; 30 31 static const __be32 ccp_sha224_init[SHA256_DIGEST_SIZE / sizeof(__be32)] = { 32 cpu_to_be32(SHA224_H0), cpu_to_be32(SHA224_H1), 33 cpu_to_be32(SHA224_H2), cpu_to_be32(SHA224_H3), 34 cpu_to_be32(SHA224_H4), cpu_to_be32(SHA224_H5), 35 cpu_to_be32(SHA224_H6), cpu_to_be32(SHA224_H7), 36 }; 37 38 static const __be32 ccp_sha256_init[SHA256_DIGEST_SIZE / sizeof(__be32)] = { 39 cpu_to_be32(SHA256_H0), cpu_to_be32(SHA256_H1), 40 cpu_to_be32(SHA256_H2), cpu_to_be32(SHA256_H3), 41 cpu_to_be32(SHA256_H4), cpu_to_be32(SHA256_H5), 42 cpu_to_be32(SHA256_H6), cpu_to_be32(SHA256_H7), 43 }; 44 45 static const __be64 ccp_sha384_init[SHA512_DIGEST_SIZE / sizeof(__be64)] = { 46 cpu_to_be64(SHA384_H0), cpu_to_be64(SHA384_H1), 47 cpu_to_be64(SHA384_H2), cpu_to_be64(SHA384_H3), 48 cpu_to_be64(SHA384_H4), cpu_to_be64(SHA384_H5), 49 cpu_to_be64(SHA384_H6), cpu_to_be64(SHA384_H7), 50 }; 51 52 static const __be64 ccp_sha512_init[SHA512_DIGEST_SIZE / sizeof(__be64)] = { 53 cpu_to_be64(SHA512_H0), cpu_to_be64(SHA512_H1), 54 cpu_to_be64(SHA512_H2), cpu_to_be64(SHA512_H3), 55 cpu_to_be64(SHA512_H4), cpu_to_be64(SHA512_H5), 56 cpu_to_be64(SHA512_H6), cpu_to_be64(SHA512_H7), 57 }; 58 59 #define CCP_NEW_JOBID(ccp) ((ccp->vdata->version == CCP_VERSION(3, 0)) ? \ 60 ccp_gen_jobid(ccp) : 0) 61 62 static u32 ccp_gen_jobid(struct ccp_device *ccp) 63 { 64 return atomic_inc_return(&ccp->current_id) & CCP_JOBID_MASK; 65 } 66 67 static void ccp_sg_free(struct ccp_sg_workarea *wa) 68 { 69 if (wa->dma_count) 70 dma_unmap_sg(wa->dma_dev, wa->dma_sg, wa->nents, wa->dma_dir); 71 72 wa->dma_count = 0; 73 } 74 75 static int ccp_init_sg_workarea(struct ccp_sg_workarea *wa, struct device *dev, 76 struct scatterlist *sg, u64 len, 77 enum dma_data_direction dma_dir) 78 { 79 memset(wa, 0, sizeof(*wa)); 80 81 wa->sg = sg; 82 if (!sg) 83 return 0; 84 85 wa->nents = sg_nents_for_len(sg, len); 86 if (wa->nents < 0) 87 return wa->nents; 88 89 wa->bytes_left = len; 90 wa->sg_used = 0; 91 92 if (len == 0) 93 return 0; 94 95 if (dma_dir == DMA_NONE) 96 return 0; 97 98 wa->dma_sg = sg; 99 wa->dma_dev = dev; 100 wa->dma_dir = dma_dir; 101 wa->dma_count = dma_map_sg(dev, sg, wa->nents, dma_dir); 102 if (!wa->dma_count) 103 return -ENOMEM; 104 105 return 0; 106 } 107 108 static void ccp_update_sg_workarea(struct ccp_sg_workarea *wa, unsigned int len) 109 { 110 unsigned int nbytes = min_t(u64, len, wa->bytes_left); 111 112 if (!wa->sg) 113 return; 114 115 wa->sg_used += nbytes; 116 wa->bytes_left -= nbytes; 117 if (wa->sg_used == wa->sg->length) { 118 wa->sg = sg_next(wa->sg); 119 wa->sg_used = 0; 120 } 121 } 122 123 static void ccp_dm_free(struct ccp_dm_workarea *wa) 124 { 125 if (wa->length <= CCP_DMAPOOL_MAX_SIZE) { 126 if (wa->address) 127 dma_pool_free(wa->dma_pool, wa->address, 128 wa->dma.address); 129 } else { 130 if (wa->dma.address) 131 dma_unmap_single(wa->dev, wa->dma.address, wa->length, 132 wa->dma.dir); 133 kfree(wa->address); 134 } 135 136 wa->address = NULL; 137 wa->dma.address = 0; 138 } 139 140 static int ccp_init_dm_workarea(struct ccp_dm_workarea *wa, 141 struct ccp_cmd_queue *cmd_q, 142 unsigned int len, 143 enum dma_data_direction dir) 144 { 145 memset(wa, 0, sizeof(*wa)); 146 147 if (!len) 148 return 0; 149 150 wa->dev = cmd_q->ccp->dev; 151 wa->length = len; 152 153 if (len <= CCP_DMAPOOL_MAX_SIZE) { 154 wa->dma_pool = cmd_q->dma_pool; 155 156 wa->address = dma_pool_alloc(wa->dma_pool, GFP_KERNEL, 157 &wa->dma.address); 158 if (!wa->address) 159 return -ENOMEM; 160 161 wa->dma.length = CCP_DMAPOOL_MAX_SIZE; 162 163 memset(wa->address, 0, CCP_DMAPOOL_MAX_SIZE); 164 } else { 165 wa->address = kzalloc(len, GFP_KERNEL); 166 if (!wa->address) 167 return -ENOMEM; 168 169 wa->dma.address = dma_map_single(wa->dev, wa->address, len, 170 dir); 171 if (!wa->dma.address) 172 return -ENOMEM; 173 174 wa->dma.length = len; 175 } 176 wa->dma.dir = dir; 177 178 return 0; 179 } 180 181 static void ccp_set_dm_area(struct ccp_dm_workarea *wa, unsigned int wa_offset, 182 struct scatterlist *sg, unsigned int sg_offset, 183 unsigned int len) 184 { 185 WARN_ON(!wa->address); 186 187 scatterwalk_map_and_copy(wa->address + wa_offset, sg, sg_offset, len, 188 0); 189 } 190 191 static void ccp_get_dm_area(struct ccp_dm_workarea *wa, unsigned int wa_offset, 192 struct scatterlist *sg, unsigned int sg_offset, 193 unsigned int len) 194 { 195 WARN_ON(!wa->address); 196 197 scatterwalk_map_and_copy(wa->address + wa_offset, sg, sg_offset, len, 198 1); 199 } 200 201 static int ccp_reverse_set_dm_area(struct ccp_dm_workarea *wa, 202 unsigned int wa_offset, 203 struct scatterlist *sg, 204 unsigned int sg_offset, 205 unsigned int len) 206 { 207 u8 *p, *q; 208 209 ccp_set_dm_area(wa, wa_offset, sg, sg_offset, len); 210 211 p = wa->address + wa_offset; 212 q = p + len - 1; 213 while (p < q) { 214 *p = *p ^ *q; 215 *q = *p ^ *q; 216 *p = *p ^ *q; 217 p++; 218 q--; 219 } 220 return 0; 221 } 222 223 static void ccp_reverse_get_dm_area(struct ccp_dm_workarea *wa, 224 unsigned int wa_offset, 225 struct scatterlist *sg, 226 unsigned int sg_offset, 227 unsigned int len) 228 { 229 u8 *p, *q; 230 231 p = wa->address + wa_offset; 232 q = p + len - 1; 233 while (p < q) { 234 *p = *p ^ *q; 235 *q = *p ^ *q; 236 *p = *p ^ *q; 237 p++; 238 q--; 239 } 240 241 ccp_get_dm_area(wa, wa_offset, sg, sg_offset, len); 242 } 243 244 static void ccp_free_data(struct ccp_data *data, struct ccp_cmd_queue *cmd_q) 245 { 246 ccp_dm_free(&data->dm_wa); 247 ccp_sg_free(&data->sg_wa); 248 } 249 250 static int ccp_init_data(struct ccp_data *data, struct ccp_cmd_queue *cmd_q, 251 struct scatterlist *sg, u64 sg_len, 252 unsigned int dm_len, 253 enum dma_data_direction dir) 254 { 255 int ret; 256 257 memset(data, 0, sizeof(*data)); 258 259 ret = ccp_init_sg_workarea(&data->sg_wa, cmd_q->ccp->dev, sg, sg_len, 260 dir); 261 if (ret) 262 goto e_err; 263 264 ret = ccp_init_dm_workarea(&data->dm_wa, cmd_q, dm_len, dir); 265 if (ret) 266 goto e_err; 267 268 return 0; 269 270 e_err: 271 ccp_free_data(data, cmd_q); 272 273 return ret; 274 } 275 276 static unsigned int ccp_queue_buf(struct ccp_data *data, unsigned int from) 277 { 278 struct ccp_sg_workarea *sg_wa = &data->sg_wa; 279 struct ccp_dm_workarea *dm_wa = &data->dm_wa; 280 unsigned int buf_count, nbytes; 281 282 /* Clear the buffer if setting it */ 283 if (!from) 284 memset(dm_wa->address, 0, dm_wa->length); 285 286 if (!sg_wa->sg) 287 return 0; 288 289 /* Perform the copy operation 290 * nbytes will always be <= UINT_MAX because dm_wa->length is 291 * an unsigned int 292 */ 293 nbytes = min_t(u64, sg_wa->bytes_left, dm_wa->length); 294 scatterwalk_map_and_copy(dm_wa->address, sg_wa->sg, sg_wa->sg_used, 295 nbytes, from); 296 297 /* Update the structures and generate the count */ 298 buf_count = 0; 299 while (sg_wa->bytes_left && (buf_count < dm_wa->length)) { 300 nbytes = min(sg_wa->sg->length - sg_wa->sg_used, 301 dm_wa->length - buf_count); 302 nbytes = min_t(u64, sg_wa->bytes_left, nbytes); 303 304 buf_count += nbytes; 305 ccp_update_sg_workarea(sg_wa, nbytes); 306 } 307 308 return buf_count; 309 } 310 311 static unsigned int ccp_fill_queue_buf(struct ccp_data *data) 312 { 313 return ccp_queue_buf(data, 0); 314 } 315 316 static unsigned int ccp_empty_queue_buf(struct ccp_data *data) 317 { 318 return ccp_queue_buf(data, 1); 319 } 320 321 static void ccp_prepare_data(struct ccp_data *src, struct ccp_data *dst, 322 struct ccp_op *op, unsigned int block_size, 323 bool blocksize_op) 324 { 325 unsigned int sg_src_len, sg_dst_len, op_len; 326 327 /* The CCP can only DMA from/to one address each per operation. This 328 * requires that we find the smallest DMA area between the source 329 * and destination. The resulting len values will always be <= UINT_MAX 330 * because the dma length is an unsigned int. 331 */ 332 sg_src_len = sg_dma_len(src->sg_wa.sg) - src->sg_wa.sg_used; 333 sg_src_len = min_t(u64, src->sg_wa.bytes_left, sg_src_len); 334 335 if (dst) { 336 sg_dst_len = sg_dma_len(dst->sg_wa.sg) - dst->sg_wa.sg_used; 337 sg_dst_len = min_t(u64, src->sg_wa.bytes_left, sg_dst_len); 338 op_len = min(sg_src_len, sg_dst_len); 339 } else { 340 op_len = sg_src_len; 341 } 342 343 /* The data operation length will be at least block_size in length 344 * or the smaller of available sg room remaining for the source or 345 * the destination 346 */ 347 op_len = max(op_len, block_size); 348 349 /* Unless we have to buffer data, there's no reason to wait */ 350 op->soc = 0; 351 352 if (sg_src_len < block_size) { 353 /* Not enough data in the sg element, so it 354 * needs to be buffered into a blocksize chunk 355 */ 356 int cp_len = ccp_fill_queue_buf(src); 357 358 op->soc = 1; 359 op->src.u.dma.address = src->dm_wa.dma.address; 360 op->src.u.dma.offset = 0; 361 op->src.u.dma.length = (blocksize_op) ? block_size : cp_len; 362 } else { 363 /* Enough data in the sg element, but we need to 364 * adjust for any previously copied data 365 */ 366 op->src.u.dma.address = sg_dma_address(src->sg_wa.sg); 367 op->src.u.dma.offset = src->sg_wa.sg_used; 368 op->src.u.dma.length = op_len & ~(block_size - 1); 369 370 ccp_update_sg_workarea(&src->sg_wa, op->src.u.dma.length); 371 } 372 373 if (dst) { 374 if (sg_dst_len < block_size) { 375 /* Not enough room in the sg element or we're on the 376 * last piece of data (when using padding), so the 377 * output needs to be buffered into a blocksize chunk 378 */ 379 op->soc = 1; 380 op->dst.u.dma.address = dst->dm_wa.dma.address; 381 op->dst.u.dma.offset = 0; 382 op->dst.u.dma.length = op->src.u.dma.length; 383 } else { 384 /* Enough room in the sg element, but we need to 385 * adjust for any previously used area 386 */ 387 op->dst.u.dma.address = sg_dma_address(dst->sg_wa.sg); 388 op->dst.u.dma.offset = dst->sg_wa.sg_used; 389 op->dst.u.dma.length = op->src.u.dma.length; 390 } 391 } 392 } 393 394 static void ccp_process_data(struct ccp_data *src, struct ccp_data *dst, 395 struct ccp_op *op) 396 { 397 op->init = 0; 398 399 if (dst) { 400 if (op->dst.u.dma.address == dst->dm_wa.dma.address) 401 ccp_empty_queue_buf(dst); 402 else 403 ccp_update_sg_workarea(&dst->sg_wa, 404 op->dst.u.dma.length); 405 } 406 } 407 408 static int ccp_copy_to_from_sb(struct ccp_cmd_queue *cmd_q, 409 struct ccp_dm_workarea *wa, u32 jobid, u32 sb, 410 u32 byte_swap, bool from) 411 { 412 struct ccp_op op; 413 414 memset(&op, 0, sizeof(op)); 415 416 op.cmd_q = cmd_q; 417 op.jobid = jobid; 418 op.eom = 1; 419 420 if (from) { 421 op.soc = 1; 422 op.src.type = CCP_MEMTYPE_SB; 423 op.src.u.sb = sb; 424 op.dst.type = CCP_MEMTYPE_SYSTEM; 425 op.dst.u.dma.address = wa->dma.address; 426 op.dst.u.dma.length = wa->length; 427 } else { 428 op.src.type = CCP_MEMTYPE_SYSTEM; 429 op.src.u.dma.address = wa->dma.address; 430 op.src.u.dma.length = wa->length; 431 op.dst.type = CCP_MEMTYPE_SB; 432 op.dst.u.sb = sb; 433 } 434 435 op.u.passthru.byte_swap = byte_swap; 436 437 return cmd_q->ccp->vdata->perform->passthru(&op); 438 } 439 440 static int ccp_copy_to_sb(struct ccp_cmd_queue *cmd_q, 441 struct ccp_dm_workarea *wa, u32 jobid, u32 sb, 442 u32 byte_swap) 443 { 444 return ccp_copy_to_from_sb(cmd_q, wa, jobid, sb, byte_swap, false); 445 } 446 447 static int ccp_copy_from_sb(struct ccp_cmd_queue *cmd_q, 448 struct ccp_dm_workarea *wa, u32 jobid, u32 sb, 449 u32 byte_swap) 450 { 451 return ccp_copy_to_from_sb(cmd_q, wa, jobid, sb, byte_swap, true); 452 } 453 454 static int ccp_run_aes_cmac_cmd(struct ccp_cmd_queue *cmd_q, 455 struct ccp_cmd *cmd) 456 { 457 struct ccp_aes_engine *aes = &cmd->u.aes; 458 struct ccp_dm_workarea key, ctx; 459 struct ccp_data src; 460 struct ccp_op op; 461 unsigned int dm_offset; 462 int ret; 463 464 if (!((aes->key_len == AES_KEYSIZE_128) || 465 (aes->key_len == AES_KEYSIZE_192) || 466 (aes->key_len == AES_KEYSIZE_256))) 467 return -EINVAL; 468 469 if (aes->src_len & (AES_BLOCK_SIZE - 1)) 470 return -EINVAL; 471 472 if (aes->iv_len != AES_BLOCK_SIZE) 473 return -EINVAL; 474 475 if (!aes->key || !aes->iv || !aes->src) 476 return -EINVAL; 477 478 if (aes->cmac_final) { 479 if (aes->cmac_key_len != AES_BLOCK_SIZE) 480 return -EINVAL; 481 482 if (!aes->cmac_key) 483 return -EINVAL; 484 } 485 486 BUILD_BUG_ON(CCP_AES_KEY_SB_COUNT != 1); 487 BUILD_BUG_ON(CCP_AES_CTX_SB_COUNT != 1); 488 489 ret = -EIO; 490 memset(&op, 0, sizeof(op)); 491 op.cmd_q = cmd_q; 492 op.jobid = CCP_NEW_JOBID(cmd_q->ccp); 493 op.sb_key = cmd_q->sb_key; 494 op.sb_ctx = cmd_q->sb_ctx; 495 op.init = 1; 496 op.u.aes.type = aes->type; 497 op.u.aes.mode = aes->mode; 498 op.u.aes.action = aes->action; 499 500 /* All supported key sizes fit in a single (32-byte) SB entry 501 * and must be in little endian format. Use the 256-bit byte 502 * swap passthru option to convert from big endian to little 503 * endian. 504 */ 505 ret = ccp_init_dm_workarea(&key, cmd_q, 506 CCP_AES_KEY_SB_COUNT * CCP_SB_BYTES, 507 DMA_TO_DEVICE); 508 if (ret) 509 return ret; 510 511 dm_offset = CCP_SB_BYTES - aes->key_len; 512 ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len); 513 ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key, 514 CCP_PASSTHRU_BYTESWAP_256BIT); 515 if (ret) { 516 cmd->engine_error = cmd_q->cmd_error; 517 goto e_key; 518 } 519 520 /* The AES context fits in a single (32-byte) SB entry and 521 * must be in little endian format. Use the 256-bit byte swap 522 * passthru option to convert from big endian to little endian. 523 */ 524 ret = ccp_init_dm_workarea(&ctx, cmd_q, 525 CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES, 526 DMA_BIDIRECTIONAL); 527 if (ret) 528 goto e_key; 529 530 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE; 531 ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len); 532 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, 533 CCP_PASSTHRU_BYTESWAP_256BIT); 534 if (ret) { 535 cmd->engine_error = cmd_q->cmd_error; 536 goto e_ctx; 537 } 538 539 /* Send data to the CCP AES engine */ 540 ret = ccp_init_data(&src, cmd_q, aes->src, aes->src_len, 541 AES_BLOCK_SIZE, DMA_TO_DEVICE); 542 if (ret) 543 goto e_ctx; 544 545 while (src.sg_wa.bytes_left) { 546 ccp_prepare_data(&src, NULL, &op, AES_BLOCK_SIZE, true); 547 if (aes->cmac_final && !src.sg_wa.bytes_left) { 548 op.eom = 1; 549 550 /* Push the K1/K2 key to the CCP now */ 551 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, 552 op.sb_ctx, 553 CCP_PASSTHRU_BYTESWAP_256BIT); 554 if (ret) { 555 cmd->engine_error = cmd_q->cmd_error; 556 goto e_src; 557 } 558 559 ccp_set_dm_area(&ctx, 0, aes->cmac_key, 0, 560 aes->cmac_key_len); 561 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, 562 CCP_PASSTHRU_BYTESWAP_256BIT); 563 if (ret) { 564 cmd->engine_error = cmd_q->cmd_error; 565 goto e_src; 566 } 567 } 568 569 ret = cmd_q->ccp->vdata->perform->aes(&op); 570 if (ret) { 571 cmd->engine_error = cmd_q->cmd_error; 572 goto e_src; 573 } 574 575 ccp_process_data(&src, NULL, &op); 576 } 577 578 /* Retrieve the AES context - convert from LE to BE using 579 * 32-byte (256-bit) byteswapping 580 */ 581 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, 582 CCP_PASSTHRU_BYTESWAP_256BIT); 583 if (ret) { 584 cmd->engine_error = cmd_q->cmd_error; 585 goto e_src; 586 } 587 588 /* ...but we only need AES_BLOCK_SIZE bytes */ 589 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE; 590 ccp_get_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len); 591 592 e_src: 593 ccp_free_data(&src, cmd_q); 594 595 e_ctx: 596 ccp_dm_free(&ctx); 597 598 e_key: 599 ccp_dm_free(&key); 600 601 return ret; 602 } 603 604 static int ccp_run_aes_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) 605 { 606 struct ccp_aes_engine *aes = &cmd->u.aes; 607 struct ccp_dm_workarea key, ctx; 608 struct ccp_data src, dst; 609 struct ccp_op op; 610 unsigned int dm_offset; 611 bool in_place = false; 612 int ret; 613 614 if (aes->mode == CCP_AES_MODE_CMAC) 615 return ccp_run_aes_cmac_cmd(cmd_q, cmd); 616 617 if (!((aes->key_len == AES_KEYSIZE_128) || 618 (aes->key_len == AES_KEYSIZE_192) || 619 (aes->key_len == AES_KEYSIZE_256))) 620 return -EINVAL; 621 622 if (((aes->mode == CCP_AES_MODE_ECB) || 623 (aes->mode == CCP_AES_MODE_CBC) || 624 (aes->mode == CCP_AES_MODE_CFB)) && 625 (aes->src_len & (AES_BLOCK_SIZE - 1))) 626 return -EINVAL; 627 628 if (!aes->key || !aes->src || !aes->dst) 629 return -EINVAL; 630 631 if (aes->mode != CCP_AES_MODE_ECB) { 632 if (aes->iv_len != AES_BLOCK_SIZE) 633 return -EINVAL; 634 635 if (!aes->iv) 636 return -EINVAL; 637 } 638 639 BUILD_BUG_ON(CCP_AES_KEY_SB_COUNT != 1); 640 BUILD_BUG_ON(CCP_AES_CTX_SB_COUNT != 1); 641 642 ret = -EIO; 643 memset(&op, 0, sizeof(op)); 644 op.cmd_q = cmd_q; 645 op.jobid = CCP_NEW_JOBID(cmd_q->ccp); 646 op.sb_key = cmd_q->sb_key; 647 op.sb_ctx = cmd_q->sb_ctx; 648 op.init = (aes->mode == CCP_AES_MODE_ECB) ? 0 : 1; 649 op.u.aes.type = aes->type; 650 op.u.aes.mode = aes->mode; 651 op.u.aes.action = aes->action; 652 653 /* All supported key sizes fit in a single (32-byte) SB entry 654 * and must be in little endian format. Use the 256-bit byte 655 * swap passthru option to convert from big endian to little 656 * endian. 657 */ 658 ret = ccp_init_dm_workarea(&key, cmd_q, 659 CCP_AES_KEY_SB_COUNT * CCP_SB_BYTES, 660 DMA_TO_DEVICE); 661 if (ret) 662 return ret; 663 664 dm_offset = CCP_SB_BYTES - aes->key_len; 665 ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len); 666 ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key, 667 CCP_PASSTHRU_BYTESWAP_256BIT); 668 if (ret) { 669 cmd->engine_error = cmd_q->cmd_error; 670 goto e_key; 671 } 672 673 /* The AES context fits in a single (32-byte) SB entry and 674 * must be in little endian format. Use the 256-bit byte swap 675 * passthru option to convert from big endian to little endian. 676 */ 677 ret = ccp_init_dm_workarea(&ctx, cmd_q, 678 CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES, 679 DMA_BIDIRECTIONAL); 680 if (ret) 681 goto e_key; 682 683 if (aes->mode != CCP_AES_MODE_ECB) { 684 /* Load the AES context - convert to LE */ 685 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE; 686 ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len); 687 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, 688 CCP_PASSTHRU_BYTESWAP_256BIT); 689 if (ret) { 690 cmd->engine_error = cmd_q->cmd_error; 691 goto e_ctx; 692 } 693 } 694 switch (aes->mode) { 695 case CCP_AES_MODE_CFB: /* CFB128 only */ 696 case CCP_AES_MODE_CTR: 697 op.u.aes.size = AES_BLOCK_SIZE * BITS_PER_BYTE - 1; 698 break; 699 default: 700 op.u.aes.size = 0; 701 } 702 703 /* Prepare the input and output data workareas. For in-place 704 * operations we need to set the dma direction to BIDIRECTIONAL 705 * and copy the src workarea to the dst workarea. 706 */ 707 if (sg_virt(aes->src) == sg_virt(aes->dst)) 708 in_place = true; 709 710 ret = ccp_init_data(&src, cmd_q, aes->src, aes->src_len, 711 AES_BLOCK_SIZE, 712 in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE); 713 if (ret) 714 goto e_ctx; 715 716 if (in_place) { 717 dst = src; 718 } else { 719 ret = ccp_init_data(&dst, cmd_q, aes->dst, aes->src_len, 720 AES_BLOCK_SIZE, DMA_FROM_DEVICE); 721 if (ret) 722 goto e_src; 723 } 724 725 /* Send data to the CCP AES engine */ 726 while (src.sg_wa.bytes_left) { 727 ccp_prepare_data(&src, &dst, &op, AES_BLOCK_SIZE, true); 728 if (!src.sg_wa.bytes_left) { 729 op.eom = 1; 730 731 /* Since we don't retrieve the AES context in ECB 732 * mode we have to wait for the operation to complete 733 * on the last piece of data 734 */ 735 if (aes->mode == CCP_AES_MODE_ECB) 736 op.soc = 1; 737 } 738 739 ret = cmd_q->ccp->vdata->perform->aes(&op); 740 if (ret) { 741 cmd->engine_error = cmd_q->cmd_error; 742 goto e_dst; 743 } 744 745 ccp_process_data(&src, &dst, &op); 746 } 747 748 if (aes->mode != CCP_AES_MODE_ECB) { 749 /* Retrieve the AES context - convert from LE to BE using 750 * 32-byte (256-bit) byteswapping 751 */ 752 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, 753 CCP_PASSTHRU_BYTESWAP_256BIT); 754 if (ret) { 755 cmd->engine_error = cmd_q->cmd_error; 756 goto e_dst; 757 } 758 759 /* ...but we only need AES_BLOCK_SIZE bytes */ 760 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE; 761 ccp_get_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len); 762 } 763 764 e_dst: 765 if (!in_place) 766 ccp_free_data(&dst, cmd_q); 767 768 e_src: 769 ccp_free_data(&src, cmd_q); 770 771 e_ctx: 772 ccp_dm_free(&ctx); 773 774 e_key: 775 ccp_dm_free(&key); 776 777 return ret; 778 } 779 780 static int ccp_run_xts_aes_cmd(struct ccp_cmd_queue *cmd_q, 781 struct ccp_cmd *cmd) 782 { 783 struct ccp_xts_aes_engine *xts = &cmd->u.xts; 784 struct ccp_dm_workarea key, ctx; 785 struct ccp_data src, dst; 786 struct ccp_op op; 787 unsigned int unit_size, dm_offset; 788 bool in_place = false; 789 int ret; 790 791 switch (xts->unit_size) { 792 case CCP_XTS_AES_UNIT_SIZE_16: 793 unit_size = 16; 794 break; 795 case CCP_XTS_AES_UNIT_SIZE_512: 796 unit_size = 512; 797 break; 798 case CCP_XTS_AES_UNIT_SIZE_1024: 799 unit_size = 1024; 800 break; 801 case CCP_XTS_AES_UNIT_SIZE_2048: 802 unit_size = 2048; 803 break; 804 case CCP_XTS_AES_UNIT_SIZE_4096: 805 unit_size = 4096; 806 break; 807 808 default: 809 return -EINVAL; 810 } 811 812 if (xts->key_len != AES_KEYSIZE_128) 813 return -EINVAL; 814 815 if (!xts->final && (xts->src_len & (AES_BLOCK_SIZE - 1))) 816 return -EINVAL; 817 818 if (xts->iv_len != AES_BLOCK_SIZE) 819 return -EINVAL; 820 821 if (!xts->key || !xts->iv || !xts->src || !xts->dst) 822 return -EINVAL; 823 824 BUILD_BUG_ON(CCP_XTS_AES_KEY_SB_COUNT != 1); 825 BUILD_BUG_ON(CCP_XTS_AES_CTX_SB_COUNT != 1); 826 827 ret = -EIO; 828 memset(&op, 0, sizeof(op)); 829 op.cmd_q = cmd_q; 830 op.jobid = CCP_NEW_JOBID(cmd_q->ccp); 831 op.sb_key = cmd_q->sb_key; 832 op.sb_ctx = cmd_q->sb_ctx; 833 op.init = 1; 834 op.u.xts.action = xts->action; 835 op.u.xts.unit_size = xts->unit_size; 836 837 /* All supported key sizes fit in a single (32-byte) SB entry 838 * and must be in little endian format. Use the 256-bit byte 839 * swap passthru option to convert from big endian to little 840 * endian. 841 */ 842 ret = ccp_init_dm_workarea(&key, cmd_q, 843 CCP_XTS_AES_KEY_SB_COUNT * CCP_SB_BYTES, 844 DMA_TO_DEVICE); 845 if (ret) 846 return ret; 847 848 dm_offset = CCP_SB_BYTES - AES_KEYSIZE_128; 849 ccp_set_dm_area(&key, dm_offset, xts->key, 0, xts->key_len); 850 ccp_set_dm_area(&key, 0, xts->key, dm_offset, xts->key_len); 851 ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key, 852 CCP_PASSTHRU_BYTESWAP_256BIT); 853 if (ret) { 854 cmd->engine_error = cmd_q->cmd_error; 855 goto e_key; 856 } 857 858 /* The AES context fits in a single (32-byte) SB entry and 859 * for XTS is already in little endian format so no byte swapping 860 * is needed. 861 */ 862 ret = ccp_init_dm_workarea(&ctx, cmd_q, 863 CCP_XTS_AES_CTX_SB_COUNT * CCP_SB_BYTES, 864 DMA_BIDIRECTIONAL); 865 if (ret) 866 goto e_key; 867 868 ccp_set_dm_area(&ctx, 0, xts->iv, 0, xts->iv_len); 869 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, 870 CCP_PASSTHRU_BYTESWAP_NOOP); 871 if (ret) { 872 cmd->engine_error = cmd_q->cmd_error; 873 goto e_ctx; 874 } 875 876 /* Prepare the input and output data workareas. For in-place 877 * operations we need to set the dma direction to BIDIRECTIONAL 878 * and copy the src workarea to the dst workarea. 879 */ 880 if (sg_virt(xts->src) == sg_virt(xts->dst)) 881 in_place = true; 882 883 ret = ccp_init_data(&src, cmd_q, xts->src, xts->src_len, 884 unit_size, 885 in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE); 886 if (ret) 887 goto e_ctx; 888 889 if (in_place) { 890 dst = src; 891 } else { 892 ret = ccp_init_data(&dst, cmd_q, xts->dst, xts->src_len, 893 unit_size, DMA_FROM_DEVICE); 894 if (ret) 895 goto e_src; 896 } 897 898 /* Send data to the CCP AES engine */ 899 while (src.sg_wa.bytes_left) { 900 ccp_prepare_data(&src, &dst, &op, unit_size, true); 901 if (!src.sg_wa.bytes_left) 902 op.eom = 1; 903 904 ret = cmd_q->ccp->vdata->perform->xts_aes(&op); 905 if (ret) { 906 cmd->engine_error = cmd_q->cmd_error; 907 goto e_dst; 908 } 909 910 ccp_process_data(&src, &dst, &op); 911 } 912 913 /* Retrieve the AES context - convert from LE to BE using 914 * 32-byte (256-bit) byteswapping 915 */ 916 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, 917 CCP_PASSTHRU_BYTESWAP_256BIT); 918 if (ret) { 919 cmd->engine_error = cmd_q->cmd_error; 920 goto e_dst; 921 } 922 923 /* ...but we only need AES_BLOCK_SIZE bytes */ 924 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE; 925 ccp_get_dm_area(&ctx, dm_offset, xts->iv, 0, xts->iv_len); 926 927 e_dst: 928 if (!in_place) 929 ccp_free_data(&dst, cmd_q); 930 931 e_src: 932 ccp_free_data(&src, cmd_q); 933 934 e_ctx: 935 ccp_dm_free(&ctx); 936 937 e_key: 938 ccp_dm_free(&key); 939 940 return ret; 941 } 942 943 static int ccp_run_des3_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) 944 { 945 struct ccp_des3_engine *des3 = &cmd->u.des3; 946 947 struct ccp_dm_workarea key, ctx; 948 struct ccp_data src, dst; 949 struct ccp_op op; 950 unsigned int dm_offset; 951 unsigned int len_singlekey; 952 bool in_place = false; 953 int ret; 954 955 /* Error checks */ 956 if (!cmd_q->ccp->vdata->perform->des3) 957 return -EINVAL; 958 959 if (des3->key_len != DES3_EDE_KEY_SIZE) 960 return -EINVAL; 961 962 if (((des3->mode == CCP_DES3_MODE_ECB) || 963 (des3->mode == CCP_DES3_MODE_CBC)) && 964 (des3->src_len & (DES3_EDE_BLOCK_SIZE - 1))) 965 return -EINVAL; 966 967 if (!des3->key || !des3->src || !des3->dst) 968 return -EINVAL; 969 970 if (des3->mode != CCP_DES3_MODE_ECB) { 971 if (des3->iv_len != DES3_EDE_BLOCK_SIZE) 972 return -EINVAL; 973 974 if (!des3->iv) 975 return -EINVAL; 976 } 977 978 ret = -EIO; 979 /* Zero out all the fields of the command desc */ 980 memset(&op, 0, sizeof(op)); 981 982 /* Set up the Function field */ 983 op.cmd_q = cmd_q; 984 op.jobid = CCP_NEW_JOBID(cmd_q->ccp); 985 op.sb_key = cmd_q->sb_key; 986 987 op.init = (des3->mode == CCP_DES3_MODE_ECB) ? 0 : 1; 988 op.u.des3.type = des3->type; 989 op.u.des3.mode = des3->mode; 990 op.u.des3.action = des3->action; 991 992 /* 993 * All supported key sizes fit in a single (32-byte) KSB entry and 994 * (like AES) must be in little endian format. Use the 256-bit byte 995 * swap passthru option to convert from big endian to little endian. 996 */ 997 ret = ccp_init_dm_workarea(&key, cmd_q, 998 CCP_DES3_KEY_SB_COUNT * CCP_SB_BYTES, 999 DMA_TO_DEVICE); 1000 if (ret) 1001 return ret; 1002 1003 /* 1004 * The contents of the key triplet are in the reverse order of what 1005 * is required by the engine. Copy the 3 pieces individually to put 1006 * them where they belong. 1007 */ 1008 dm_offset = CCP_SB_BYTES - des3->key_len; /* Basic offset */ 1009 1010 len_singlekey = des3->key_len / 3; 1011 ccp_set_dm_area(&key, dm_offset + 2 * len_singlekey, 1012 des3->key, 0, len_singlekey); 1013 ccp_set_dm_area(&key, dm_offset + len_singlekey, 1014 des3->key, len_singlekey, len_singlekey); 1015 ccp_set_dm_area(&key, dm_offset, 1016 des3->key, 2 * len_singlekey, len_singlekey); 1017 1018 /* Copy the key to the SB */ 1019 ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key, 1020 CCP_PASSTHRU_BYTESWAP_256BIT); 1021 if (ret) { 1022 cmd->engine_error = cmd_q->cmd_error; 1023 goto e_key; 1024 } 1025 1026 /* 1027 * The DES3 context fits in a single (32-byte) KSB entry and 1028 * must be in little endian format. Use the 256-bit byte swap 1029 * passthru option to convert from big endian to little endian. 1030 */ 1031 if (des3->mode != CCP_DES3_MODE_ECB) { 1032 u32 load_mode; 1033 1034 op.sb_ctx = cmd_q->sb_ctx; 1035 1036 ret = ccp_init_dm_workarea(&ctx, cmd_q, 1037 CCP_DES3_CTX_SB_COUNT * CCP_SB_BYTES, 1038 DMA_BIDIRECTIONAL); 1039 if (ret) 1040 goto e_key; 1041 1042 /* Load the context into the LSB */ 1043 dm_offset = CCP_SB_BYTES - des3->iv_len; 1044 ccp_set_dm_area(&ctx, dm_offset, des3->iv, 0, des3->iv_len); 1045 1046 if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0)) 1047 load_mode = CCP_PASSTHRU_BYTESWAP_NOOP; 1048 else 1049 load_mode = CCP_PASSTHRU_BYTESWAP_256BIT; 1050 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, 1051 load_mode); 1052 if (ret) { 1053 cmd->engine_error = cmd_q->cmd_error; 1054 goto e_ctx; 1055 } 1056 } 1057 1058 /* 1059 * Prepare the input and output data workareas. For in-place 1060 * operations we need to set the dma direction to BIDIRECTIONAL 1061 * and copy the src workarea to the dst workarea. 1062 */ 1063 if (sg_virt(des3->src) == sg_virt(des3->dst)) 1064 in_place = true; 1065 1066 ret = ccp_init_data(&src, cmd_q, des3->src, des3->src_len, 1067 DES3_EDE_BLOCK_SIZE, 1068 in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE); 1069 if (ret) 1070 goto e_ctx; 1071 1072 if (in_place) 1073 dst = src; 1074 else { 1075 ret = ccp_init_data(&dst, cmd_q, des3->dst, des3->src_len, 1076 DES3_EDE_BLOCK_SIZE, DMA_FROM_DEVICE); 1077 if (ret) 1078 goto e_src; 1079 } 1080 1081 /* Send data to the CCP DES3 engine */ 1082 while (src.sg_wa.bytes_left) { 1083 ccp_prepare_data(&src, &dst, &op, DES3_EDE_BLOCK_SIZE, true); 1084 if (!src.sg_wa.bytes_left) { 1085 op.eom = 1; 1086 1087 /* Since we don't retrieve the context in ECB mode 1088 * we have to wait for the operation to complete 1089 * on the last piece of data 1090 */ 1091 op.soc = 0; 1092 } 1093 1094 ret = cmd_q->ccp->vdata->perform->des3(&op); 1095 if (ret) { 1096 cmd->engine_error = cmd_q->cmd_error; 1097 goto e_dst; 1098 } 1099 1100 ccp_process_data(&src, &dst, &op); 1101 } 1102 1103 if (des3->mode != CCP_DES3_MODE_ECB) { 1104 /* Retrieve the context and make BE */ 1105 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, 1106 CCP_PASSTHRU_BYTESWAP_256BIT); 1107 if (ret) { 1108 cmd->engine_error = cmd_q->cmd_error; 1109 goto e_dst; 1110 } 1111 1112 /* ...but we only need the last DES3_EDE_BLOCK_SIZE bytes */ 1113 if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0)) 1114 dm_offset = CCP_SB_BYTES - des3->iv_len; 1115 else 1116 dm_offset = 0; 1117 ccp_get_dm_area(&ctx, dm_offset, des3->iv, 0, 1118 DES3_EDE_BLOCK_SIZE); 1119 } 1120 e_dst: 1121 if (!in_place) 1122 ccp_free_data(&dst, cmd_q); 1123 1124 e_src: 1125 ccp_free_data(&src, cmd_q); 1126 1127 e_ctx: 1128 if (des3->mode != CCP_DES3_MODE_ECB) 1129 ccp_dm_free(&ctx); 1130 1131 e_key: 1132 ccp_dm_free(&key); 1133 1134 return ret; 1135 } 1136 1137 static int ccp_run_sha_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) 1138 { 1139 struct ccp_sha_engine *sha = &cmd->u.sha; 1140 struct ccp_dm_workarea ctx; 1141 struct ccp_data src; 1142 struct ccp_op op; 1143 unsigned int ioffset, ooffset; 1144 unsigned int digest_size; 1145 int sb_count; 1146 const void *init; 1147 u64 block_size; 1148 int ctx_size; 1149 int ret; 1150 1151 switch (sha->type) { 1152 case CCP_SHA_TYPE_1: 1153 if (sha->ctx_len < SHA1_DIGEST_SIZE) 1154 return -EINVAL; 1155 block_size = SHA1_BLOCK_SIZE; 1156 break; 1157 case CCP_SHA_TYPE_224: 1158 if (sha->ctx_len < SHA224_DIGEST_SIZE) 1159 return -EINVAL; 1160 block_size = SHA224_BLOCK_SIZE; 1161 break; 1162 case CCP_SHA_TYPE_256: 1163 if (sha->ctx_len < SHA256_DIGEST_SIZE) 1164 return -EINVAL; 1165 block_size = SHA256_BLOCK_SIZE; 1166 break; 1167 case CCP_SHA_TYPE_384: 1168 if (cmd_q->ccp->vdata->version < CCP_VERSION(4, 0) 1169 || sha->ctx_len < SHA384_DIGEST_SIZE) 1170 return -EINVAL; 1171 block_size = SHA384_BLOCK_SIZE; 1172 break; 1173 case CCP_SHA_TYPE_512: 1174 if (cmd_q->ccp->vdata->version < CCP_VERSION(4, 0) 1175 || sha->ctx_len < SHA512_DIGEST_SIZE) 1176 return -EINVAL; 1177 block_size = SHA512_BLOCK_SIZE; 1178 break; 1179 default: 1180 return -EINVAL; 1181 } 1182 1183 if (!sha->ctx) 1184 return -EINVAL; 1185 1186 if (!sha->final && (sha->src_len & (block_size - 1))) 1187 return -EINVAL; 1188 1189 /* The version 3 device can't handle zero-length input */ 1190 if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0)) { 1191 1192 if (!sha->src_len) { 1193 unsigned int digest_len; 1194 const u8 *sha_zero; 1195 1196 /* Not final, just return */ 1197 if (!sha->final) 1198 return 0; 1199 1200 /* CCP can't do a zero length sha operation so the 1201 * caller must buffer the data. 1202 */ 1203 if (sha->msg_bits) 1204 return -EINVAL; 1205 1206 /* The CCP cannot perform zero-length sha operations 1207 * so the caller is required to buffer data for the 1208 * final operation. However, a sha operation for a 1209 * message with a total length of zero is valid so 1210 * known values are required to supply the result. 1211 */ 1212 switch (sha->type) { 1213 case CCP_SHA_TYPE_1: 1214 sha_zero = sha1_zero_message_hash; 1215 digest_len = SHA1_DIGEST_SIZE; 1216 break; 1217 case CCP_SHA_TYPE_224: 1218 sha_zero = sha224_zero_message_hash; 1219 digest_len = SHA224_DIGEST_SIZE; 1220 break; 1221 case CCP_SHA_TYPE_256: 1222 sha_zero = sha256_zero_message_hash; 1223 digest_len = SHA256_DIGEST_SIZE; 1224 break; 1225 default: 1226 return -EINVAL; 1227 } 1228 1229 scatterwalk_map_and_copy((void *)sha_zero, sha->ctx, 0, 1230 digest_len, 1); 1231 1232 return 0; 1233 } 1234 } 1235 1236 /* Set variables used throughout */ 1237 switch (sha->type) { 1238 case CCP_SHA_TYPE_1: 1239 digest_size = SHA1_DIGEST_SIZE; 1240 init = (void *) ccp_sha1_init; 1241 ctx_size = SHA1_DIGEST_SIZE; 1242 sb_count = 1; 1243 if (cmd_q->ccp->vdata->version != CCP_VERSION(3, 0)) 1244 ooffset = ioffset = CCP_SB_BYTES - SHA1_DIGEST_SIZE; 1245 else 1246 ooffset = ioffset = 0; 1247 break; 1248 case CCP_SHA_TYPE_224: 1249 digest_size = SHA224_DIGEST_SIZE; 1250 init = (void *) ccp_sha224_init; 1251 ctx_size = SHA256_DIGEST_SIZE; 1252 sb_count = 1; 1253 ioffset = 0; 1254 if (cmd_q->ccp->vdata->version != CCP_VERSION(3, 0)) 1255 ooffset = CCP_SB_BYTES - SHA224_DIGEST_SIZE; 1256 else 1257 ooffset = 0; 1258 break; 1259 case CCP_SHA_TYPE_256: 1260 digest_size = SHA256_DIGEST_SIZE; 1261 init = (void *) ccp_sha256_init; 1262 ctx_size = SHA256_DIGEST_SIZE; 1263 sb_count = 1; 1264 ooffset = ioffset = 0; 1265 break; 1266 case CCP_SHA_TYPE_384: 1267 digest_size = SHA384_DIGEST_SIZE; 1268 init = (void *) ccp_sha384_init; 1269 ctx_size = SHA512_DIGEST_SIZE; 1270 sb_count = 2; 1271 ioffset = 0; 1272 ooffset = 2 * CCP_SB_BYTES - SHA384_DIGEST_SIZE; 1273 break; 1274 case CCP_SHA_TYPE_512: 1275 digest_size = SHA512_DIGEST_SIZE; 1276 init = (void *) ccp_sha512_init; 1277 ctx_size = SHA512_DIGEST_SIZE; 1278 sb_count = 2; 1279 ooffset = ioffset = 0; 1280 break; 1281 default: 1282 ret = -EINVAL; 1283 goto e_data; 1284 } 1285 1286 /* For zero-length plaintext the src pointer is ignored; 1287 * otherwise both parts must be valid 1288 */ 1289 if (sha->src_len && !sha->src) 1290 return -EINVAL; 1291 1292 memset(&op, 0, sizeof(op)); 1293 op.cmd_q = cmd_q; 1294 op.jobid = CCP_NEW_JOBID(cmd_q->ccp); 1295 op.sb_ctx = cmd_q->sb_ctx; /* Pre-allocated */ 1296 op.u.sha.type = sha->type; 1297 op.u.sha.msg_bits = sha->msg_bits; 1298 1299 /* For SHA1/224/256 the context fits in a single (32-byte) SB entry; 1300 * SHA384/512 require 2 adjacent SB slots, with the right half in the 1301 * first slot, and the left half in the second. Each portion must then 1302 * be in little endian format: use the 256-bit byte swap option. 1303 */ 1304 ret = ccp_init_dm_workarea(&ctx, cmd_q, sb_count * CCP_SB_BYTES, 1305 DMA_BIDIRECTIONAL); 1306 if (ret) 1307 return ret; 1308 if (sha->first) { 1309 switch (sha->type) { 1310 case CCP_SHA_TYPE_1: 1311 case CCP_SHA_TYPE_224: 1312 case CCP_SHA_TYPE_256: 1313 memcpy(ctx.address + ioffset, init, ctx_size); 1314 break; 1315 case CCP_SHA_TYPE_384: 1316 case CCP_SHA_TYPE_512: 1317 memcpy(ctx.address + ctx_size / 2, init, 1318 ctx_size / 2); 1319 memcpy(ctx.address, init + ctx_size / 2, 1320 ctx_size / 2); 1321 break; 1322 default: 1323 ret = -EINVAL; 1324 goto e_ctx; 1325 } 1326 } else { 1327 /* Restore the context */ 1328 ccp_set_dm_area(&ctx, 0, sha->ctx, 0, 1329 sb_count * CCP_SB_BYTES); 1330 } 1331 1332 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, 1333 CCP_PASSTHRU_BYTESWAP_256BIT); 1334 if (ret) { 1335 cmd->engine_error = cmd_q->cmd_error; 1336 goto e_ctx; 1337 } 1338 1339 if (sha->src) { 1340 /* Send data to the CCP SHA engine; block_size is set above */ 1341 ret = ccp_init_data(&src, cmd_q, sha->src, sha->src_len, 1342 block_size, DMA_TO_DEVICE); 1343 if (ret) 1344 goto e_ctx; 1345 1346 while (src.sg_wa.bytes_left) { 1347 ccp_prepare_data(&src, NULL, &op, block_size, false); 1348 if (sha->final && !src.sg_wa.bytes_left) 1349 op.eom = 1; 1350 1351 ret = cmd_q->ccp->vdata->perform->sha(&op); 1352 if (ret) { 1353 cmd->engine_error = cmd_q->cmd_error; 1354 goto e_data; 1355 } 1356 1357 ccp_process_data(&src, NULL, &op); 1358 } 1359 } else { 1360 op.eom = 1; 1361 ret = cmd_q->ccp->vdata->perform->sha(&op); 1362 if (ret) { 1363 cmd->engine_error = cmd_q->cmd_error; 1364 goto e_data; 1365 } 1366 } 1367 1368 /* Retrieve the SHA context - convert from LE to BE using 1369 * 32-byte (256-bit) byteswapping to BE 1370 */ 1371 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, 1372 CCP_PASSTHRU_BYTESWAP_256BIT); 1373 if (ret) { 1374 cmd->engine_error = cmd_q->cmd_error; 1375 goto e_data; 1376 } 1377 1378 if (sha->final) { 1379 /* Finishing up, so get the digest */ 1380 switch (sha->type) { 1381 case CCP_SHA_TYPE_1: 1382 case CCP_SHA_TYPE_224: 1383 case CCP_SHA_TYPE_256: 1384 ccp_get_dm_area(&ctx, ooffset, 1385 sha->ctx, 0, 1386 digest_size); 1387 break; 1388 case CCP_SHA_TYPE_384: 1389 case CCP_SHA_TYPE_512: 1390 ccp_get_dm_area(&ctx, 0, 1391 sha->ctx, LSB_ITEM_SIZE - ooffset, 1392 LSB_ITEM_SIZE); 1393 ccp_get_dm_area(&ctx, LSB_ITEM_SIZE + ooffset, 1394 sha->ctx, 0, 1395 LSB_ITEM_SIZE - ooffset); 1396 break; 1397 default: 1398 ret = -EINVAL; 1399 goto e_ctx; 1400 } 1401 } else { 1402 /* Stash the context */ 1403 ccp_get_dm_area(&ctx, 0, sha->ctx, 0, 1404 sb_count * CCP_SB_BYTES); 1405 } 1406 1407 if (sha->final && sha->opad) { 1408 /* HMAC operation, recursively perform final SHA */ 1409 struct ccp_cmd hmac_cmd; 1410 struct scatterlist sg; 1411 u8 *hmac_buf; 1412 1413 if (sha->opad_len != block_size) { 1414 ret = -EINVAL; 1415 goto e_data; 1416 } 1417 1418 hmac_buf = kmalloc(block_size + digest_size, GFP_KERNEL); 1419 if (!hmac_buf) { 1420 ret = -ENOMEM; 1421 goto e_data; 1422 } 1423 sg_init_one(&sg, hmac_buf, block_size + digest_size); 1424 1425 scatterwalk_map_and_copy(hmac_buf, sha->opad, 0, block_size, 0); 1426 switch (sha->type) { 1427 case CCP_SHA_TYPE_1: 1428 case CCP_SHA_TYPE_224: 1429 case CCP_SHA_TYPE_256: 1430 memcpy(hmac_buf + block_size, 1431 ctx.address + ooffset, 1432 digest_size); 1433 break; 1434 case CCP_SHA_TYPE_384: 1435 case CCP_SHA_TYPE_512: 1436 memcpy(hmac_buf + block_size, 1437 ctx.address + LSB_ITEM_SIZE + ooffset, 1438 LSB_ITEM_SIZE); 1439 memcpy(hmac_buf + block_size + 1440 (LSB_ITEM_SIZE - ooffset), 1441 ctx.address, 1442 LSB_ITEM_SIZE); 1443 break; 1444 default: 1445 ret = -EINVAL; 1446 goto e_ctx; 1447 } 1448 1449 memset(&hmac_cmd, 0, sizeof(hmac_cmd)); 1450 hmac_cmd.engine = CCP_ENGINE_SHA; 1451 hmac_cmd.u.sha.type = sha->type; 1452 hmac_cmd.u.sha.ctx = sha->ctx; 1453 hmac_cmd.u.sha.ctx_len = sha->ctx_len; 1454 hmac_cmd.u.sha.src = &sg; 1455 hmac_cmd.u.sha.src_len = block_size + digest_size; 1456 hmac_cmd.u.sha.opad = NULL; 1457 hmac_cmd.u.sha.opad_len = 0; 1458 hmac_cmd.u.sha.first = 1; 1459 hmac_cmd.u.sha.final = 1; 1460 hmac_cmd.u.sha.msg_bits = (block_size + digest_size) << 3; 1461 1462 ret = ccp_run_sha_cmd(cmd_q, &hmac_cmd); 1463 if (ret) 1464 cmd->engine_error = hmac_cmd.engine_error; 1465 1466 kfree(hmac_buf); 1467 } 1468 1469 e_data: 1470 if (sha->src) 1471 ccp_free_data(&src, cmd_q); 1472 1473 e_ctx: 1474 ccp_dm_free(&ctx); 1475 1476 return ret; 1477 } 1478 1479 static int ccp_run_rsa_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) 1480 { 1481 struct ccp_rsa_engine *rsa = &cmd->u.rsa; 1482 struct ccp_dm_workarea exp, src; 1483 struct ccp_data dst; 1484 struct ccp_op op; 1485 unsigned int sb_count, i_len, o_len; 1486 int ret; 1487 1488 if (rsa->key_size > CCP_RSA_MAX_WIDTH) 1489 return -EINVAL; 1490 1491 if (!rsa->exp || !rsa->mod || !rsa->src || !rsa->dst) 1492 return -EINVAL; 1493 1494 /* The RSA modulus must precede the message being acted upon, so 1495 * it must be copied to a DMA area where the message and the 1496 * modulus can be concatenated. Therefore the input buffer 1497 * length required is twice the output buffer length (which 1498 * must be a multiple of 256-bits). 1499 */ 1500 o_len = ((rsa->key_size + 255) / 256) * 32; 1501 i_len = o_len * 2; 1502 1503 sb_count = o_len / CCP_SB_BYTES; 1504 1505 memset(&op, 0, sizeof(op)); 1506 op.cmd_q = cmd_q; 1507 op.jobid = ccp_gen_jobid(cmd_q->ccp); 1508 op.sb_key = cmd_q->ccp->vdata->perform->sballoc(cmd_q, sb_count); 1509 1510 if (!op.sb_key) 1511 return -EIO; 1512 1513 /* The RSA exponent may span multiple (32-byte) SB entries and must 1514 * be in little endian format. Reverse copy each 32-byte chunk 1515 * of the exponent (En chunk to E0 chunk, E(n-1) chunk to E1 chunk) 1516 * and each byte within that chunk and do not perform any byte swap 1517 * operations on the passthru operation. 1518 */ 1519 ret = ccp_init_dm_workarea(&exp, cmd_q, o_len, DMA_TO_DEVICE); 1520 if (ret) 1521 goto e_sb; 1522 1523 ret = ccp_reverse_set_dm_area(&exp, 0, rsa->exp, 0, rsa->exp_len); 1524 if (ret) 1525 goto e_exp; 1526 ret = ccp_copy_to_sb(cmd_q, &exp, op.jobid, op.sb_key, 1527 CCP_PASSTHRU_BYTESWAP_NOOP); 1528 if (ret) { 1529 cmd->engine_error = cmd_q->cmd_error; 1530 goto e_exp; 1531 } 1532 1533 /* Concatenate the modulus and the message. Both the modulus and 1534 * the operands must be in little endian format. Since the input 1535 * is in big endian format it must be converted. 1536 */ 1537 ret = ccp_init_dm_workarea(&src, cmd_q, i_len, DMA_TO_DEVICE); 1538 if (ret) 1539 goto e_exp; 1540 1541 ret = ccp_reverse_set_dm_area(&src, 0, rsa->mod, 0, rsa->mod_len); 1542 if (ret) 1543 goto e_src; 1544 ret = ccp_reverse_set_dm_area(&src, o_len, rsa->src, 0, rsa->src_len); 1545 if (ret) 1546 goto e_src; 1547 1548 /* Prepare the output area for the operation */ 1549 ret = ccp_init_data(&dst, cmd_q, rsa->dst, rsa->mod_len, 1550 o_len, DMA_FROM_DEVICE); 1551 if (ret) 1552 goto e_src; 1553 1554 op.soc = 1; 1555 op.src.u.dma.address = src.dma.address; 1556 op.src.u.dma.offset = 0; 1557 op.src.u.dma.length = i_len; 1558 op.dst.u.dma.address = dst.dm_wa.dma.address; 1559 op.dst.u.dma.offset = 0; 1560 op.dst.u.dma.length = o_len; 1561 1562 op.u.rsa.mod_size = rsa->key_size; 1563 op.u.rsa.input_len = i_len; 1564 1565 ret = cmd_q->ccp->vdata->perform->rsa(&op); 1566 if (ret) { 1567 cmd->engine_error = cmd_q->cmd_error; 1568 goto e_dst; 1569 } 1570 1571 ccp_reverse_get_dm_area(&dst.dm_wa, 0, rsa->dst, 0, rsa->mod_len); 1572 1573 e_dst: 1574 ccp_free_data(&dst, cmd_q); 1575 1576 e_src: 1577 ccp_dm_free(&src); 1578 1579 e_exp: 1580 ccp_dm_free(&exp); 1581 1582 e_sb: 1583 cmd_q->ccp->vdata->perform->sbfree(cmd_q, op.sb_key, sb_count); 1584 1585 return ret; 1586 } 1587 1588 static int ccp_run_passthru_cmd(struct ccp_cmd_queue *cmd_q, 1589 struct ccp_cmd *cmd) 1590 { 1591 struct ccp_passthru_engine *pt = &cmd->u.passthru; 1592 struct ccp_dm_workarea mask; 1593 struct ccp_data src, dst; 1594 struct ccp_op op; 1595 bool in_place = false; 1596 unsigned int i; 1597 int ret = 0; 1598 1599 if (!pt->final && (pt->src_len & (CCP_PASSTHRU_BLOCKSIZE - 1))) 1600 return -EINVAL; 1601 1602 if (!pt->src || !pt->dst) 1603 return -EINVAL; 1604 1605 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) { 1606 if (pt->mask_len != CCP_PASSTHRU_MASKSIZE) 1607 return -EINVAL; 1608 if (!pt->mask) 1609 return -EINVAL; 1610 } 1611 1612 BUILD_BUG_ON(CCP_PASSTHRU_SB_COUNT != 1); 1613 1614 memset(&op, 0, sizeof(op)); 1615 op.cmd_q = cmd_q; 1616 op.jobid = CCP_NEW_JOBID(cmd_q->ccp); 1617 1618 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) { 1619 /* Load the mask */ 1620 op.sb_key = cmd_q->sb_key; 1621 1622 ret = ccp_init_dm_workarea(&mask, cmd_q, 1623 CCP_PASSTHRU_SB_COUNT * 1624 CCP_SB_BYTES, 1625 DMA_TO_DEVICE); 1626 if (ret) 1627 return ret; 1628 1629 ccp_set_dm_area(&mask, 0, pt->mask, 0, pt->mask_len); 1630 ret = ccp_copy_to_sb(cmd_q, &mask, op.jobid, op.sb_key, 1631 CCP_PASSTHRU_BYTESWAP_NOOP); 1632 if (ret) { 1633 cmd->engine_error = cmd_q->cmd_error; 1634 goto e_mask; 1635 } 1636 } 1637 1638 /* Prepare the input and output data workareas. For in-place 1639 * operations we need to set the dma direction to BIDIRECTIONAL 1640 * and copy the src workarea to the dst workarea. 1641 */ 1642 if (sg_virt(pt->src) == sg_virt(pt->dst)) 1643 in_place = true; 1644 1645 ret = ccp_init_data(&src, cmd_q, pt->src, pt->src_len, 1646 CCP_PASSTHRU_MASKSIZE, 1647 in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE); 1648 if (ret) 1649 goto e_mask; 1650 1651 if (in_place) { 1652 dst = src; 1653 } else { 1654 ret = ccp_init_data(&dst, cmd_q, pt->dst, pt->src_len, 1655 CCP_PASSTHRU_MASKSIZE, DMA_FROM_DEVICE); 1656 if (ret) 1657 goto e_src; 1658 } 1659 1660 /* Send data to the CCP Passthru engine 1661 * Because the CCP engine works on a single source and destination 1662 * dma address at a time, each entry in the source scatterlist 1663 * (after the dma_map_sg call) must be less than or equal to the 1664 * (remaining) length in the destination scatterlist entry and the 1665 * length must be a multiple of CCP_PASSTHRU_BLOCKSIZE 1666 */ 1667 dst.sg_wa.sg_used = 0; 1668 for (i = 1; i <= src.sg_wa.dma_count; i++) { 1669 if (!dst.sg_wa.sg || 1670 (dst.sg_wa.sg->length < src.sg_wa.sg->length)) { 1671 ret = -EINVAL; 1672 goto e_dst; 1673 } 1674 1675 if (i == src.sg_wa.dma_count) { 1676 op.eom = 1; 1677 op.soc = 1; 1678 } 1679 1680 op.src.type = CCP_MEMTYPE_SYSTEM; 1681 op.src.u.dma.address = sg_dma_address(src.sg_wa.sg); 1682 op.src.u.dma.offset = 0; 1683 op.src.u.dma.length = sg_dma_len(src.sg_wa.sg); 1684 1685 op.dst.type = CCP_MEMTYPE_SYSTEM; 1686 op.dst.u.dma.address = sg_dma_address(dst.sg_wa.sg); 1687 op.dst.u.dma.offset = dst.sg_wa.sg_used; 1688 op.dst.u.dma.length = op.src.u.dma.length; 1689 1690 ret = cmd_q->ccp->vdata->perform->passthru(&op); 1691 if (ret) { 1692 cmd->engine_error = cmd_q->cmd_error; 1693 goto e_dst; 1694 } 1695 1696 dst.sg_wa.sg_used += src.sg_wa.sg->length; 1697 if (dst.sg_wa.sg_used == dst.sg_wa.sg->length) { 1698 dst.sg_wa.sg = sg_next(dst.sg_wa.sg); 1699 dst.sg_wa.sg_used = 0; 1700 } 1701 src.sg_wa.sg = sg_next(src.sg_wa.sg); 1702 } 1703 1704 e_dst: 1705 if (!in_place) 1706 ccp_free_data(&dst, cmd_q); 1707 1708 e_src: 1709 ccp_free_data(&src, cmd_q); 1710 1711 e_mask: 1712 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) 1713 ccp_dm_free(&mask); 1714 1715 return ret; 1716 } 1717 1718 static int ccp_run_passthru_nomap_cmd(struct ccp_cmd_queue *cmd_q, 1719 struct ccp_cmd *cmd) 1720 { 1721 struct ccp_passthru_nomap_engine *pt = &cmd->u.passthru_nomap; 1722 struct ccp_dm_workarea mask; 1723 struct ccp_op op; 1724 int ret; 1725 1726 if (!pt->final && (pt->src_len & (CCP_PASSTHRU_BLOCKSIZE - 1))) 1727 return -EINVAL; 1728 1729 if (!pt->src_dma || !pt->dst_dma) 1730 return -EINVAL; 1731 1732 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) { 1733 if (pt->mask_len != CCP_PASSTHRU_MASKSIZE) 1734 return -EINVAL; 1735 if (!pt->mask) 1736 return -EINVAL; 1737 } 1738 1739 BUILD_BUG_ON(CCP_PASSTHRU_SB_COUNT != 1); 1740 1741 memset(&op, 0, sizeof(op)); 1742 op.cmd_q = cmd_q; 1743 op.jobid = ccp_gen_jobid(cmd_q->ccp); 1744 1745 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) { 1746 /* Load the mask */ 1747 op.sb_key = cmd_q->sb_key; 1748 1749 mask.length = pt->mask_len; 1750 mask.dma.address = pt->mask; 1751 mask.dma.length = pt->mask_len; 1752 1753 ret = ccp_copy_to_sb(cmd_q, &mask, op.jobid, op.sb_key, 1754 CCP_PASSTHRU_BYTESWAP_NOOP); 1755 if (ret) { 1756 cmd->engine_error = cmd_q->cmd_error; 1757 return ret; 1758 } 1759 } 1760 1761 /* Send data to the CCP Passthru engine */ 1762 op.eom = 1; 1763 op.soc = 1; 1764 1765 op.src.type = CCP_MEMTYPE_SYSTEM; 1766 op.src.u.dma.address = pt->src_dma; 1767 op.src.u.dma.offset = 0; 1768 op.src.u.dma.length = pt->src_len; 1769 1770 op.dst.type = CCP_MEMTYPE_SYSTEM; 1771 op.dst.u.dma.address = pt->dst_dma; 1772 op.dst.u.dma.offset = 0; 1773 op.dst.u.dma.length = pt->src_len; 1774 1775 ret = cmd_q->ccp->vdata->perform->passthru(&op); 1776 if (ret) 1777 cmd->engine_error = cmd_q->cmd_error; 1778 1779 return ret; 1780 } 1781 1782 static int ccp_run_ecc_mm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) 1783 { 1784 struct ccp_ecc_engine *ecc = &cmd->u.ecc; 1785 struct ccp_dm_workarea src, dst; 1786 struct ccp_op op; 1787 int ret; 1788 u8 *save; 1789 1790 if (!ecc->u.mm.operand_1 || 1791 (ecc->u.mm.operand_1_len > CCP_ECC_MODULUS_BYTES)) 1792 return -EINVAL; 1793 1794 if (ecc->function != CCP_ECC_FUNCTION_MINV_384BIT) 1795 if (!ecc->u.mm.operand_2 || 1796 (ecc->u.mm.operand_2_len > CCP_ECC_MODULUS_BYTES)) 1797 return -EINVAL; 1798 1799 if (!ecc->u.mm.result || 1800 (ecc->u.mm.result_len < CCP_ECC_MODULUS_BYTES)) 1801 return -EINVAL; 1802 1803 memset(&op, 0, sizeof(op)); 1804 op.cmd_q = cmd_q; 1805 op.jobid = CCP_NEW_JOBID(cmd_q->ccp); 1806 1807 /* Concatenate the modulus and the operands. Both the modulus and 1808 * the operands must be in little endian format. Since the input 1809 * is in big endian format it must be converted and placed in a 1810 * fixed length buffer. 1811 */ 1812 ret = ccp_init_dm_workarea(&src, cmd_q, CCP_ECC_SRC_BUF_SIZE, 1813 DMA_TO_DEVICE); 1814 if (ret) 1815 return ret; 1816 1817 /* Save the workarea address since it is updated in order to perform 1818 * the concatenation 1819 */ 1820 save = src.address; 1821 1822 /* Copy the ECC modulus */ 1823 ret = ccp_reverse_set_dm_area(&src, 0, ecc->mod, 0, ecc->mod_len); 1824 if (ret) 1825 goto e_src; 1826 src.address += CCP_ECC_OPERAND_SIZE; 1827 1828 /* Copy the first operand */ 1829 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.mm.operand_1, 0, 1830 ecc->u.mm.operand_1_len); 1831 if (ret) 1832 goto e_src; 1833 src.address += CCP_ECC_OPERAND_SIZE; 1834 1835 if (ecc->function != CCP_ECC_FUNCTION_MINV_384BIT) { 1836 /* Copy the second operand */ 1837 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.mm.operand_2, 0, 1838 ecc->u.mm.operand_2_len); 1839 if (ret) 1840 goto e_src; 1841 src.address += CCP_ECC_OPERAND_SIZE; 1842 } 1843 1844 /* Restore the workarea address */ 1845 src.address = save; 1846 1847 /* Prepare the output area for the operation */ 1848 ret = ccp_init_dm_workarea(&dst, cmd_q, CCP_ECC_DST_BUF_SIZE, 1849 DMA_FROM_DEVICE); 1850 if (ret) 1851 goto e_src; 1852 1853 op.soc = 1; 1854 op.src.u.dma.address = src.dma.address; 1855 op.src.u.dma.offset = 0; 1856 op.src.u.dma.length = src.length; 1857 op.dst.u.dma.address = dst.dma.address; 1858 op.dst.u.dma.offset = 0; 1859 op.dst.u.dma.length = dst.length; 1860 1861 op.u.ecc.function = cmd->u.ecc.function; 1862 1863 ret = cmd_q->ccp->vdata->perform->ecc(&op); 1864 if (ret) { 1865 cmd->engine_error = cmd_q->cmd_error; 1866 goto e_dst; 1867 } 1868 1869 ecc->ecc_result = le16_to_cpup( 1870 (const __le16 *)(dst.address + CCP_ECC_RESULT_OFFSET)); 1871 if (!(ecc->ecc_result & CCP_ECC_RESULT_SUCCESS)) { 1872 ret = -EIO; 1873 goto e_dst; 1874 } 1875 1876 /* Save the ECC result */ 1877 ccp_reverse_get_dm_area(&dst, 0, ecc->u.mm.result, 0, 1878 CCP_ECC_MODULUS_BYTES); 1879 1880 e_dst: 1881 ccp_dm_free(&dst); 1882 1883 e_src: 1884 ccp_dm_free(&src); 1885 1886 return ret; 1887 } 1888 1889 static int ccp_run_ecc_pm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) 1890 { 1891 struct ccp_ecc_engine *ecc = &cmd->u.ecc; 1892 struct ccp_dm_workarea src, dst; 1893 struct ccp_op op; 1894 int ret; 1895 u8 *save; 1896 1897 if (!ecc->u.pm.point_1.x || 1898 (ecc->u.pm.point_1.x_len > CCP_ECC_MODULUS_BYTES) || 1899 !ecc->u.pm.point_1.y || 1900 (ecc->u.pm.point_1.y_len > CCP_ECC_MODULUS_BYTES)) 1901 return -EINVAL; 1902 1903 if (ecc->function == CCP_ECC_FUNCTION_PADD_384BIT) { 1904 if (!ecc->u.pm.point_2.x || 1905 (ecc->u.pm.point_2.x_len > CCP_ECC_MODULUS_BYTES) || 1906 !ecc->u.pm.point_2.y || 1907 (ecc->u.pm.point_2.y_len > CCP_ECC_MODULUS_BYTES)) 1908 return -EINVAL; 1909 } else { 1910 if (!ecc->u.pm.domain_a || 1911 (ecc->u.pm.domain_a_len > CCP_ECC_MODULUS_BYTES)) 1912 return -EINVAL; 1913 1914 if (ecc->function == CCP_ECC_FUNCTION_PMUL_384BIT) 1915 if (!ecc->u.pm.scalar || 1916 (ecc->u.pm.scalar_len > CCP_ECC_MODULUS_BYTES)) 1917 return -EINVAL; 1918 } 1919 1920 if (!ecc->u.pm.result.x || 1921 (ecc->u.pm.result.x_len < CCP_ECC_MODULUS_BYTES) || 1922 !ecc->u.pm.result.y || 1923 (ecc->u.pm.result.y_len < CCP_ECC_MODULUS_BYTES)) 1924 return -EINVAL; 1925 1926 memset(&op, 0, sizeof(op)); 1927 op.cmd_q = cmd_q; 1928 op.jobid = CCP_NEW_JOBID(cmd_q->ccp); 1929 1930 /* Concatenate the modulus and the operands. Both the modulus and 1931 * the operands must be in little endian format. Since the input 1932 * is in big endian format it must be converted and placed in a 1933 * fixed length buffer. 1934 */ 1935 ret = ccp_init_dm_workarea(&src, cmd_q, CCP_ECC_SRC_BUF_SIZE, 1936 DMA_TO_DEVICE); 1937 if (ret) 1938 return ret; 1939 1940 /* Save the workarea address since it is updated in order to perform 1941 * the concatenation 1942 */ 1943 save = src.address; 1944 1945 /* Copy the ECC modulus */ 1946 ret = ccp_reverse_set_dm_area(&src, 0, ecc->mod, 0, ecc->mod_len); 1947 if (ret) 1948 goto e_src; 1949 src.address += CCP_ECC_OPERAND_SIZE; 1950 1951 /* Copy the first point X and Y coordinate */ 1952 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_1.x, 0, 1953 ecc->u.pm.point_1.x_len); 1954 if (ret) 1955 goto e_src; 1956 src.address += CCP_ECC_OPERAND_SIZE; 1957 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_1.y, 0, 1958 ecc->u.pm.point_1.y_len); 1959 if (ret) 1960 goto e_src; 1961 src.address += CCP_ECC_OPERAND_SIZE; 1962 1963 /* Set the first point Z coordinate to 1 */ 1964 *src.address = 0x01; 1965 src.address += CCP_ECC_OPERAND_SIZE; 1966 1967 if (ecc->function == CCP_ECC_FUNCTION_PADD_384BIT) { 1968 /* Copy the second point X and Y coordinate */ 1969 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_2.x, 0, 1970 ecc->u.pm.point_2.x_len); 1971 if (ret) 1972 goto e_src; 1973 src.address += CCP_ECC_OPERAND_SIZE; 1974 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_2.y, 0, 1975 ecc->u.pm.point_2.y_len); 1976 if (ret) 1977 goto e_src; 1978 src.address += CCP_ECC_OPERAND_SIZE; 1979 1980 /* Set the second point Z coordinate to 1 */ 1981 *src.address = 0x01; 1982 src.address += CCP_ECC_OPERAND_SIZE; 1983 } else { 1984 /* Copy the Domain "a" parameter */ 1985 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.domain_a, 0, 1986 ecc->u.pm.domain_a_len); 1987 if (ret) 1988 goto e_src; 1989 src.address += CCP_ECC_OPERAND_SIZE; 1990 1991 if (ecc->function == CCP_ECC_FUNCTION_PMUL_384BIT) { 1992 /* Copy the scalar value */ 1993 ret = ccp_reverse_set_dm_area(&src, 0, 1994 ecc->u.pm.scalar, 0, 1995 ecc->u.pm.scalar_len); 1996 if (ret) 1997 goto e_src; 1998 src.address += CCP_ECC_OPERAND_SIZE; 1999 } 2000 } 2001 2002 /* Restore the workarea address */ 2003 src.address = save; 2004 2005 /* Prepare the output area for the operation */ 2006 ret = ccp_init_dm_workarea(&dst, cmd_q, CCP_ECC_DST_BUF_SIZE, 2007 DMA_FROM_DEVICE); 2008 if (ret) 2009 goto e_src; 2010 2011 op.soc = 1; 2012 op.src.u.dma.address = src.dma.address; 2013 op.src.u.dma.offset = 0; 2014 op.src.u.dma.length = src.length; 2015 op.dst.u.dma.address = dst.dma.address; 2016 op.dst.u.dma.offset = 0; 2017 op.dst.u.dma.length = dst.length; 2018 2019 op.u.ecc.function = cmd->u.ecc.function; 2020 2021 ret = cmd_q->ccp->vdata->perform->ecc(&op); 2022 if (ret) { 2023 cmd->engine_error = cmd_q->cmd_error; 2024 goto e_dst; 2025 } 2026 2027 ecc->ecc_result = le16_to_cpup( 2028 (const __le16 *)(dst.address + CCP_ECC_RESULT_OFFSET)); 2029 if (!(ecc->ecc_result & CCP_ECC_RESULT_SUCCESS)) { 2030 ret = -EIO; 2031 goto e_dst; 2032 } 2033 2034 /* Save the workarea address since it is updated as we walk through 2035 * to copy the point math result 2036 */ 2037 save = dst.address; 2038 2039 /* Save the ECC result X and Y coordinates */ 2040 ccp_reverse_get_dm_area(&dst, 0, ecc->u.pm.result.x, 0, 2041 CCP_ECC_MODULUS_BYTES); 2042 dst.address += CCP_ECC_OUTPUT_SIZE; 2043 ccp_reverse_get_dm_area(&dst, 0, ecc->u.pm.result.y, 0, 2044 CCP_ECC_MODULUS_BYTES); 2045 dst.address += CCP_ECC_OUTPUT_SIZE; 2046 2047 /* Restore the workarea address */ 2048 dst.address = save; 2049 2050 e_dst: 2051 ccp_dm_free(&dst); 2052 2053 e_src: 2054 ccp_dm_free(&src); 2055 2056 return ret; 2057 } 2058 2059 static int ccp_run_ecc_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) 2060 { 2061 struct ccp_ecc_engine *ecc = &cmd->u.ecc; 2062 2063 ecc->ecc_result = 0; 2064 2065 if (!ecc->mod || 2066 (ecc->mod_len > CCP_ECC_MODULUS_BYTES)) 2067 return -EINVAL; 2068 2069 switch (ecc->function) { 2070 case CCP_ECC_FUNCTION_MMUL_384BIT: 2071 case CCP_ECC_FUNCTION_MADD_384BIT: 2072 case CCP_ECC_FUNCTION_MINV_384BIT: 2073 return ccp_run_ecc_mm_cmd(cmd_q, cmd); 2074 2075 case CCP_ECC_FUNCTION_PADD_384BIT: 2076 case CCP_ECC_FUNCTION_PMUL_384BIT: 2077 case CCP_ECC_FUNCTION_PDBL_384BIT: 2078 return ccp_run_ecc_pm_cmd(cmd_q, cmd); 2079 2080 default: 2081 return -EINVAL; 2082 } 2083 } 2084 2085 int ccp_run_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) 2086 { 2087 int ret; 2088 2089 cmd->engine_error = 0; 2090 cmd_q->cmd_error = 0; 2091 cmd_q->int_rcvd = 0; 2092 cmd_q->free_slots = cmd_q->ccp->vdata->perform->get_free_slots(cmd_q); 2093 2094 switch (cmd->engine) { 2095 case CCP_ENGINE_AES: 2096 ret = ccp_run_aes_cmd(cmd_q, cmd); 2097 break; 2098 case CCP_ENGINE_XTS_AES_128: 2099 ret = ccp_run_xts_aes_cmd(cmd_q, cmd); 2100 break; 2101 case CCP_ENGINE_DES3: 2102 ret = ccp_run_des3_cmd(cmd_q, cmd); 2103 break; 2104 case CCP_ENGINE_SHA: 2105 ret = ccp_run_sha_cmd(cmd_q, cmd); 2106 break; 2107 case CCP_ENGINE_RSA: 2108 ret = ccp_run_rsa_cmd(cmd_q, cmd); 2109 break; 2110 case CCP_ENGINE_PASSTHRU: 2111 if (cmd->flags & CCP_CMD_PASSTHRU_NO_DMA_MAP) 2112 ret = ccp_run_passthru_nomap_cmd(cmd_q, cmd); 2113 else 2114 ret = ccp_run_passthru_cmd(cmd_q, cmd); 2115 break; 2116 case CCP_ENGINE_ECC: 2117 ret = ccp_run_ecc_cmd(cmd_q, cmd); 2118 break; 2119 default: 2120 ret = -EINVAL; 2121 } 2122 2123 return ret; 2124 } 2125