1 /* 2 * AMD Cryptographic Coprocessor (CCP) driver 3 * 4 * Copyright (C) 2013,2016 Advanced Micro Devices, Inc. 5 * 6 * Author: Tom Lendacky <thomas.lendacky@amd.com> 7 * Author: Gary R Hook <gary.hook@amd.com> 8 * 9 * This program is free software; you can redistribute it and/or modify 10 * it under the terms of the GNU General Public License version 2 as 11 * published by the Free Software Foundation. 12 */ 13 14 #include <linux/module.h> 15 #include <linux/kernel.h> 16 #include <linux/pci.h> 17 #include <linux/interrupt.h> 18 #include <crypto/scatterwalk.h> 19 #include <linux/ccp.h> 20 21 #include "ccp-dev.h" 22 23 /* SHA initial context values */ 24 static const __be32 ccp_sha1_init[SHA1_DIGEST_SIZE / sizeof(__be32)] = { 25 cpu_to_be32(SHA1_H0), cpu_to_be32(SHA1_H1), 26 cpu_to_be32(SHA1_H2), cpu_to_be32(SHA1_H3), 27 cpu_to_be32(SHA1_H4), 28 }; 29 30 static const __be32 ccp_sha224_init[SHA256_DIGEST_SIZE / sizeof(__be32)] = { 31 cpu_to_be32(SHA224_H0), cpu_to_be32(SHA224_H1), 32 cpu_to_be32(SHA224_H2), cpu_to_be32(SHA224_H3), 33 cpu_to_be32(SHA224_H4), cpu_to_be32(SHA224_H5), 34 cpu_to_be32(SHA224_H6), cpu_to_be32(SHA224_H7), 35 }; 36 37 static const __be32 ccp_sha256_init[SHA256_DIGEST_SIZE / sizeof(__be32)] = { 38 cpu_to_be32(SHA256_H0), cpu_to_be32(SHA256_H1), 39 cpu_to_be32(SHA256_H2), cpu_to_be32(SHA256_H3), 40 cpu_to_be32(SHA256_H4), cpu_to_be32(SHA256_H5), 41 cpu_to_be32(SHA256_H6), cpu_to_be32(SHA256_H7), 42 }; 43 44 #define CCP_NEW_JOBID(ccp) ((ccp->vdata->version == CCP_VERSION(3, 0)) ? \ 45 ccp_gen_jobid(ccp) : 0) 46 47 static u32 ccp_gen_jobid(struct ccp_device *ccp) 48 { 49 return atomic_inc_return(&ccp->current_id) & CCP_JOBID_MASK; 50 } 51 52 static void ccp_sg_free(struct ccp_sg_workarea *wa) 53 { 54 if (wa->dma_count) 55 dma_unmap_sg(wa->dma_dev, wa->dma_sg, wa->nents, wa->dma_dir); 56 57 wa->dma_count = 0; 58 } 59 60 static int ccp_init_sg_workarea(struct ccp_sg_workarea *wa, struct device *dev, 61 struct scatterlist *sg, u64 len, 62 enum dma_data_direction dma_dir) 63 { 64 memset(wa, 0, sizeof(*wa)); 65 66 wa->sg = sg; 67 if (!sg) 68 return 0; 69 70 wa->nents = sg_nents_for_len(sg, len); 71 if (wa->nents < 0) 72 return wa->nents; 73 74 wa->bytes_left = len; 75 wa->sg_used = 0; 76 77 if (len == 0) 78 return 0; 79 80 if (dma_dir == DMA_NONE) 81 return 0; 82 83 wa->dma_sg = sg; 84 wa->dma_dev = dev; 85 wa->dma_dir = dma_dir; 86 wa->dma_count = dma_map_sg(dev, sg, wa->nents, dma_dir); 87 if (!wa->dma_count) 88 return -ENOMEM; 89 90 return 0; 91 } 92 93 static void ccp_update_sg_workarea(struct ccp_sg_workarea *wa, unsigned int len) 94 { 95 unsigned int nbytes = min_t(u64, len, wa->bytes_left); 96 97 if (!wa->sg) 98 return; 99 100 wa->sg_used += nbytes; 101 wa->bytes_left -= nbytes; 102 if (wa->sg_used == wa->sg->length) { 103 wa->sg = sg_next(wa->sg); 104 wa->sg_used = 0; 105 } 106 } 107 108 static void ccp_dm_free(struct ccp_dm_workarea *wa) 109 { 110 if (wa->length <= CCP_DMAPOOL_MAX_SIZE) { 111 if (wa->address) 112 dma_pool_free(wa->dma_pool, wa->address, 113 wa->dma.address); 114 } else { 115 if (wa->dma.address) 116 dma_unmap_single(wa->dev, wa->dma.address, wa->length, 117 wa->dma.dir); 118 kfree(wa->address); 119 } 120 121 wa->address = NULL; 122 wa->dma.address = 0; 123 } 124 125 static int ccp_init_dm_workarea(struct ccp_dm_workarea *wa, 126 struct ccp_cmd_queue *cmd_q, 127 unsigned int len, 128 enum dma_data_direction dir) 129 { 130 memset(wa, 0, sizeof(*wa)); 131 132 if (!len) 133 return 0; 134 135 wa->dev = cmd_q->ccp->dev; 136 wa->length = len; 137 138 if (len <= CCP_DMAPOOL_MAX_SIZE) { 139 wa->dma_pool = cmd_q->dma_pool; 140 141 wa->address = dma_pool_alloc(wa->dma_pool, GFP_KERNEL, 142 &wa->dma.address); 143 if (!wa->address) 144 return -ENOMEM; 145 146 wa->dma.length = CCP_DMAPOOL_MAX_SIZE; 147 148 memset(wa->address, 0, CCP_DMAPOOL_MAX_SIZE); 149 } else { 150 wa->address = kzalloc(len, GFP_KERNEL); 151 if (!wa->address) 152 return -ENOMEM; 153 154 wa->dma.address = dma_map_single(wa->dev, wa->address, len, 155 dir); 156 if (!wa->dma.address) 157 return -ENOMEM; 158 159 wa->dma.length = len; 160 } 161 wa->dma.dir = dir; 162 163 return 0; 164 } 165 166 static void ccp_set_dm_area(struct ccp_dm_workarea *wa, unsigned int wa_offset, 167 struct scatterlist *sg, unsigned int sg_offset, 168 unsigned int len) 169 { 170 WARN_ON(!wa->address); 171 172 scatterwalk_map_and_copy(wa->address + wa_offset, sg, sg_offset, len, 173 0); 174 } 175 176 static void ccp_get_dm_area(struct ccp_dm_workarea *wa, unsigned int wa_offset, 177 struct scatterlist *sg, unsigned int sg_offset, 178 unsigned int len) 179 { 180 WARN_ON(!wa->address); 181 182 scatterwalk_map_and_copy(wa->address + wa_offset, sg, sg_offset, len, 183 1); 184 } 185 186 static int ccp_reverse_set_dm_area(struct ccp_dm_workarea *wa, 187 unsigned int wa_offset, 188 struct scatterlist *sg, 189 unsigned int sg_offset, 190 unsigned int len) 191 { 192 u8 *p, *q; 193 194 ccp_set_dm_area(wa, wa_offset, sg, sg_offset, len); 195 196 p = wa->address + wa_offset; 197 q = p + len - 1; 198 while (p < q) { 199 *p = *p ^ *q; 200 *q = *p ^ *q; 201 *p = *p ^ *q; 202 p++; 203 q--; 204 } 205 return 0; 206 } 207 208 static void ccp_reverse_get_dm_area(struct ccp_dm_workarea *wa, 209 unsigned int wa_offset, 210 struct scatterlist *sg, 211 unsigned int sg_offset, 212 unsigned int len) 213 { 214 u8 *p, *q; 215 216 p = wa->address + wa_offset; 217 q = p + len - 1; 218 while (p < q) { 219 *p = *p ^ *q; 220 *q = *p ^ *q; 221 *p = *p ^ *q; 222 p++; 223 q--; 224 } 225 226 ccp_get_dm_area(wa, wa_offset, sg, sg_offset, len); 227 } 228 229 static void ccp_free_data(struct ccp_data *data, struct ccp_cmd_queue *cmd_q) 230 { 231 ccp_dm_free(&data->dm_wa); 232 ccp_sg_free(&data->sg_wa); 233 } 234 235 static int ccp_init_data(struct ccp_data *data, struct ccp_cmd_queue *cmd_q, 236 struct scatterlist *sg, u64 sg_len, 237 unsigned int dm_len, 238 enum dma_data_direction dir) 239 { 240 int ret; 241 242 memset(data, 0, sizeof(*data)); 243 244 ret = ccp_init_sg_workarea(&data->sg_wa, cmd_q->ccp->dev, sg, sg_len, 245 dir); 246 if (ret) 247 goto e_err; 248 249 ret = ccp_init_dm_workarea(&data->dm_wa, cmd_q, dm_len, dir); 250 if (ret) 251 goto e_err; 252 253 return 0; 254 255 e_err: 256 ccp_free_data(data, cmd_q); 257 258 return ret; 259 } 260 261 static unsigned int ccp_queue_buf(struct ccp_data *data, unsigned int from) 262 { 263 struct ccp_sg_workarea *sg_wa = &data->sg_wa; 264 struct ccp_dm_workarea *dm_wa = &data->dm_wa; 265 unsigned int buf_count, nbytes; 266 267 /* Clear the buffer if setting it */ 268 if (!from) 269 memset(dm_wa->address, 0, dm_wa->length); 270 271 if (!sg_wa->sg) 272 return 0; 273 274 /* Perform the copy operation 275 * nbytes will always be <= UINT_MAX because dm_wa->length is 276 * an unsigned int 277 */ 278 nbytes = min_t(u64, sg_wa->bytes_left, dm_wa->length); 279 scatterwalk_map_and_copy(dm_wa->address, sg_wa->sg, sg_wa->sg_used, 280 nbytes, from); 281 282 /* Update the structures and generate the count */ 283 buf_count = 0; 284 while (sg_wa->bytes_left && (buf_count < dm_wa->length)) { 285 nbytes = min(sg_wa->sg->length - sg_wa->sg_used, 286 dm_wa->length - buf_count); 287 nbytes = min_t(u64, sg_wa->bytes_left, nbytes); 288 289 buf_count += nbytes; 290 ccp_update_sg_workarea(sg_wa, nbytes); 291 } 292 293 return buf_count; 294 } 295 296 static unsigned int ccp_fill_queue_buf(struct ccp_data *data) 297 { 298 return ccp_queue_buf(data, 0); 299 } 300 301 static unsigned int ccp_empty_queue_buf(struct ccp_data *data) 302 { 303 return ccp_queue_buf(data, 1); 304 } 305 306 static void ccp_prepare_data(struct ccp_data *src, struct ccp_data *dst, 307 struct ccp_op *op, unsigned int block_size, 308 bool blocksize_op) 309 { 310 unsigned int sg_src_len, sg_dst_len, op_len; 311 312 /* The CCP can only DMA from/to one address each per operation. This 313 * requires that we find the smallest DMA area between the source 314 * and destination. The resulting len values will always be <= UINT_MAX 315 * because the dma length is an unsigned int. 316 */ 317 sg_src_len = sg_dma_len(src->sg_wa.sg) - src->sg_wa.sg_used; 318 sg_src_len = min_t(u64, src->sg_wa.bytes_left, sg_src_len); 319 320 if (dst) { 321 sg_dst_len = sg_dma_len(dst->sg_wa.sg) - dst->sg_wa.sg_used; 322 sg_dst_len = min_t(u64, src->sg_wa.bytes_left, sg_dst_len); 323 op_len = min(sg_src_len, sg_dst_len); 324 } else { 325 op_len = sg_src_len; 326 } 327 328 /* The data operation length will be at least block_size in length 329 * or the smaller of available sg room remaining for the source or 330 * the destination 331 */ 332 op_len = max(op_len, block_size); 333 334 /* Unless we have to buffer data, there's no reason to wait */ 335 op->soc = 0; 336 337 if (sg_src_len < block_size) { 338 /* Not enough data in the sg element, so it 339 * needs to be buffered into a blocksize chunk 340 */ 341 int cp_len = ccp_fill_queue_buf(src); 342 343 op->soc = 1; 344 op->src.u.dma.address = src->dm_wa.dma.address; 345 op->src.u.dma.offset = 0; 346 op->src.u.dma.length = (blocksize_op) ? block_size : cp_len; 347 } else { 348 /* Enough data in the sg element, but we need to 349 * adjust for any previously copied data 350 */ 351 op->src.u.dma.address = sg_dma_address(src->sg_wa.sg); 352 op->src.u.dma.offset = src->sg_wa.sg_used; 353 op->src.u.dma.length = op_len & ~(block_size - 1); 354 355 ccp_update_sg_workarea(&src->sg_wa, op->src.u.dma.length); 356 } 357 358 if (dst) { 359 if (sg_dst_len < block_size) { 360 /* Not enough room in the sg element or we're on the 361 * last piece of data (when using padding), so the 362 * output needs to be buffered into a blocksize chunk 363 */ 364 op->soc = 1; 365 op->dst.u.dma.address = dst->dm_wa.dma.address; 366 op->dst.u.dma.offset = 0; 367 op->dst.u.dma.length = op->src.u.dma.length; 368 } else { 369 /* Enough room in the sg element, but we need to 370 * adjust for any previously used area 371 */ 372 op->dst.u.dma.address = sg_dma_address(dst->sg_wa.sg); 373 op->dst.u.dma.offset = dst->sg_wa.sg_used; 374 op->dst.u.dma.length = op->src.u.dma.length; 375 } 376 } 377 } 378 379 static void ccp_process_data(struct ccp_data *src, struct ccp_data *dst, 380 struct ccp_op *op) 381 { 382 op->init = 0; 383 384 if (dst) { 385 if (op->dst.u.dma.address == dst->dm_wa.dma.address) 386 ccp_empty_queue_buf(dst); 387 else 388 ccp_update_sg_workarea(&dst->sg_wa, 389 op->dst.u.dma.length); 390 } 391 } 392 393 static int ccp_copy_to_from_sb(struct ccp_cmd_queue *cmd_q, 394 struct ccp_dm_workarea *wa, u32 jobid, u32 sb, 395 u32 byte_swap, bool from) 396 { 397 struct ccp_op op; 398 399 memset(&op, 0, sizeof(op)); 400 401 op.cmd_q = cmd_q; 402 op.jobid = jobid; 403 op.eom = 1; 404 405 if (from) { 406 op.soc = 1; 407 op.src.type = CCP_MEMTYPE_SB; 408 op.src.u.sb = sb; 409 op.dst.type = CCP_MEMTYPE_SYSTEM; 410 op.dst.u.dma.address = wa->dma.address; 411 op.dst.u.dma.length = wa->length; 412 } else { 413 op.src.type = CCP_MEMTYPE_SYSTEM; 414 op.src.u.dma.address = wa->dma.address; 415 op.src.u.dma.length = wa->length; 416 op.dst.type = CCP_MEMTYPE_SB; 417 op.dst.u.sb = sb; 418 } 419 420 op.u.passthru.byte_swap = byte_swap; 421 422 return cmd_q->ccp->vdata->perform->passthru(&op); 423 } 424 425 static int ccp_copy_to_sb(struct ccp_cmd_queue *cmd_q, 426 struct ccp_dm_workarea *wa, u32 jobid, u32 sb, 427 u32 byte_swap) 428 { 429 return ccp_copy_to_from_sb(cmd_q, wa, jobid, sb, byte_swap, false); 430 } 431 432 static int ccp_copy_from_sb(struct ccp_cmd_queue *cmd_q, 433 struct ccp_dm_workarea *wa, u32 jobid, u32 sb, 434 u32 byte_swap) 435 { 436 return ccp_copy_to_from_sb(cmd_q, wa, jobid, sb, byte_swap, true); 437 } 438 439 static int ccp_run_aes_cmac_cmd(struct ccp_cmd_queue *cmd_q, 440 struct ccp_cmd *cmd) 441 { 442 struct ccp_aes_engine *aes = &cmd->u.aes; 443 struct ccp_dm_workarea key, ctx; 444 struct ccp_data src; 445 struct ccp_op op; 446 unsigned int dm_offset; 447 int ret; 448 449 if (!((aes->key_len == AES_KEYSIZE_128) || 450 (aes->key_len == AES_KEYSIZE_192) || 451 (aes->key_len == AES_KEYSIZE_256))) 452 return -EINVAL; 453 454 if (aes->src_len & (AES_BLOCK_SIZE - 1)) 455 return -EINVAL; 456 457 if (aes->iv_len != AES_BLOCK_SIZE) 458 return -EINVAL; 459 460 if (!aes->key || !aes->iv || !aes->src) 461 return -EINVAL; 462 463 if (aes->cmac_final) { 464 if (aes->cmac_key_len != AES_BLOCK_SIZE) 465 return -EINVAL; 466 467 if (!aes->cmac_key) 468 return -EINVAL; 469 } 470 471 BUILD_BUG_ON(CCP_AES_KEY_SB_COUNT != 1); 472 BUILD_BUG_ON(CCP_AES_CTX_SB_COUNT != 1); 473 474 ret = -EIO; 475 memset(&op, 0, sizeof(op)); 476 op.cmd_q = cmd_q; 477 op.jobid = CCP_NEW_JOBID(cmd_q->ccp); 478 op.sb_key = cmd_q->sb_key; 479 op.sb_ctx = cmd_q->sb_ctx; 480 op.init = 1; 481 op.u.aes.type = aes->type; 482 op.u.aes.mode = aes->mode; 483 op.u.aes.action = aes->action; 484 485 /* All supported key sizes fit in a single (32-byte) SB entry 486 * and must be in little endian format. Use the 256-bit byte 487 * swap passthru option to convert from big endian to little 488 * endian. 489 */ 490 ret = ccp_init_dm_workarea(&key, cmd_q, 491 CCP_AES_KEY_SB_COUNT * CCP_SB_BYTES, 492 DMA_TO_DEVICE); 493 if (ret) 494 return ret; 495 496 dm_offset = CCP_SB_BYTES - aes->key_len; 497 ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len); 498 ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key, 499 CCP_PASSTHRU_BYTESWAP_256BIT); 500 if (ret) { 501 cmd->engine_error = cmd_q->cmd_error; 502 goto e_key; 503 } 504 505 /* The AES context fits in a single (32-byte) SB entry and 506 * must be in little endian format. Use the 256-bit byte swap 507 * passthru option to convert from big endian to little endian. 508 */ 509 ret = ccp_init_dm_workarea(&ctx, cmd_q, 510 CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES, 511 DMA_BIDIRECTIONAL); 512 if (ret) 513 goto e_key; 514 515 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE; 516 ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len); 517 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, 518 CCP_PASSTHRU_BYTESWAP_256BIT); 519 if (ret) { 520 cmd->engine_error = cmd_q->cmd_error; 521 goto e_ctx; 522 } 523 524 /* Send data to the CCP AES engine */ 525 ret = ccp_init_data(&src, cmd_q, aes->src, aes->src_len, 526 AES_BLOCK_SIZE, DMA_TO_DEVICE); 527 if (ret) 528 goto e_ctx; 529 530 while (src.sg_wa.bytes_left) { 531 ccp_prepare_data(&src, NULL, &op, AES_BLOCK_SIZE, true); 532 if (aes->cmac_final && !src.sg_wa.bytes_left) { 533 op.eom = 1; 534 535 /* Push the K1/K2 key to the CCP now */ 536 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, 537 op.sb_ctx, 538 CCP_PASSTHRU_BYTESWAP_256BIT); 539 if (ret) { 540 cmd->engine_error = cmd_q->cmd_error; 541 goto e_src; 542 } 543 544 ccp_set_dm_area(&ctx, 0, aes->cmac_key, 0, 545 aes->cmac_key_len); 546 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, 547 CCP_PASSTHRU_BYTESWAP_256BIT); 548 if (ret) { 549 cmd->engine_error = cmd_q->cmd_error; 550 goto e_src; 551 } 552 } 553 554 ret = cmd_q->ccp->vdata->perform->aes(&op); 555 if (ret) { 556 cmd->engine_error = cmd_q->cmd_error; 557 goto e_src; 558 } 559 560 ccp_process_data(&src, NULL, &op); 561 } 562 563 /* Retrieve the AES context - convert from LE to BE using 564 * 32-byte (256-bit) byteswapping 565 */ 566 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, 567 CCP_PASSTHRU_BYTESWAP_256BIT); 568 if (ret) { 569 cmd->engine_error = cmd_q->cmd_error; 570 goto e_src; 571 } 572 573 /* ...but we only need AES_BLOCK_SIZE bytes */ 574 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE; 575 ccp_get_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len); 576 577 e_src: 578 ccp_free_data(&src, cmd_q); 579 580 e_ctx: 581 ccp_dm_free(&ctx); 582 583 e_key: 584 ccp_dm_free(&key); 585 586 return ret; 587 } 588 589 static int ccp_run_aes_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) 590 { 591 struct ccp_aes_engine *aes = &cmd->u.aes; 592 struct ccp_dm_workarea key, ctx; 593 struct ccp_data src, dst; 594 struct ccp_op op; 595 unsigned int dm_offset; 596 bool in_place = false; 597 int ret; 598 599 if (aes->mode == CCP_AES_MODE_CMAC) 600 return ccp_run_aes_cmac_cmd(cmd_q, cmd); 601 602 if (!((aes->key_len == AES_KEYSIZE_128) || 603 (aes->key_len == AES_KEYSIZE_192) || 604 (aes->key_len == AES_KEYSIZE_256))) 605 return -EINVAL; 606 607 if (((aes->mode == CCP_AES_MODE_ECB) || 608 (aes->mode == CCP_AES_MODE_CBC) || 609 (aes->mode == CCP_AES_MODE_CFB)) && 610 (aes->src_len & (AES_BLOCK_SIZE - 1))) 611 return -EINVAL; 612 613 if (!aes->key || !aes->src || !aes->dst) 614 return -EINVAL; 615 616 if (aes->mode != CCP_AES_MODE_ECB) { 617 if (aes->iv_len != AES_BLOCK_SIZE) 618 return -EINVAL; 619 620 if (!aes->iv) 621 return -EINVAL; 622 } 623 624 BUILD_BUG_ON(CCP_AES_KEY_SB_COUNT != 1); 625 BUILD_BUG_ON(CCP_AES_CTX_SB_COUNT != 1); 626 627 ret = -EIO; 628 memset(&op, 0, sizeof(op)); 629 op.cmd_q = cmd_q; 630 op.jobid = CCP_NEW_JOBID(cmd_q->ccp); 631 op.sb_key = cmd_q->sb_key; 632 op.sb_ctx = cmd_q->sb_ctx; 633 op.init = (aes->mode == CCP_AES_MODE_ECB) ? 0 : 1; 634 op.u.aes.type = aes->type; 635 op.u.aes.mode = aes->mode; 636 op.u.aes.action = aes->action; 637 638 /* All supported key sizes fit in a single (32-byte) SB entry 639 * and must be in little endian format. Use the 256-bit byte 640 * swap passthru option to convert from big endian to little 641 * endian. 642 */ 643 ret = ccp_init_dm_workarea(&key, cmd_q, 644 CCP_AES_KEY_SB_COUNT * CCP_SB_BYTES, 645 DMA_TO_DEVICE); 646 if (ret) 647 return ret; 648 649 dm_offset = CCP_SB_BYTES - aes->key_len; 650 ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len); 651 ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key, 652 CCP_PASSTHRU_BYTESWAP_256BIT); 653 if (ret) { 654 cmd->engine_error = cmd_q->cmd_error; 655 goto e_key; 656 } 657 658 /* The AES context fits in a single (32-byte) SB entry and 659 * must be in little endian format. Use the 256-bit byte swap 660 * passthru option to convert from big endian to little endian. 661 */ 662 ret = ccp_init_dm_workarea(&ctx, cmd_q, 663 CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES, 664 DMA_BIDIRECTIONAL); 665 if (ret) 666 goto e_key; 667 668 if (aes->mode != CCP_AES_MODE_ECB) { 669 /* Load the AES context - convert to LE */ 670 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE; 671 ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len); 672 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, 673 CCP_PASSTHRU_BYTESWAP_256BIT); 674 if (ret) { 675 cmd->engine_error = cmd_q->cmd_error; 676 goto e_ctx; 677 } 678 } 679 switch (aes->mode) { 680 case CCP_AES_MODE_CFB: /* CFB128 only */ 681 case CCP_AES_MODE_CTR: 682 op.u.aes.size = AES_BLOCK_SIZE * BITS_PER_BYTE - 1; 683 break; 684 default: 685 op.u.aes.size = 0; 686 } 687 688 /* Prepare the input and output data workareas. For in-place 689 * operations we need to set the dma direction to BIDIRECTIONAL 690 * and copy the src workarea to the dst workarea. 691 */ 692 if (sg_virt(aes->src) == sg_virt(aes->dst)) 693 in_place = true; 694 695 ret = ccp_init_data(&src, cmd_q, aes->src, aes->src_len, 696 AES_BLOCK_SIZE, 697 in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE); 698 if (ret) 699 goto e_ctx; 700 701 if (in_place) { 702 dst = src; 703 } else { 704 ret = ccp_init_data(&dst, cmd_q, aes->dst, aes->src_len, 705 AES_BLOCK_SIZE, DMA_FROM_DEVICE); 706 if (ret) 707 goto e_src; 708 } 709 710 /* Send data to the CCP AES engine */ 711 while (src.sg_wa.bytes_left) { 712 ccp_prepare_data(&src, &dst, &op, AES_BLOCK_SIZE, true); 713 if (!src.sg_wa.bytes_left) { 714 op.eom = 1; 715 716 /* Since we don't retrieve the AES context in ECB 717 * mode we have to wait for the operation to complete 718 * on the last piece of data 719 */ 720 if (aes->mode == CCP_AES_MODE_ECB) 721 op.soc = 1; 722 } 723 724 ret = cmd_q->ccp->vdata->perform->aes(&op); 725 if (ret) { 726 cmd->engine_error = cmd_q->cmd_error; 727 goto e_dst; 728 } 729 730 ccp_process_data(&src, &dst, &op); 731 } 732 733 if (aes->mode != CCP_AES_MODE_ECB) { 734 /* Retrieve the AES context - convert from LE to BE using 735 * 32-byte (256-bit) byteswapping 736 */ 737 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, 738 CCP_PASSTHRU_BYTESWAP_256BIT); 739 if (ret) { 740 cmd->engine_error = cmd_q->cmd_error; 741 goto e_dst; 742 } 743 744 /* ...but we only need AES_BLOCK_SIZE bytes */ 745 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE; 746 ccp_get_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len); 747 } 748 749 e_dst: 750 if (!in_place) 751 ccp_free_data(&dst, cmd_q); 752 753 e_src: 754 ccp_free_data(&src, cmd_q); 755 756 e_ctx: 757 ccp_dm_free(&ctx); 758 759 e_key: 760 ccp_dm_free(&key); 761 762 return ret; 763 } 764 765 static int ccp_run_xts_aes_cmd(struct ccp_cmd_queue *cmd_q, 766 struct ccp_cmd *cmd) 767 { 768 struct ccp_xts_aes_engine *xts = &cmd->u.xts; 769 struct ccp_dm_workarea key, ctx; 770 struct ccp_data src, dst; 771 struct ccp_op op; 772 unsigned int unit_size, dm_offset; 773 bool in_place = false; 774 int ret; 775 776 switch (xts->unit_size) { 777 case CCP_XTS_AES_UNIT_SIZE_16: 778 unit_size = 16; 779 break; 780 case CCP_XTS_AES_UNIT_SIZE_512: 781 unit_size = 512; 782 break; 783 case CCP_XTS_AES_UNIT_SIZE_1024: 784 unit_size = 1024; 785 break; 786 case CCP_XTS_AES_UNIT_SIZE_2048: 787 unit_size = 2048; 788 break; 789 case CCP_XTS_AES_UNIT_SIZE_4096: 790 unit_size = 4096; 791 break; 792 793 default: 794 return -EINVAL; 795 } 796 797 if (xts->key_len != AES_KEYSIZE_128) 798 return -EINVAL; 799 800 if (!xts->final && (xts->src_len & (AES_BLOCK_SIZE - 1))) 801 return -EINVAL; 802 803 if (xts->iv_len != AES_BLOCK_SIZE) 804 return -EINVAL; 805 806 if (!xts->key || !xts->iv || !xts->src || !xts->dst) 807 return -EINVAL; 808 809 BUILD_BUG_ON(CCP_XTS_AES_KEY_SB_COUNT != 1); 810 BUILD_BUG_ON(CCP_XTS_AES_CTX_SB_COUNT != 1); 811 812 ret = -EIO; 813 memset(&op, 0, sizeof(op)); 814 op.cmd_q = cmd_q; 815 op.jobid = CCP_NEW_JOBID(cmd_q->ccp); 816 op.sb_key = cmd_q->sb_key; 817 op.sb_ctx = cmd_q->sb_ctx; 818 op.init = 1; 819 op.u.xts.action = xts->action; 820 op.u.xts.unit_size = xts->unit_size; 821 822 /* All supported key sizes fit in a single (32-byte) SB entry 823 * and must be in little endian format. Use the 256-bit byte 824 * swap passthru option to convert from big endian to little 825 * endian. 826 */ 827 ret = ccp_init_dm_workarea(&key, cmd_q, 828 CCP_XTS_AES_KEY_SB_COUNT * CCP_SB_BYTES, 829 DMA_TO_DEVICE); 830 if (ret) 831 return ret; 832 833 dm_offset = CCP_SB_BYTES - AES_KEYSIZE_128; 834 ccp_set_dm_area(&key, dm_offset, xts->key, 0, xts->key_len); 835 ccp_set_dm_area(&key, 0, xts->key, dm_offset, xts->key_len); 836 ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key, 837 CCP_PASSTHRU_BYTESWAP_256BIT); 838 if (ret) { 839 cmd->engine_error = cmd_q->cmd_error; 840 goto e_key; 841 } 842 843 /* The AES context fits in a single (32-byte) SB entry and 844 * for XTS is already in little endian format so no byte swapping 845 * is needed. 846 */ 847 ret = ccp_init_dm_workarea(&ctx, cmd_q, 848 CCP_XTS_AES_CTX_SB_COUNT * CCP_SB_BYTES, 849 DMA_BIDIRECTIONAL); 850 if (ret) 851 goto e_key; 852 853 ccp_set_dm_area(&ctx, 0, xts->iv, 0, xts->iv_len); 854 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, 855 CCP_PASSTHRU_BYTESWAP_NOOP); 856 if (ret) { 857 cmd->engine_error = cmd_q->cmd_error; 858 goto e_ctx; 859 } 860 861 /* Prepare the input and output data workareas. For in-place 862 * operations we need to set the dma direction to BIDIRECTIONAL 863 * and copy the src workarea to the dst workarea. 864 */ 865 if (sg_virt(xts->src) == sg_virt(xts->dst)) 866 in_place = true; 867 868 ret = ccp_init_data(&src, cmd_q, xts->src, xts->src_len, 869 unit_size, 870 in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE); 871 if (ret) 872 goto e_ctx; 873 874 if (in_place) { 875 dst = src; 876 } else { 877 ret = ccp_init_data(&dst, cmd_q, xts->dst, xts->src_len, 878 unit_size, DMA_FROM_DEVICE); 879 if (ret) 880 goto e_src; 881 } 882 883 /* Send data to the CCP AES engine */ 884 while (src.sg_wa.bytes_left) { 885 ccp_prepare_data(&src, &dst, &op, unit_size, true); 886 if (!src.sg_wa.bytes_left) 887 op.eom = 1; 888 889 ret = cmd_q->ccp->vdata->perform->xts_aes(&op); 890 if (ret) { 891 cmd->engine_error = cmd_q->cmd_error; 892 goto e_dst; 893 } 894 895 ccp_process_data(&src, &dst, &op); 896 } 897 898 /* Retrieve the AES context - convert from LE to BE using 899 * 32-byte (256-bit) byteswapping 900 */ 901 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, 902 CCP_PASSTHRU_BYTESWAP_256BIT); 903 if (ret) { 904 cmd->engine_error = cmd_q->cmd_error; 905 goto e_dst; 906 } 907 908 /* ...but we only need AES_BLOCK_SIZE bytes */ 909 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE; 910 ccp_get_dm_area(&ctx, dm_offset, xts->iv, 0, xts->iv_len); 911 912 e_dst: 913 if (!in_place) 914 ccp_free_data(&dst, cmd_q); 915 916 e_src: 917 ccp_free_data(&src, cmd_q); 918 919 e_ctx: 920 ccp_dm_free(&ctx); 921 922 e_key: 923 ccp_dm_free(&key); 924 925 return ret; 926 } 927 928 static int ccp_run_sha_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) 929 { 930 struct ccp_sha_engine *sha = &cmd->u.sha; 931 struct ccp_dm_workarea ctx; 932 struct ccp_data src; 933 struct ccp_op op; 934 unsigned int ioffset, ooffset; 935 unsigned int digest_size; 936 int sb_count; 937 const void *init; 938 u64 block_size; 939 int ctx_size; 940 int ret; 941 942 switch (sha->type) { 943 case CCP_SHA_TYPE_1: 944 if (sha->ctx_len < SHA1_DIGEST_SIZE) 945 return -EINVAL; 946 block_size = SHA1_BLOCK_SIZE; 947 break; 948 case CCP_SHA_TYPE_224: 949 if (sha->ctx_len < SHA224_DIGEST_SIZE) 950 return -EINVAL; 951 block_size = SHA224_BLOCK_SIZE; 952 break; 953 case CCP_SHA_TYPE_256: 954 if (sha->ctx_len < SHA256_DIGEST_SIZE) 955 return -EINVAL; 956 block_size = SHA256_BLOCK_SIZE; 957 break; 958 default: 959 return -EINVAL; 960 } 961 962 if (!sha->ctx) 963 return -EINVAL; 964 965 if (!sha->final && (sha->src_len & (block_size - 1))) 966 return -EINVAL; 967 968 /* The version 3 device can't handle zero-length input */ 969 if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0)) { 970 971 if (!sha->src_len) { 972 unsigned int digest_len; 973 const u8 *sha_zero; 974 975 /* Not final, just return */ 976 if (!sha->final) 977 return 0; 978 979 /* CCP can't do a zero length sha operation so the 980 * caller must buffer the data. 981 */ 982 if (sha->msg_bits) 983 return -EINVAL; 984 985 /* The CCP cannot perform zero-length sha operations 986 * so the caller is required to buffer data for the 987 * final operation. However, a sha operation for a 988 * message with a total length of zero is valid so 989 * known values are required to supply the result. 990 */ 991 switch (sha->type) { 992 case CCP_SHA_TYPE_1: 993 sha_zero = sha1_zero_message_hash; 994 digest_len = SHA1_DIGEST_SIZE; 995 break; 996 case CCP_SHA_TYPE_224: 997 sha_zero = sha224_zero_message_hash; 998 digest_len = SHA224_DIGEST_SIZE; 999 break; 1000 case CCP_SHA_TYPE_256: 1001 sha_zero = sha256_zero_message_hash; 1002 digest_len = SHA256_DIGEST_SIZE; 1003 break; 1004 default: 1005 return -EINVAL; 1006 } 1007 1008 scatterwalk_map_and_copy((void *)sha_zero, sha->ctx, 0, 1009 digest_len, 1); 1010 1011 return 0; 1012 } 1013 } 1014 1015 /* Set variables used throughout */ 1016 switch (sha->type) { 1017 case CCP_SHA_TYPE_1: 1018 digest_size = SHA1_DIGEST_SIZE; 1019 init = (void *) ccp_sha1_init; 1020 ctx_size = SHA1_DIGEST_SIZE; 1021 sb_count = 1; 1022 if (cmd_q->ccp->vdata->version != CCP_VERSION(3, 0)) 1023 ooffset = ioffset = CCP_SB_BYTES - SHA1_DIGEST_SIZE; 1024 else 1025 ooffset = ioffset = 0; 1026 break; 1027 case CCP_SHA_TYPE_224: 1028 digest_size = SHA224_DIGEST_SIZE; 1029 init = (void *) ccp_sha224_init; 1030 ctx_size = SHA256_DIGEST_SIZE; 1031 sb_count = 1; 1032 ioffset = 0; 1033 if (cmd_q->ccp->vdata->version != CCP_VERSION(3, 0)) 1034 ooffset = CCP_SB_BYTES - SHA224_DIGEST_SIZE; 1035 else 1036 ooffset = 0; 1037 break; 1038 case CCP_SHA_TYPE_256: 1039 digest_size = SHA256_DIGEST_SIZE; 1040 init = (void *) ccp_sha256_init; 1041 ctx_size = SHA256_DIGEST_SIZE; 1042 sb_count = 1; 1043 ooffset = ioffset = 0; 1044 break; 1045 default: 1046 ret = -EINVAL; 1047 goto e_data; 1048 } 1049 1050 /* For zero-length plaintext the src pointer is ignored; 1051 * otherwise both parts must be valid 1052 */ 1053 if (sha->src_len && !sha->src) 1054 return -EINVAL; 1055 1056 memset(&op, 0, sizeof(op)); 1057 op.cmd_q = cmd_q; 1058 op.jobid = CCP_NEW_JOBID(cmd_q->ccp); 1059 op.sb_ctx = cmd_q->sb_ctx; /* Pre-allocated */ 1060 op.u.sha.type = sha->type; 1061 op.u.sha.msg_bits = sha->msg_bits; 1062 1063 ret = ccp_init_dm_workarea(&ctx, cmd_q, sb_count * CCP_SB_BYTES, 1064 DMA_BIDIRECTIONAL); 1065 if (ret) 1066 return ret; 1067 if (sha->first) { 1068 switch (sha->type) { 1069 case CCP_SHA_TYPE_1: 1070 case CCP_SHA_TYPE_224: 1071 case CCP_SHA_TYPE_256: 1072 memcpy(ctx.address + ioffset, init, ctx_size); 1073 break; 1074 default: 1075 ret = -EINVAL; 1076 goto e_ctx; 1077 } 1078 } else { 1079 /* Restore the context */ 1080 ccp_set_dm_area(&ctx, 0, sha->ctx, 0, 1081 sb_count * CCP_SB_BYTES); 1082 } 1083 1084 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, 1085 CCP_PASSTHRU_BYTESWAP_256BIT); 1086 if (ret) { 1087 cmd->engine_error = cmd_q->cmd_error; 1088 goto e_ctx; 1089 } 1090 1091 if (sha->src) { 1092 /* Send data to the CCP SHA engine; block_size is set above */ 1093 ret = ccp_init_data(&src, cmd_q, sha->src, sha->src_len, 1094 block_size, DMA_TO_DEVICE); 1095 if (ret) 1096 goto e_ctx; 1097 1098 while (src.sg_wa.bytes_left) { 1099 ccp_prepare_data(&src, NULL, &op, block_size, false); 1100 if (sha->final && !src.sg_wa.bytes_left) 1101 op.eom = 1; 1102 1103 ret = cmd_q->ccp->vdata->perform->sha(&op); 1104 if (ret) { 1105 cmd->engine_error = cmd_q->cmd_error; 1106 goto e_data; 1107 } 1108 1109 ccp_process_data(&src, NULL, &op); 1110 } 1111 } else { 1112 op.eom = 1; 1113 ret = cmd_q->ccp->vdata->perform->sha(&op); 1114 if (ret) { 1115 cmd->engine_error = cmd_q->cmd_error; 1116 goto e_data; 1117 } 1118 } 1119 1120 /* Retrieve the SHA context - convert from LE to BE using 1121 * 32-byte (256-bit) byteswapping to BE 1122 */ 1123 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, 1124 CCP_PASSTHRU_BYTESWAP_256BIT); 1125 if (ret) { 1126 cmd->engine_error = cmd_q->cmd_error; 1127 goto e_data; 1128 } 1129 1130 if (sha->final) { 1131 /* Finishing up, so get the digest */ 1132 switch (sha->type) { 1133 case CCP_SHA_TYPE_1: 1134 case CCP_SHA_TYPE_224: 1135 case CCP_SHA_TYPE_256: 1136 ccp_get_dm_area(&ctx, ooffset, 1137 sha->ctx, 0, 1138 digest_size); 1139 break; 1140 default: 1141 ret = -EINVAL; 1142 goto e_ctx; 1143 } 1144 } else { 1145 /* Stash the context */ 1146 ccp_get_dm_area(&ctx, 0, sha->ctx, 0, 1147 sb_count * CCP_SB_BYTES); 1148 } 1149 1150 if (sha->final && sha->opad) { 1151 /* HMAC operation, recursively perform final SHA */ 1152 struct ccp_cmd hmac_cmd; 1153 struct scatterlist sg; 1154 u8 *hmac_buf; 1155 1156 if (sha->opad_len != block_size) { 1157 ret = -EINVAL; 1158 goto e_data; 1159 } 1160 1161 hmac_buf = kmalloc(block_size + digest_size, GFP_KERNEL); 1162 if (!hmac_buf) { 1163 ret = -ENOMEM; 1164 goto e_data; 1165 } 1166 sg_init_one(&sg, hmac_buf, block_size + digest_size); 1167 1168 scatterwalk_map_and_copy(hmac_buf, sha->opad, 0, block_size, 0); 1169 switch (sha->type) { 1170 case CCP_SHA_TYPE_1: 1171 case CCP_SHA_TYPE_224: 1172 case CCP_SHA_TYPE_256: 1173 memcpy(hmac_buf + block_size, 1174 ctx.address + ooffset, 1175 digest_size); 1176 break; 1177 default: 1178 ret = -EINVAL; 1179 goto e_ctx; 1180 } 1181 1182 memset(&hmac_cmd, 0, sizeof(hmac_cmd)); 1183 hmac_cmd.engine = CCP_ENGINE_SHA; 1184 hmac_cmd.u.sha.type = sha->type; 1185 hmac_cmd.u.sha.ctx = sha->ctx; 1186 hmac_cmd.u.sha.ctx_len = sha->ctx_len; 1187 hmac_cmd.u.sha.src = &sg; 1188 hmac_cmd.u.sha.src_len = block_size + digest_size; 1189 hmac_cmd.u.sha.opad = NULL; 1190 hmac_cmd.u.sha.opad_len = 0; 1191 hmac_cmd.u.sha.first = 1; 1192 hmac_cmd.u.sha.final = 1; 1193 hmac_cmd.u.sha.msg_bits = (block_size + digest_size) << 3; 1194 1195 ret = ccp_run_sha_cmd(cmd_q, &hmac_cmd); 1196 if (ret) 1197 cmd->engine_error = hmac_cmd.engine_error; 1198 1199 kfree(hmac_buf); 1200 } 1201 1202 e_data: 1203 if (sha->src) 1204 ccp_free_data(&src, cmd_q); 1205 1206 e_ctx: 1207 ccp_dm_free(&ctx); 1208 1209 return ret; 1210 } 1211 1212 static int ccp_run_rsa_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) 1213 { 1214 struct ccp_rsa_engine *rsa = &cmd->u.rsa; 1215 struct ccp_dm_workarea exp, src; 1216 struct ccp_data dst; 1217 struct ccp_op op; 1218 unsigned int sb_count, i_len, o_len; 1219 int ret; 1220 1221 if (rsa->key_size > CCP_RSA_MAX_WIDTH) 1222 return -EINVAL; 1223 1224 if (!rsa->exp || !rsa->mod || !rsa->src || !rsa->dst) 1225 return -EINVAL; 1226 1227 /* The RSA modulus must precede the message being acted upon, so 1228 * it must be copied to a DMA area where the message and the 1229 * modulus can be concatenated. Therefore the input buffer 1230 * length required is twice the output buffer length (which 1231 * must be a multiple of 256-bits). 1232 */ 1233 o_len = ((rsa->key_size + 255) / 256) * 32; 1234 i_len = o_len * 2; 1235 1236 sb_count = o_len / CCP_SB_BYTES; 1237 1238 memset(&op, 0, sizeof(op)); 1239 op.cmd_q = cmd_q; 1240 op.jobid = ccp_gen_jobid(cmd_q->ccp); 1241 op.sb_key = cmd_q->ccp->vdata->perform->sballoc(cmd_q, sb_count); 1242 1243 if (!op.sb_key) 1244 return -EIO; 1245 1246 /* The RSA exponent may span multiple (32-byte) SB entries and must 1247 * be in little endian format. Reverse copy each 32-byte chunk 1248 * of the exponent (En chunk to E0 chunk, E(n-1) chunk to E1 chunk) 1249 * and each byte within that chunk and do not perform any byte swap 1250 * operations on the passthru operation. 1251 */ 1252 ret = ccp_init_dm_workarea(&exp, cmd_q, o_len, DMA_TO_DEVICE); 1253 if (ret) 1254 goto e_sb; 1255 1256 ret = ccp_reverse_set_dm_area(&exp, 0, rsa->exp, 0, rsa->exp_len); 1257 if (ret) 1258 goto e_exp; 1259 ret = ccp_copy_to_sb(cmd_q, &exp, op.jobid, op.sb_key, 1260 CCP_PASSTHRU_BYTESWAP_NOOP); 1261 if (ret) { 1262 cmd->engine_error = cmd_q->cmd_error; 1263 goto e_exp; 1264 } 1265 1266 /* Concatenate the modulus and the message. Both the modulus and 1267 * the operands must be in little endian format. Since the input 1268 * is in big endian format it must be converted. 1269 */ 1270 ret = ccp_init_dm_workarea(&src, cmd_q, i_len, DMA_TO_DEVICE); 1271 if (ret) 1272 goto e_exp; 1273 1274 ret = ccp_reverse_set_dm_area(&src, 0, rsa->mod, 0, rsa->mod_len); 1275 if (ret) 1276 goto e_src; 1277 ret = ccp_reverse_set_dm_area(&src, o_len, rsa->src, 0, rsa->src_len); 1278 if (ret) 1279 goto e_src; 1280 1281 /* Prepare the output area for the operation */ 1282 ret = ccp_init_data(&dst, cmd_q, rsa->dst, rsa->mod_len, 1283 o_len, DMA_FROM_DEVICE); 1284 if (ret) 1285 goto e_src; 1286 1287 op.soc = 1; 1288 op.src.u.dma.address = src.dma.address; 1289 op.src.u.dma.offset = 0; 1290 op.src.u.dma.length = i_len; 1291 op.dst.u.dma.address = dst.dm_wa.dma.address; 1292 op.dst.u.dma.offset = 0; 1293 op.dst.u.dma.length = o_len; 1294 1295 op.u.rsa.mod_size = rsa->key_size; 1296 op.u.rsa.input_len = i_len; 1297 1298 ret = cmd_q->ccp->vdata->perform->rsa(&op); 1299 if (ret) { 1300 cmd->engine_error = cmd_q->cmd_error; 1301 goto e_dst; 1302 } 1303 1304 ccp_reverse_get_dm_area(&dst.dm_wa, 0, rsa->dst, 0, rsa->mod_len); 1305 1306 e_dst: 1307 ccp_free_data(&dst, cmd_q); 1308 1309 e_src: 1310 ccp_dm_free(&src); 1311 1312 e_exp: 1313 ccp_dm_free(&exp); 1314 1315 e_sb: 1316 cmd_q->ccp->vdata->perform->sbfree(cmd_q, op.sb_key, sb_count); 1317 1318 return ret; 1319 } 1320 1321 static int ccp_run_passthru_cmd(struct ccp_cmd_queue *cmd_q, 1322 struct ccp_cmd *cmd) 1323 { 1324 struct ccp_passthru_engine *pt = &cmd->u.passthru; 1325 struct ccp_dm_workarea mask; 1326 struct ccp_data src, dst; 1327 struct ccp_op op; 1328 bool in_place = false; 1329 unsigned int i; 1330 int ret = 0; 1331 1332 if (!pt->final && (pt->src_len & (CCP_PASSTHRU_BLOCKSIZE - 1))) 1333 return -EINVAL; 1334 1335 if (!pt->src || !pt->dst) 1336 return -EINVAL; 1337 1338 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) { 1339 if (pt->mask_len != CCP_PASSTHRU_MASKSIZE) 1340 return -EINVAL; 1341 if (!pt->mask) 1342 return -EINVAL; 1343 } 1344 1345 BUILD_BUG_ON(CCP_PASSTHRU_SB_COUNT != 1); 1346 1347 memset(&op, 0, sizeof(op)); 1348 op.cmd_q = cmd_q; 1349 op.jobid = CCP_NEW_JOBID(cmd_q->ccp); 1350 1351 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) { 1352 /* Load the mask */ 1353 op.sb_key = cmd_q->sb_key; 1354 1355 ret = ccp_init_dm_workarea(&mask, cmd_q, 1356 CCP_PASSTHRU_SB_COUNT * 1357 CCP_SB_BYTES, 1358 DMA_TO_DEVICE); 1359 if (ret) 1360 return ret; 1361 1362 ccp_set_dm_area(&mask, 0, pt->mask, 0, pt->mask_len); 1363 ret = ccp_copy_to_sb(cmd_q, &mask, op.jobid, op.sb_key, 1364 CCP_PASSTHRU_BYTESWAP_NOOP); 1365 if (ret) { 1366 cmd->engine_error = cmd_q->cmd_error; 1367 goto e_mask; 1368 } 1369 } 1370 1371 /* Prepare the input and output data workareas. For in-place 1372 * operations we need to set the dma direction to BIDIRECTIONAL 1373 * and copy the src workarea to the dst workarea. 1374 */ 1375 if (sg_virt(pt->src) == sg_virt(pt->dst)) 1376 in_place = true; 1377 1378 ret = ccp_init_data(&src, cmd_q, pt->src, pt->src_len, 1379 CCP_PASSTHRU_MASKSIZE, 1380 in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE); 1381 if (ret) 1382 goto e_mask; 1383 1384 if (in_place) { 1385 dst = src; 1386 } else { 1387 ret = ccp_init_data(&dst, cmd_q, pt->dst, pt->src_len, 1388 CCP_PASSTHRU_MASKSIZE, DMA_FROM_DEVICE); 1389 if (ret) 1390 goto e_src; 1391 } 1392 1393 /* Send data to the CCP Passthru engine 1394 * Because the CCP engine works on a single source and destination 1395 * dma address at a time, each entry in the source scatterlist 1396 * (after the dma_map_sg call) must be less than or equal to the 1397 * (remaining) length in the destination scatterlist entry and the 1398 * length must be a multiple of CCP_PASSTHRU_BLOCKSIZE 1399 */ 1400 dst.sg_wa.sg_used = 0; 1401 for (i = 1; i <= src.sg_wa.dma_count; i++) { 1402 if (!dst.sg_wa.sg || 1403 (dst.sg_wa.sg->length < src.sg_wa.sg->length)) { 1404 ret = -EINVAL; 1405 goto e_dst; 1406 } 1407 1408 if (i == src.sg_wa.dma_count) { 1409 op.eom = 1; 1410 op.soc = 1; 1411 } 1412 1413 op.src.type = CCP_MEMTYPE_SYSTEM; 1414 op.src.u.dma.address = sg_dma_address(src.sg_wa.sg); 1415 op.src.u.dma.offset = 0; 1416 op.src.u.dma.length = sg_dma_len(src.sg_wa.sg); 1417 1418 op.dst.type = CCP_MEMTYPE_SYSTEM; 1419 op.dst.u.dma.address = sg_dma_address(dst.sg_wa.sg); 1420 op.dst.u.dma.offset = dst.sg_wa.sg_used; 1421 op.dst.u.dma.length = op.src.u.dma.length; 1422 1423 ret = cmd_q->ccp->vdata->perform->passthru(&op); 1424 if (ret) { 1425 cmd->engine_error = cmd_q->cmd_error; 1426 goto e_dst; 1427 } 1428 1429 dst.sg_wa.sg_used += src.sg_wa.sg->length; 1430 if (dst.sg_wa.sg_used == dst.sg_wa.sg->length) { 1431 dst.sg_wa.sg = sg_next(dst.sg_wa.sg); 1432 dst.sg_wa.sg_used = 0; 1433 } 1434 src.sg_wa.sg = sg_next(src.sg_wa.sg); 1435 } 1436 1437 e_dst: 1438 if (!in_place) 1439 ccp_free_data(&dst, cmd_q); 1440 1441 e_src: 1442 ccp_free_data(&src, cmd_q); 1443 1444 e_mask: 1445 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) 1446 ccp_dm_free(&mask); 1447 1448 return ret; 1449 } 1450 1451 static int ccp_run_passthru_nomap_cmd(struct ccp_cmd_queue *cmd_q, 1452 struct ccp_cmd *cmd) 1453 { 1454 struct ccp_passthru_nomap_engine *pt = &cmd->u.passthru_nomap; 1455 struct ccp_dm_workarea mask; 1456 struct ccp_op op; 1457 int ret; 1458 1459 if (!pt->final && (pt->src_len & (CCP_PASSTHRU_BLOCKSIZE - 1))) 1460 return -EINVAL; 1461 1462 if (!pt->src_dma || !pt->dst_dma) 1463 return -EINVAL; 1464 1465 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) { 1466 if (pt->mask_len != CCP_PASSTHRU_MASKSIZE) 1467 return -EINVAL; 1468 if (!pt->mask) 1469 return -EINVAL; 1470 } 1471 1472 BUILD_BUG_ON(CCP_PASSTHRU_SB_COUNT != 1); 1473 1474 memset(&op, 0, sizeof(op)); 1475 op.cmd_q = cmd_q; 1476 op.jobid = ccp_gen_jobid(cmd_q->ccp); 1477 1478 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) { 1479 /* Load the mask */ 1480 op.sb_key = cmd_q->sb_key; 1481 1482 mask.length = pt->mask_len; 1483 mask.dma.address = pt->mask; 1484 mask.dma.length = pt->mask_len; 1485 1486 ret = ccp_copy_to_sb(cmd_q, &mask, op.jobid, op.sb_key, 1487 CCP_PASSTHRU_BYTESWAP_NOOP); 1488 if (ret) { 1489 cmd->engine_error = cmd_q->cmd_error; 1490 return ret; 1491 } 1492 } 1493 1494 /* Send data to the CCP Passthru engine */ 1495 op.eom = 1; 1496 op.soc = 1; 1497 1498 op.src.type = CCP_MEMTYPE_SYSTEM; 1499 op.src.u.dma.address = pt->src_dma; 1500 op.src.u.dma.offset = 0; 1501 op.src.u.dma.length = pt->src_len; 1502 1503 op.dst.type = CCP_MEMTYPE_SYSTEM; 1504 op.dst.u.dma.address = pt->dst_dma; 1505 op.dst.u.dma.offset = 0; 1506 op.dst.u.dma.length = pt->src_len; 1507 1508 ret = cmd_q->ccp->vdata->perform->passthru(&op); 1509 if (ret) 1510 cmd->engine_error = cmd_q->cmd_error; 1511 1512 return ret; 1513 } 1514 1515 static int ccp_run_ecc_mm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) 1516 { 1517 struct ccp_ecc_engine *ecc = &cmd->u.ecc; 1518 struct ccp_dm_workarea src, dst; 1519 struct ccp_op op; 1520 int ret; 1521 u8 *save; 1522 1523 if (!ecc->u.mm.operand_1 || 1524 (ecc->u.mm.operand_1_len > CCP_ECC_MODULUS_BYTES)) 1525 return -EINVAL; 1526 1527 if (ecc->function != CCP_ECC_FUNCTION_MINV_384BIT) 1528 if (!ecc->u.mm.operand_2 || 1529 (ecc->u.mm.operand_2_len > CCP_ECC_MODULUS_BYTES)) 1530 return -EINVAL; 1531 1532 if (!ecc->u.mm.result || 1533 (ecc->u.mm.result_len < CCP_ECC_MODULUS_BYTES)) 1534 return -EINVAL; 1535 1536 memset(&op, 0, sizeof(op)); 1537 op.cmd_q = cmd_q; 1538 op.jobid = CCP_NEW_JOBID(cmd_q->ccp); 1539 1540 /* Concatenate the modulus and the operands. Both the modulus and 1541 * the operands must be in little endian format. Since the input 1542 * is in big endian format it must be converted and placed in a 1543 * fixed length buffer. 1544 */ 1545 ret = ccp_init_dm_workarea(&src, cmd_q, CCP_ECC_SRC_BUF_SIZE, 1546 DMA_TO_DEVICE); 1547 if (ret) 1548 return ret; 1549 1550 /* Save the workarea address since it is updated in order to perform 1551 * the concatenation 1552 */ 1553 save = src.address; 1554 1555 /* Copy the ECC modulus */ 1556 ret = ccp_reverse_set_dm_area(&src, 0, ecc->mod, 0, ecc->mod_len); 1557 if (ret) 1558 goto e_src; 1559 src.address += CCP_ECC_OPERAND_SIZE; 1560 1561 /* Copy the first operand */ 1562 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.mm.operand_1, 0, 1563 ecc->u.mm.operand_1_len); 1564 if (ret) 1565 goto e_src; 1566 src.address += CCP_ECC_OPERAND_SIZE; 1567 1568 if (ecc->function != CCP_ECC_FUNCTION_MINV_384BIT) { 1569 /* Copy the second operand */ 1570 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.mm.operand_2, 0, 1571 ecc->u.mm.operand_2_len); 1572 if (ret) 1573 goto e_src; 1574 src.address += CCP_ECC_OPERAND_SIZE; 1575 } 1576 1577 /* Restore the workarea address */ 1578 src.address = save; 1579 1580 /* Prepare the output area for the operation */ 1581 ret = ccp_init_dm_workarea(&dst, cmd_q, CCP_ECC_DST_BUF_SIZE, 1582 DMA_FROM_DEVICE); 1583 if (ret) 1584 goto e_src; 1585 1586 op.soc = 1; 1587 op.src.u.dma.address = src.dma.address; 1588 op.src.u.dma.offset = 0; 1589 op.src.u.dma.length = src.length; 1590 op.dst.u.dma.address = dst.dma.address; 1591 op.dst.u.dma.offset = 0; 1592 op.dst.u.dma.length = dst.length; 1593 1594 op.u.ecc.function = cmd->u.ecc.function; 1595 1596 ret = cmd_q->ccp->vdata->perform->ecc(&op); 1597 if (ret) { 1598 cmd->engine_error = cmd_q->cmd_error; 1599 goto e_dst; 1600 } 1601 1602 ecc->ecc_result = le16_to_cpup( 1603 (const __le16 *)(dst.address + CCP_ECC_RESULT_OFFSET)); 1604 if (!(ecc->ecc_result & CCP_ECC_RESULT_SUCCESS)) { 1605 ret = -EIO; 1606 goto e_dst; 1607 } 1608 1609 /* Save the ECC result */ 1610 ccp_reverse_get_dm_area(&dst, 0, ecc->u.mm.result, 0, 1611 CCP_ECC_MODULUS_BYTES); 1612 1613 e_dst: 1614 ccp_dm_free(&dst); 1615 1616 e_src: 1617 ccp_dm_free(&src); 1618 1619 return ret; 1620 } 1621 1622 static int ccp_run_ecc_pm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) 1623 { 1624 struct ccp_ecc_engine *ecc = &cmd->u.ecc; 1625 struct ccp_dm_workarea src, dst; 1626 struct ccp_op op; 1627 int ret; 1628 u8 *save; 1629 1630 if (!ecc->u.pm.point_1.x || 1631 (ecc->u.pm.point_1.x_len > CCP_ECC_MODULUS_BYTES) || 1632 !ecc->u.pm.point_1.y || 1633 (ecc->u.pm.point_1.y_len > CCP_ECC_MODULUS_BYTES)) 1634 return -EINVAL; 1635 1636 if (ecc->function == CCP_ECC_FUNCTION_PADD_384BIT) { 1637 if (!ecc->u.pm.point_2.x || 1638 (ecc->u.pm.point_2.x_len > CCP_ECC_MODULUS_BYTES) || 1639 !ecc->u.pm.point_2.y || 1640 (ecc->u.pm.point_2.y_len > CCP_ECC_MODULUS_BYTES)) 1641 return -EINVAL; 1642 } else { 1643 if (!ecc->u.pm.domain_a || 1644 (ecc->u.pm.domain_a_len > CCP_ECC_MODULUS_BYTES)) 1645 return -EINVAL; 1646 1647 if (ecc->function == CCP_ECC_FUNCTION_PMUL_384BIT) 1648 if (!ecc->u.pm.scalar || 1649 (ecc->u.pm.scalar_len > CCP_ECC_MODULUS_BYTES)) 1650 return -EINVAL; 1651 } 1652 1653 if (!ecc->u.pm.result.x || 1654 (ecc->u.pm.result.x_len < CCP_ECC_MODULUS_BYTES) || 1655 !ecc->u.pm.result.y || 1656 (ecc->u.pm.result.y_len < CCP_ECC_MODULUS_BYTES)) 1657 return -EINVAL; 1658 1659 memset(&op, 0, sizeof(op)); 1660 op.cmd_q = cmd_q; 1661 op.jobid = CCP_NEW_JOBID(cmd_q->ccp); 1662 1663 /* Concatenate the modulus and the operands. Both the modulus and 1664 * the operands must be in little endian format. Since the input 1665 * is in big endian format it must be converted and placed in a 1666 * fixed length buffer. 1667 */ 1668 ret = ccp_init_dm_workarea(&src, cmd_q, CCP_ECC_SRC_BUF_SIZE, 1669 DMA_TO_DEVICE); 1670 if (ret) 1671 return ret; 1672 1673 /* Save the workarea address since it is updated in order to perform 1674 * the concatenation 1675 */ 1676 save = src.address; 1677 1678 /* Copy the ECC modulus */ 1679 ret = ccp_reverse_set_dm_area(&src, 0, ecc->mod, 0, ecc->mod_len); 1680 if (ret) 1681 goto e_src; 1682 src.address += CCP_ECC_OPERAND_SIZE; 1683 1684 /* Copy the first point X and Y coordinate */ 1685 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_1.x, 0, 1686 ecc->u.pm.point_1.x_len); 1687 if (ret) 1688 goto e_src; 1689 src.address += CCP_ECC_OPERAND_SIZE; 1690 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_1.y, 0, 1691 ecc->u.pm.point_1.y_len); 1692 if (ret) 1693 goto e_src; 1694 src.address += CCP_ECC_OPERAND_SIZE; 1695 1696 /* Set the first point Z coordinate to 1 */ 1697 *src.address = 0x01; 1698 src.address += CCP_ECC_OPERAND_SIZE; 1699 1700 if (ecc->function == CCP_ECC_FUNCTION_PADD_384BIT) { 1701 /* Copy the second point X and Y coordinate */ 1702 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_2.x, 0, 1703 ecc->u.pm.point_2.x_len); 1704 if (ret) 1705 goto e_src; 1706 src.address += CCP_ECC_OPERAND_SIZE; 1707 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_2.y, 0, 1708 ecc->u.pm.point_2.y_len); 1709 if (ret) 1710 goto e_src; 1711 src.address += CCP_ECC_OPERAND_SIZE; 1712 1713 /* Set the second point Z coordinate to 1 */ 1714 *src.address = 0x01; 1715 src.address += CCP_ECC_OPERAND_SIZE; 1716 } else { 1717 /* Copy the Domain "a" parameter */ 1718 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.domain_a, 0, 1719 ecc->u.pm.domain_a_len); 1720 if (ret) 1721 goto e_src; 1722 src.address += CCP_ECC_OPERAND_SIZE; 1723 1724 if (ecc->function == CCP_ECC_FUNCTION_PMUL_384BIT) { 1725 /* Copy the scalar value */ 1726 ret = ccp_reverse_set_dm_area(&src, 0, 1727 ecc->u.pm.scalar, 0, 1728 ecc->u.pm.scalar_len); 1729 if (ret) 1730 goto e_src; 1731 src.address += CCP_ECC_OPERAND_SIZE; 1732 } 1733 } 1734 1735 /* Restore the workarea address */ 1736 src.address = save; 1737 1738 /* Prepare the output area for the operation */ 1739 ret = ccp_init_dm_workarea(&dst, cmd_q, CCP_ECC_DST_BUF_SIZE, 1740 DMA_FROM_DEVICE); 1741 if (ret) 1742 goto e_src; 1743 1744 op.soc = 1; 1745 op.src.u.dma.address = src.dma.address; 1746 op.src.u.dma.offset = 0; 1747 op.src.u.dma.length = src.length; 1748 op.dst.u.dma.address = dst.dma.address; 1749 op.dst.u.dma.offset = 0; 1750 op.dst.u.dma.length = dst.length; 1751 1752 op.u.ecc.function = cmd->u.ecc.function; 1753 1754 ret = cmd_q->ccp->vdata->perform->ecc(&op); 1755 if (ret) { 1756 cmd->engine_error = cmd_q->cmd_error; 1757 goto e_dst; 1758 } 1759 1760 ecc->ecc_result = le16_to_cpup( 1761 (const __le16 *)(dst.address + CCP_ECC_RESULT_OFFSET)); 1762 if (!(ecc->ecc_result & CCP_ECC_RESULT_SUCCESS)) { 1763 ret = -EIO; 1764 goto e_dst; 1765 } 1766 1767 /* Save the workarea address since it is updated as we walk through 1768 * to copy the point math result 1769 */ 1770 save = dst.address; 1771 1772 /* Save the ECC result X and Y coordinates */ 1773 ccp_reverse_get_dm_area(&dst, 0, ecc->u.pm.result.x, 0, 1774 CCP_ECC_MODULUS_BYTES); 1775 dst.address += CCP_ECC_OUTPUT_SIZE; 1776 ccp_reverse_get_dm_area(&dst, 0, ecc->u.pm.result.y, 0, 1777 CCP_ECC_MODULUS_BYTES); 1778 dst.address += CCP_ECC_OUTPUT_SIZE; 1779 1780 /* Restore the workarea address */ 1781 dst.address = save; 1782 1783 e_dst: 1784 ccp_dm_free(&dst); 1785 1786 e_src: 1787 ccp_dm_free(&src); 1788 1789 return ret; 1790 } 1791 1792 static int ccp_run_ecc_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) 1793 { 1794 struct ccp_ecc_engine *ecc = &cmd->u.ecc; 1795 1796 ecc->ecc_result = 0; 1797 1798 if (!ecc->mod || 1799 (ecc->mod_len > CCP_ECC_MODULUS_BYTES)) 1800 return -EINVAL; 1801 1802 switch (ecc->function) { 1803 case CCP_ECC_FUNCTION_MMUL_384BIT: 1804 case CCP_ECC_FUNCTION_MADD_384BIT: 1805 case CCP_ECC_FUNCTION_MINV_384BIT: 1806 return ccp_run_ecc_mm_cmd(cmd_q, cmd); 1807 1808 case CCP_ECC_FUNCTION_PADD_384BIT: 1809 case CCP_ECC_FUNCTION_PMUL_384BIT: 1810 case CCP_ECC_FUNCTION_PDBL_384BIT: 1811 return ccp_run_ecc_pm_cmd(cmd_q, cmd); 1812 1813 default: 1814 return -EINVAL; 1815 } 1816 } 1817 1818 int ccp_run_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) 1819 { 1820 int ret; 1821 1822 cmd->engine_error = 0; 1823 cmd_q->cmd_error = 0; 1824 cmd_q->int_rcvd = 0; 1825 cmd_q->free_slots = cmd_q->ccp->vdata->perform->get_free_slots(cmd_q); 1826 1827 switch (cmd->engine) { 1828 case CCP_ENGINE_AES: 1829 ret = ccp_run_aes_cmd(cmd_q, cmd); 1830 break; 1831 case CCP_ENGINE_XTS_AES_128: 1832 ret = ccp_run_xts_aes_cmd(cmd_q, cmd); 1833 break; 1834 case CCP_ENGINE_SHA: 1835 ret = ccp_run_sha_cmd(cmd_q, cmd); 1836 break; 1837 case CCP_ENGINE_RSA: 1838 ret = ccp_run_rsa_cmd(cmd_q, cmd); 1839 break; 1840 case CCP_ENGINE_PASSTHRU: 1841 if (cmd->flags & CCP_CMD_PASSTHRU_NO_DMA_MAP) 1842 ret = ccp_run_passthru_nomap_cmd(cmd_q, cmd); 1843 else 1844 ret = ccp_run_passthru_cmd(cmd_q, cmd); 1845 break; 1846 case CCP_ENGINE_ECC: 1847 ret = ccp_run_ecc_cmd(cmd_q, cmd); 1848 break; 1849 default: 1850 ret = -EINVAL; 1851 } 1852 1853 return ret; 1854 } 1855