1 /* 2 * AMD Cryptographic Coprocessor (CCP) driver 3 * 4 * Copyright (C) 2013 Advanced Micro Devices, Inc. 5 * 6 * Author: Tom Lendacky <thomas.lendacky@amd.com> 7 * 8 * This program is free software; you can redistribute it and/or modify 9 * it under the terms of the GNU General Public License version 2 as 10 * published by the Free Software Foundation. 11 */ 12 13 #include <linux/module.h> 14 #include <linux/kernel.h> 15 #include <linux/pci.h> 16 #include <linux/pci_ids.h> 17 #include <linux/kthread.h> 18 #include <linux/sched.h> 19 #include <linux/interrupt.h> 20 #include <linux/spinlock.h> 21 #include <linux/mutex.h> 22 #include <linux/delay.h> 23 #include <linux/ccp.h> 24 #include <linux/scatterlist.h> 25 #include <crypto/scatterwalk.h> 26 #include <crypto/sha.h> 27 28 #include "ccp-dev.h" 29 30 enum ccp_memtype { 31 CCP_MEMTYPE_SYSTEM = 0, 32 CCP_MEMTYPE_KSB, 33 CCP_MEMTYPE_LOCAL, 34 CCP_MEMTYPE__LAST, 35 }; 36 37 struct ccp_dma_info { 38 dma_addr_t address; 39 unsigned int offset; 40 unsigned int length; 41 enum dma_data_direction dir; 42 }; 43 44 struct ccp_dm_workarea { 45 struct device *dev; 46 struct dma_pool *dma_pool; 47 unsigned int length; 48 49 u8 *address; 50 struct ccp_dma_info dma; 51 }; 52 53 struct ccp_sg_workarea { 54 struct scatterlist *sg; 55 int nents; 56 57 struct scatterlist *dma_sg; 58 struct device *dma_dev; 59 unsigned int dma_count; 60 enum dma_data_direction dma_dir; 61 62 unsigned int sg_used; 63 64 u64 bytes_left; 65 }; 66 67 struct ccp_data { 68 struct ccp_sg_workarea sg_wa; 69 struct ccp_dm_workarea dm_wa; 70 }; 71 72 struct ccp_mem { 73 enum ccp_memtype type; 74 union { 75 struct ccp_dma_info dma; 76 u32 ksb; 77 } u; 78 }; 79 80 struct ccp_aes_op { 81 enum ccp_aes_type type; 82 enum ccp_aes_mode mode; 83 enum ccp_aes_action action; 84 }; 85 86 struct ccp_xts_aes_op { 87 enum ccp_aes_action action; 88 enum ccp_xts_aes_unit_size unit_size; 89 }; 90 91 struct ccp_sha_op { 92 enum ccp_sha_type type; 93 u64 msg_bits; 94 }; 95 96 struct ccp_rsa_op { 97 u32 mod_size; 98 u32 input_len; 99 }; 100 101 struct ccp_passthru_op { 102 enum ccp_passthru_bitwise bit_mod; 103 enum ccp_passthru_byteswap byte_swap; 104 }; 105 106 struct ccp_ecc_op { 107 enum ccp_ecc_function function; 108 }; 109 110 struct ccp_op { 111 struct ccp_cmd_queue *cmd_q; 112 113 u32 jobid; 114 u32 ioc; 115 u32 soc; 116 u32 ksb_key; 117 u32 ksb_ctx; 118 u32 init; 119 u32 eom; 120 121 struct ccp_mem src; 122 struct ccp_mem dst; 123 124 union { 125 struct ccp_aes_op aes; 126 struct ccp_xts_aes_op xts; 127 struct ccp_sha_op sha; 128 struct ccp_rsa_op rsa; 129 struct ccp_passthru_op passthru; 130 struct ccp_ecc_op ecc; 131 } u; 132 }; 133 134 /* SHA initial context values */ 135 static const __be32 ccp_sha1_init[CCP_SHA_CTXSIZE / sizeof(__be32)] = { 136 cpu_to_be32(SHA1_H0), cpu_to_be32(SHA1_H1), 137 cpu_to_be32(SHA1_H2), cpu_to_be32(SHA1_H3), 138 cpu_to_be32(SHA1_H4), 0, 0, 0, 139 }; 140 141 static const __be32 ccp_sha224_init[CCP_SHA_CTXSIZE / sizeof(__be32)] = { 142 cpu_to_be32(SHA224_H0), cpu_to_be32(SHA224_H1), 143 cpu_to_be32(SHA224_H2), cpu_to_be32(SHA224_H3), 144 cpu_to_be32(SHA224_H4), cpu_to_be32(SHA224_H5), 145 cpu_to_be32(SHA224_H6), cpu_to_be32(SHA224_H7), 146 }; 147 148 static const __be32 ccp_sha256_init[CCP_SHA_CTXSIZE / sizeof(__be32)] = { 149 cpu_to_be32(SHA256_H0), cpu_to_be32(SHA256_H1), 150 cpu_to_be32(SHA256_H2), cpu_to_be32(SHA256_H3), 151 cpu_to_be32(SHA256_H4), cpu_to_be32(SHA256_H5), 152 cpu_to_be32(SHA256_H6), cpu_to_be32(SHA256_H7), 153 }; 154 155 /* The CCP cannot perform zero-length sha operations so the caller 156 * is required to buffer data for the final operation. However, a 157 * sha operation for a message with a total length of zero is valid 158 * so known values are required to supply the result. 159 */ 160 static const u8 ccp_sha1_zero[CCP_SHA_CTXSIZE] = { 161 0xda, 0x39, 0xa3, 0xee, 0x5e, 0x6b, 0x4b, 0x0d, 162 0x32, 0x55, 0xbf, 0xef, 0x95, 0x60, 0x18, 0x90, 163 0xaf, 0xd8, 0x07, 0x09, 0x00, 0x00, 0x00, 0x00, 164 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 165 }; 166 167 static const u8 ccp_sha224_zero[CCP_SHA_CTXSIZE] = { 168 0xd1, 0x4a, 0x02, 0x8c, 0x2a, 0x3a, 0x2b, 0xc9, 169 0x47, 0x61, 0x02, 0xbb, 0x28, 0x82, 0x34, 0xc4, 170 0x15, 0xa2, 0xb0, 0x1f, 0x82, 0x8e, 0xa6, 0x2a, 171 0xc5, 0xb3, 0xe4, 0x2f, 0x00, 0x00, 0x00, 0x00, 172 }; 173 174 static const u8 ccp_sha256_zero[CCP_SHA_CTXSIZE] = { 175 0xe3, 0xb0, 0xc4, 0x42, 0x98, 0xfc, 0x1c, 0x14, 176 0x9a, 0xfb, 0xf4, 0xc8, 0x99, 0x6f, 0xb9, 0x24, 177 0x27, 0xae, 0x41, 0xe4, 0x64, 0x9b, 0x93, 0x4c, 178 0xa4, 0x95, 0x99, 0x1b, 0x78, 0x52, 0xb8, 0x55, 179 }; 180 181 static u32 ccp_addr_lo(struct ccp_dma_info *info) 182 { 183 return lower_32_bits(info->address + info->offset); 184 } 185 186 static u32 ccp_addr_hi(struct ccp_dma_info *info) 187 { 188 return upper_32_bits(info->address + info->offset) & 0x0000ffff; 189 } 190 191 static int ccp_do_cmd(struct ccp_op *op, u32 *cr, unsigned int cr_count) 192 { 193 struct ccp_cmd_queue *cmd_q = op->cmd_q; 194 struct ccp_device *ccp = cmd_q->ccp; 195 void __iomem *cr_addr; 196 u32 cr0, cmd; 197 unsigned int i; 198 int ret = 0; 199 200 /* We could read a status register to see how many free slots 201 * are actually available, but reading that register resets it 202 * and you could lose some error information. 203 */ 204 cmd_q->free_slots--; 205 206 cr0 = (cmd_q->id << REQ0_CMD_Q_SHIFT) 207 | (op->jobid << REQ0_JOBID_SHIFT) 208 | REQ0_WAIT_FOR_WRITE; 209 210 if (op->soc) 211 cr0 |= REQ0_STOP_ON_COMPLETE 212 | REQ0_INT_ON_COMPLETE; 213 214 if (op->ioc || !cmd_q->free_slots) 215 cr0 |= REQ0_INT_ON_COMPLETE; 216 217 /* Start at CMD_REQ1 */ 218 cr_addr = ccp->io_regs + CMD_REQ0 + CMD_REQ_INCR; 219 220 mutex_lock(&ccp->req_mutex); 221 222 /* Write CMD_REQ1 through CMD_REQx first */ 223 for (i = 0; i < cr_count; i++, cr_addr += CMD_REQ_INCR) 224 iowrite32(*(cr + i), cr_addr); 225 226 /* Tell the CCP to start */ 227 wmb(); 228 iowrite32(cr0, ccp->io_regs + CMD_REQ0); 229 230 mutex_unlock(&ccp->req_mutex); 231 232 if (cr0 & REQ0_INT_ON_COMPLETE) { 233 /* Wait for the job to complete */ 234 ret = wait_event_interruptible(cmd_q->int_queue, 235 cmd_q->int_rcvd); 236 if (ret || cmd_q->cmd_error) { 237 /* On error delete all related jobs from the queue */ 238 cmd = (cmd_q->id << DEL_Q_ID_SHIFT) 239 | op->jobid; 240 241 iowrite32(cmd, ccp->io_regs + DEL_CMD_Q_JOB); 242 243 if (!ret) 244 ret = -EIO; 245 } else if (op->soc) { 246 /* Delete just head job from the queue on SoC */ 247 cmd = DEL_Q_ACTIVE 248 | (cmd_q->id << DEL_Q_ID_SHIFT) 249 | op->jobid; 250 251 iowrite32(cmd, ccp->io_regs + DEL_CMD_Q_JOB); 252 } 253 254 cmd_q->free_slots = CMD_Q_DEPTH(cmd_q->q_status); 255 256 cmd_q->int_rcvd = 0; 257 } 258 259 return ret; 260 } 261 262 static int ccp_perform_aes(struct ccp_op *op) 263 { 264 u32 cr[6]; 265 266 /* Fill out the register contents for REQ1 through REQ6 */ 267 cr[0] = (CCP_ENGINE_AES << REQ1_ENGINE_SHIFT) 268 | (op->u.aes.type << REQ1_AES_TYPE_SHIFT) 269 | (op->u.aes.mode << REQ1_AES_MODE_SHIFT) 270 | (op->u.aes.action << REQ1_AES_ACTION_SHIFT) 271 | (op->ksb_key << REQ1_KEY_KSB_SHIFT); 272 cr[1] = op->src.u.dma.length - 1; 273 cr[2] = ccp_addr_lo(&op->src.u.dma); 274 cr[3] = (op->ksb_ctx << REQ4_KSB_SHIFT) 275 | (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT) 276 | ccp_addr_hi(&op->src.u.dma); 277 cr[4] = ccp_addr_lo(&op->dst.u.dma); 278 cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT) 279 | ccp_addr_hi(&op->dst.u.dma); 280 281 if (op->u.aes.mode == CCP_AES_MODE_CFB) 282 cr[0] |= ((0x7f) << REQ1_AES_CFB_SIZE_SHIFT); 283 284 if (op->eom) 285 cr[0] |= REQ1_EOM; 286 287 if (op->init) 288 cr[0] |= REQ1_INIT; 289 290 return ccp_do_cmd(op, cr, ARRAY_SIZE(cr)); 291 } 292 293 static int ccp_perform_xts_aes(struct ccp_op *op) 294 { 295 u32 cr[6]; 296 297 /* Fill out the register contents for REQ1 through REQ6 */ 298 cr[0] = (CCP_ENGINE_XTS_AES_128 << REQ1_ENGINE_SHIFT) 299 | (op->u.xts.action << REQ1_AES_ACTION_SHIFT) 300 | (op->u.xts.unit_size << REQ1_XTS_AES_SIZE_SHIFT) 301 | (op->ksb_key << REQ1_KEY_KSB_SHIFT); 302 cr[1] = op->src.u.dma.length - 1; 303 cr[2] = ccp_addr_lo(&op->src.u.dma); 304 cr[3] = (op->ksb_ctx << REQ4_KSB_SHIFT) 305 | (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT) 306 | ccp_addr_hi(&op->src.u.dma); 307 cr[4] = ccp_addr_lo(&op->dst.u.dma); 308 cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT) 309 | ccp_addr_hi(&op->dst.u.dma); 310 311 if (op->eom) 312 cr[0] |= REQ1_EOM; 313 314 if (op->init) 315 cr[0] |= REQ1_INIT; 316 317 return ccp_do_cmd(op, cr, ARRAY_SIZE(cr)); 318 } 319 320 static int ccp_perform_sha(struct ccp_op *op) 321 { 322 u32 cr[6]; 323 324 /* Fill out the register contents for REQ1 through REQ6 */ 325 cr[0] = (CCP_ENGINE_SHA << REQ1_ENGINE_SHIFT) 326 | (op->u.sha.type << REQ1_SHA_TYPE_SHIFT) 327 | REQ1_INIT; 328 cr[1] = op->src.u.dma.length - 1; 329 cr[2] = ccp_addr_lo(&op->src.u.dma); 330 cr[3] = (op->ksb_ctx << REQ4_KSB_SHIFT) 331 | (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT) 332 | ccp_addr_hi(&op->src.u.dma); 333 334 if (op->eom) { 335 cr[0] |= REQ1_EOM; 336 cr[4] = lower_32_bits(op->u.sha.msg_bits); 337 cr[5] = upper_32_bits(op->u.sha.msg_bits); 338 } else { 339 cr[4] = 0; 340 cr[5] = 0; 341 } 342 343 return ccp_do_cmd(op, cr, ARRAY_SIZE(cr)); 344 } 345 346 static int ccp_perform_rsa(struct ccp_op *op) 347 { 348 u32 cr[6]; 349 350 /* Fill out the register contents for REQ1 through REQ6 */ 351 cr[0] = (CCP_ENGINE_RSA << REQ1_ENGINE_SHIFT) 352 | (op->u.rsa.mod_size << REQ1_RSA_MOD_SIZE_SHIFT) 353 | (op->ksb_key << REQ1_KEY_KSB_SHIFT) 354 | REQ1_EOM; 355 cr[1] = op->u.rsa.input_len - 1; 356 cr[2] = ccp_addr_lo(&op->src.u.dma); 357 cr[3] = (op->ksb_ctx << REQ4_KSB_SHIFT) 358 | (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT) 359 | ccp_addr_hi(&op->src.u.dma); 360 cr[4] = ccp_addr_lo(&op->dst.u.dma); 361 cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT) 362 | ccp_addr_hi(&op->dst.u.dma); 363 364 return ccp_do_cmd(op, cr, ARRAY_SIZE(cr)); 365 } 366 367 static int ccp_perform_passthru(struct ccp_op *op) 368 { 369 u32 cr[6]; 370 371 /* Fill out the register contents for REQ1 through REQ6 */ 372 cr[0] = (CCP_ENGINE_PASSTHRU << REQ1_ENGINE_SHIFT) 373 | (op->u.passthru.bit_mod << REQ1_PT_BW_SHIFT) 374 | (op->u.passthru.byte_swap << REQ1_PT_BS_SHIFT); 375 376 if (op->src.type == CCP_MEMTYPE_SYSTEM) 377 cr[1] = op->src.u.dma.length - 1; 378 else 379 cr[1] = op->dst.u.dma.length - 1; 380 381 if (op->src.type == CCP_MEMTYPE_SYSTEM) { 382 cr[2] = ccp_addr_lo(&op->src.u.dma); 383 cr[3] = (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT) 384 | ccp_addr_hi(&op->src.u.dma); 385 386 if (op->u.passthru.bit_mod != CCP_PASSTHRU_BITWISE_NOOP) 387 cr[3] |= (op->ksb_key << REQ4_KSB_SHIFT); 388 } else { 389 cr[2] = op->src.u.ksb * CCP_KSB_BYTES; 390 cr[3] = (CCP_MEMTYPE_KSB << REQ4_MEMTYPE_SHIFT); 391 } 392 393 if (op->dst.type == CCP_MEMTYPE_SYSTEM) { 394 cr[4] = ccp_addr_lo(&op->dst.u.dma); 395 cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT) 396 | ccp_addr_hi(&op->dst.u.dma); 397 } else { 398 cr[4] = op->dst.u.ksb * CCP_KSB_BYTES; 399 cr[5] = (CCP_MEMTYPE_KSB << REQ6_MEMTYPE_SHIFT); 400 } 401 402 if (op->eom) 403 cr[0] |= REQ1_EOM; 404 405 return ccp_do_cmd(op, cr, ARRAY_SIZE(cr)); 406 } 407 408 static int ccp_perform_ecc(struct ccp_op *op) 409 { 410 u32 cr[6]; 411 412 /* Fill out the register contents for REQ1 through REQ6 */ 413 cr[0] = REQ1_ECC_AFFINE_CONVERT 414 | (CCP_ENGINE_ECC << REQ1_ENGINE_SHIFT) 415 | (op->u.ecc.function << REQ1_ECC_FUNCTION_SHIFT) 416 | REQ1_EOM; 417 cr[1] = op->src.u.dma.length - 1; 418 cr[2] = ccp_addr_lo(&op->src.u.dma); 419 cr[3] = (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT) 420 | ccp_addr_hi(&op->src.u.dma); 421 cr[4] = ccp_addr_lo(&op->dst.u.dma); 422 cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT) 423 | ccp_addr_hi(&op->dst.u.dma); 424 425 return ccp_do_cmd(op, cr, ARRAY_SIZE(cr)); 426 } 427 428 static u32 ccp_alloc_ksb(struct ccp_device *ccp, unsigned int count) 429 { 430 int start; 431 432 for (;;) { 433 mutex_lock(&ccp->ksb_mutex); 434 435 start = (u32)bitmap_find_next_zero_area(ccp->ksb, 436 ccp->ksb_count, 437 ccp->ksb_start, 438 count, 0); 439 if (start <= ccp->ksb_count) { 440 bitmap_set(ccp->ksb, start, count); 441 442 mutex_unlock(&ccp->ksb_mutex); 443 break; 444 } 445 446 ccp->ksb_avail = 0; 447 448 mutex_unlock(&ccp->ksb_mutex); 449 450 /* Wait for KSB entries to become available */ 451 if (wait_event_interruptible(ccp->ksb_queue, ccp->ksb_avail)) 452 return 0; 453 } 454 455 return KSB_START + start; 456 } 457 458 static void ccp_free_ksb(struct ccp_device *ccp, unsigned int start, 459 unsigned int count) 460 { 461 if (!start) 462 return; 463 464 mutex_lock(&ccp->ksb_mutex); 465 466 bitmap_clear(ccp->ksb, start - KSB_START, count); 467 468 ccp->ksb_avail = 1; 469 470 mutex_unlock(&ccp->ksb_mutex); 471 472 wake_up_interruptible_all(&ccp->ksb_queue); 473 } 474 475 static u32 ccp_gen_jobid(struct ccp_device *ccp) 476 { 477 return atomic_inc_return(&ccp->current_id) & CCP_JOBID_MASK; 478 } 479 480 static void ccp_sg_free(struct ccp_sg_workarea *wa) 481 { 482 if (wa->dma_count) 483 dma_unmap_sg(wa->dma_dev, wa->dma_sg, wa->nents, wa->dma_dir); 484 485 wa->dma_count = 0; 486 } 487 488 static int ccp_init_sg_workarea(struct ccp_sg_workarea *wa, struct device *dev, 489 struct scatterlist *sg, u64 len, 490 enum dma_data_direction dma_dir) 491 { 492 memset(wa, 0, sizeof(*wa)); 493 494 wa->sg = sg; 495 if (!sg) 496 return 0; 497 498 wa->nents = sg_nents_for_len(sg, len); 499 if (wa->nents < 0) 500 return wa->nents; 501 502 wa->bytes_left = len; 503 wa->sg_used = 0; 504 505 if (len == 0) 506 return 0; 507 508 if (dma_dir == DMA_NONE) 509 return 0; 510 511 wa->dma_sg = sg; 512 wa->dma_dev = dev; 513 wa->dma_dir = dma_dir; 514 wa->dma_count = dma_map_sg(dev, sg, wa->nents, dma_dir); 515 if (!wa->dma_count) 516 return -ENOMEM; 517 518 return 0; 519 } 520 521 static void ccp_update_sg_workarea(struct ccp_sg_workarea *wa, unsigned int len) 522 { 523 unsigned int nbytes = min_t(u64, len, wa->bytes_left); 524 525 if (!wa->sg) 526 return; 527 528 wa->sg_used += nbytes; 529 wa->bytes_left -= nbytes; 530 if (wa->sg_used == wa->sg->length) { 531 wa->sg = sg_next(wa->sg); 532 wa->sg_used = 0; 533 } 534 } 535 536 static void ccp_dm_free(struct ccp_dm_workarea *wa) 537 { 538 if (wa->length <= CCP_DMAPOOL_MAX_SIZE) { 539 if (wa->address) 540 dma_pool_free(wa->dma_pool, wa->address, 541 wa->dma.address); 542 } else { 543 if (wa->dma.address) 544 dma_unmap_single(wa->dev, wa->dma.address, wa->length, 545 wa->dma.dir); 546 kfree(wa->address); 547 } 548 549 wa->address = NULL; 550 wa->dma.address = 0; 551 } 552 553 static int ccp_init_dm_workarea(struct ccp_dm_workarea *wa, 554 struct ccp_cmd_queue *cmd_q, 555 unsigned int len, 556 enum dma_data_direction dir) 557 { 558 memset(wa, 0, sizeof(*wa)); 559 560 if (!len) 561 return 0; 562 563 wa->dev = cmd_q->ccp->dev; 564 wa->length = len; 565 566 if (len <= CCP_DMAPOOL_MAX_SIZE) { 567 wa->dma_pool = cmd_q->dma_pool; 568 569 wa->address = dma_pool_alloc(wa->dma_pool, GFP_KERNEL, 570 &wa->dma.address); 571 if (!wa->address) 572 return -ENOMEM; 573 574 wa->dma.length = CCP_DMAPOOL_MAX_SIZE; 575 576 memset(wa->address, 0, CCP_DMAPOOL_MAX_SIZE); 577 } else { 578 wa->address = kzalloc(len, GFP_KERNEL); 579 if (!wa->address) 580 return -ENOMEM; 581 582 wa->dma.address = dma_map_single(wa->dev, wa->address, len, 583 dir); 584 if (!wa->dma.address) 585 return -ENOMEM; 586 587 wa->dma.length = len; 588 } 589 wa->dma.dir = dir; 590 591 return 0; 592 } 593 594 static void ccp_set_dm_area(struct ccp_dm_workarea *wa, unsigned int wa_offset, 595 struct scatterlist *sg, unsigned int sg_offset, 596 unsigned int len) 597 { 598 WARN_ON(!wa->address); 599 600 scatterwalk_map_and_copy(wa->address + wa_offset, sg, sg_offset, len, 601 0); 602 } 603 604 static void ccp_get_dm_area(struct ccp_dm_workarea *wa, unsigned int wa_offset, 605 struct scatterlist *sg, unsigned int sg_offset, 606 unsigned int len) 607 { 608 WARN_ON(!wa->address); 609 610 scatterwalk_map_and_copy(wa->address + wa_offset, sg, sg_offset, len, 611 1); 612 } 613 614 static int ccp_reverse_set_dm_area(struct ccp_dm_workarea *wa, 615 struct scatterlist *sg, 616 unsigned int len, unsigned int se_len, 617 bool sign_extend) 618 { 619 unsigned int nbytes, sg_offset, dm_offset, ksb_len, i; 620 u8 buffer[CCP_REVERSE_BUF_SIZE]; 621 622 if (WARN_ON(se_len > sizeof(buffer))) 623 return -EINVAL; 624 625 sg_offset = len; 626 dm_offset = 0; 627 nbytes = len; 628 while (nbytes) { 629 ksb_len = min_t(unsigned int, nbytes, se_len); 630 sg_offset -= ksb_len; 631 632 scatterwalk_map_and_copy(buffer, sg, sg_offset, ksb_len, 0); 633 for (i = 0; i < ksb_len; i++) 634 wa->address[dm_offset + i] = buffer[ksb_len - i - 1]; 635 636 dm_offset += ksb_len; 637 nbytes -= ksb_len; 638 639 if ((ksb_len != se_len) && sign_extend) { 640 /* Must sign-extend to nearest sign-extend length */ 641 if (wa->address[dm_offset - 1] & 0x80) 642 memset(wa->address + dm_offset, 0xff, 643 se_len - ksb_len); 644 } 645 } 646 647 return 0; 648 } 649 650 static void ccp_reverse_get_dm_area(struct ccp_dm_workarea *wa, 651 struct scatterlist *sg, 652 unsigned int len) 653 { 654 unsigned int nbytes, sg_offset, dm_offset, ksb_len, i; 655 u8 buffer[CCP_REVERSE_BUF_SIZE]; 656 657 sg_offset = 0; 658 dm_offset = len; 659 nbytes = len; 660 while (nbytes) { 661 ksb_len = min_t(unsigned int, nbytes, sizeof(buffer)); 662 dm_offset -= ksb_len; 663 664 for (i = 0; i < ksb_len; i++) 665 buffer[ksb_len - i - 1] = wa->address[dm_offset + i]; 666 scatterwalk_map_and_copy(buffer, sg, sg_offset, ksb_len, 1); 667 668 sg_offset += ksb_len; 669 nbytes -= ksb_len; 670 } 671 } 672 673 static void ccp_free_data(struct ccp_data *data, struct ccp_cmd_queue *cmd_q) 674 { 675 ccp_dm_free(&data->dm_wa); 676 ccp_sg_free(&data->sg_wa); 677 } 678 679 static int ccp_init_data(struct ccp_data *data, struct ccp_cmd_queue *cmd_q, 680 struct scatterlist *sg, u64 sg_len, 681 unsigned int dm_len, 682 enum dma_data_direction dir) 683 { 684 int ret; 685 686 memset(data, 0, sizeof(*data)); 687 688 ret = ccp_init_sg_workarea(&data->sg_wa, cmd_q->ccp->dev, sg, sg_len, 689 dir); 690 if (ret) 691 goto e_err; 692 693 ret = ccp_init_dm_workarea(&data->dm_wa, cmd_q, dm_len, dir); 694 if (ret) 695 goto e_err; 696 697 return 0; 698 699 e_err: 700 ccp_free_data(data, cmd_q); 701 702 return ret; 703 } 704 705 static unsigned int ccp_queue_buf(struct ccp_data *data, unsigned int from) 706 { 707 struct ccp_sg_workarea *sg_wa = &data->sg_wa; 708 struct ccp_dm_workarea *dm_wa = &data->dm_wa; 709 unsigned int buf_count, nbytes; 710 711 /* Clear the buffer if setting it */ 712 if (!from) 713 memset(dm_wa->address, 0, dm_wa->length); 714 715 if (!sg_wa->sg) 716 return 0; 717 718 /* Perform the copy operation 719 * nbytes will always be <= UINT_MAX because dm_wa->length is 720 * an unsigned int 721 */ 722 nbytes = min_t(u64, sg_wa->bytes_left, dm_wa->length); 723 scatterwalk_map_and_copy(dm_wa->address, sg_wa->sg, sg_wa->sg_used, 724 nbytes, from); 725 726 /* Update the structures and generate the count */ 727 buf_count = 0; 728 while (sg_wa->bytes_left && (buf_count < dm_wa->length)) { 729 nbytes = min(sg_wa->sg->length - sg_wa->sg_used, 730 dm_wa->length - buf_count); 731 nbytes = min_t(u64, sg_wa->bytes_left, nbytes); 732 733 buf_count += nbytes; 734 ccp_update_sg_workarea(sg_wa, nbytes); 735 } 736 737 return buf_count; 738 } 739 740 static unsigned int ccp_fill_queue_buf(struct ccp_data *data) 741 { 742 return ccp_queue_buf(data, 0); 743 } 744 745 static unsigned int ccp_empty_queue_buf(struct ccp_data *data) 746 { 747 return ccp_queue_buf(data, 1); 748 } 749 750 static void ccp_prepare_data(struct ccp_data *src, struct ccp_data *dst, 751 struct ccp_op *op, unsigned int block_size, 752 bool blocksize_op) 753 { 754 unsigned int sg_src_len, sg_dst_len, op_len; 755 756 /* The CCP can only DMA from/to one address each per operation. This 757 * requires that we find the smallest DMA area between the source 758 * and destination. The resulting len values will always be <= UINT_MAX 759 * because the dma length is an unsigned int. 760 */ 761 sg_src_len = sg_dma_len(src->sg_wa.sg) - src->sg_wa.sg_used; 762 sg_src_len = min_t(u64, src->sg_wa.bytes_left, sg_src_len); 763 764 if (dst) { 765 sg_dst_len = sg_dma_len(dst->sg_wa.sg) - dst->sg_wa.sg_used; 766 sg_dst_len = min_t(u64, src->sg_wa.bytes_left, sg_dst_len); 767 op_len = min(sg_src_len, sg_dst_len); 768 } else { 769 op_len = sg_src_len; 770 } 771 772 /* The data operation length will be at least block_size in length 773 * or the smaller of available sg room remaining for the source or 774 * the destination 775 */ 776 op_len = max(op_len, block_size); 777 778 /* Unless we have to buffer data, there's no reason to wait */ 779 op->soc = 0; 780 781 if (sg_src_len < block_size) { 782 /* Not enough data in the sg element, so it 783 * needs to be buffered into a blocksize chunk 784 */ 785 int cp_len = ccp_fill_queue_buf(src); 786 787 op->soc = 1; 788 op->src.u.dma.address = src->dm_wa.dma.address; 789 op->src.u.dma.offset = 0; 790 op->src.u.dma.length = (blocksize_op) ? block_size : cp_len; 791 } else { 792 /* Enough data in the sg element, but we need to 793 * adjust for any previously copied data 794 */ 795 op->src.u.dma.address = sg_dma_address(src->sg_wa.sg); 796 op->src.u.dma.offset = src->sg_wa.sg_used; 797 op->src.u.dma.length = op_len & ~(block_size - 1); 798 799 ccp_update_sg_workarea(&src->sg_wa, op->src.u.dma.length); 800 } 801 802 if (dst) { 803 if (sg_dst_len < block_size) { 804 /* Not enough room in the sg element or we're on the 805 * last piece of data (when using padding), so the 806 * output needs to be buffered into a blocksize chunk 807 */ 808 op->soc = 1; 809 op->dst.u.dma.address = dst->dm_wa.dma.address; 810 op->dst.u.dma.offset = 0; 811 op->dst.u.dma.length = op->src.u.dma.length; 812 } else { 813 /* Enough room in the sg element, but we need to 814 * adjust for any previously used area 815 */ 816 op->dst.u.dma.address = sg_dma_address(dst->sg_wa.sg); 817 op->dst.u.dma.offset = dst->sg_wa.sg_used; 818 op->dst.u.dma.length = op->src.u.dma.length; 819 } 820 } 821 } 822 823 static void ccp_process_data(struct ccp_data *src, struct ccp_data *dst, 824 struct ccp_op *op) 825 { 826 op->init = 0; 827 828 if (dst) { 829 if (op->dst.u.dma.address == dst->dm_wa.dma.address) 830 ccp_empty_queue_buf(dst); 831 else 832 ccp_update_sg_workarea(&dst->sg_wa, 833 op->dst.u.dma.length); 834 } 835 } 836 837 static int ccp_copy_to_from_ksb(struct ccp_cmd_queue *cmd_q, 838 struct ccp_dm_workarea *wa, u32 jobid, u32 ksb, 839 u32 byte_swap, bool from) 840 { 841 struct ccp_op op; 842 843 memset(&op, 0, sizeof(op)); 844 845 op.cmd_q = cmd_q; 846 op.jobid = jobid; 847 op.eom = 1; 848 849 if (from) { 850 op.soc = 1; 851 op.src.type = CCP_MEMTYPE_KSB; 852 op.src.u.ksb = ksb; 853 op.dst.type = CCP_MEMTYPE_SYSTEM; 854 op.dst.u.dma.address = wa->dma.address; 855 op.dst.u.dma.length = wa->length; 856 } else { 857 op.src.type = CCP_MEMTYPE_SYSTEM; 858 op.src.u.dma.address = wa->dma.address; 859 op.src.u.dma.length = wa->length; 860 op.dst.type = CCP_MEMTYPE_KSB; 861 op.dst.u.ksb = ksb; 862 } 863 864 op.u.passthru.byte_swap = byte_swap; 865 866 return ccp_perform_passthru(&op); 867 } 868 869 static int ccp_copy_to_ksb(struct ccp_cmd_queue *cmd_q, 870 struct ccp_dm_workarea *wa, u32 jobid, u32 ksb, 871 u32 byte_swap) 872 { 873 return ccp_copy_to_from_ksb(cmd_q, wa, jobid, ksb, byte_swap, false); 874 } 875 876 static int ccp_copy_from_ksb(struct ccp_cmd_queue *cmd_q, 877 struct ccp_dm_workarea *wa, u32 jobid, u32 ksb, 878 u32 byte_swap) 879 { 880 return ccp_copy_to_from_ksb(cmd_q, wa, jobid, ksb, byte_swap, true); 881 } 882 883 static int ccp_run_aes_cmac_cmd(struct ccp_cmd_queue *cmd_q, 884 struct ccp_cmd *cmd) 885 { 886 struct ccp_aes_engine *aes = &cmd->u.aes; 887 struct ccp_dm_workarea key, ctx; 888 struct ccp_data src; 889 struct ccp_op op; 890 unsigned int dm_offset; 891 int ret; 892 893 if (!((aes->key_len == AES_KEYSIZE_128) || 894 (aes->key_len == AES_KEYSIZE_192) || 895 (aes->key_len == AES_KEYSIZE_256))) 896 return -EINVAL; 897 898 if (aes->src_len & (AES_BLOCK_SIZE - 1)) 899 return -EINVAL; 900 901 if (aes->iv_len != AES_BLOCK_SIZE) 902 return -EINVAL; 903 904 if (!aes->key || !aes->iv || !aes->src) 905 return -EINVAL; 906 907 if (aes->cmac_final) { 908 if (aes->cmac_key_len != AES_BLOCK_SIZE) 909 return -EINVAL; 910 911 if (!aes->cmac_key) 912 return -EINVAL; 913 } 914 915 BUILD_BUG_ON(CCP_AES_KEY_KSB_COUNT != 1); 916 BUILD_BUG_ON(CCP_AES_CTX_KSB_COUNT != 1); 917 918 ret = -EIO; 919 memset(&op, 0, sizeof(op)); 920 op.cmd_q = cmd_q; 921 op.jobid = ccp_gen_jobid(cmd_q->ccp); 922 op.ksb_key = cmd_q->ksb_key; 923 op.ksb_ctx = cmd_q->ksb_ctx; 924 op.init = 1; 925 op.u.aes.type = aes->type; 926 op.u.aes.mode = aes->mode; 927 op.u.aes.action = aes->action; 928 929 /* All supported key sizes fit in a single (32-byte) KSB entry 930 * and must be in little endian format. Use the 256-bit byte 931 * swap passthru option to convert from big endian to little 932 * endian. 933 */ 934 ret = ccp_init_dm_workarea(&key, cmd_q, 935 CCP_AES_KEY_KSB_COUNT * CCP_KSB_BYTES, 936 DMA_TO_DEVICE); 937 if (ret) 938 return ret; 939 940 dm_offset = CCP_KSB_BYTES - aes->key_len; 941 ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len); 942 ret = ccp_copy_to_ksb(cmd_q, &key, op.jobid, op.ksb_key, 943 CCP_PASSTHRU_BYTESWAP_256BIT); 944 if (ret) { 945 cmd->engine_error = cmd_q->cmd_error; 946 goto e_key; 947 } 948 949 /* The AES context fits in a single (32-byte) KSB entry and 950 * must be in little endian format. Use the 256-bit byte swap 951 * passthru option to convert from big endian to little endian. 952 */ 953 ret = ccp_init_dm_workarea(&ctx, cmd_q, 954 CCP_AES_CTX_KSB_COUNT * CCP_KSB_BYTES, 955 DMA_BIDIRECTIONAL); 956 if (ret) 957 goto e_key; 958 959 dm_offset = CCP_KSB_BYTES - AES_BLOCK_SIZE; 960 ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len); 961 ret = ccp_copy_to_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx, 962 CCP_PASSTHRU_BYTESWAP_256BIT); 963 if (ret) { 964 cmd->engine_error = cmd_q->cmd_error; 965 goto e_ctx; 966 } 967 968 /* Send data to the CCP AES engine */ 969 ret = ccp_init_data(&src, cmd_q, aes->src, aes->src_len, 970 AES_BLOCK_SIZE, DMA_TO_DEVICE); 971 if (ret) 972 goto e_ctx; 973 974 while (src.sg_wa.bytes_left) { 975 ccp_prepare_data(&src, NULL, &op, AES_BLOCK_SIZE, true); 976 if (aes->cmac_final && !src.sg_wa.bytes_left) { 977 op.eom = 1; 978 979 /* Push the K1/K2 key to the CCP now */ 980 ret = ccp_copy_from_ksb(cmd_q, &ctx, op.jobid, 981 op.ksb_ctx, 982 CCP_PASSTHRU_BYTESWAP_256BIT); 983 if (ret) { 984 cmd->engine_error = cmd_q->cmd_error; 985 goto e_src; 986 } 987 988 ccp_set_dm_area(&ctx, 0, aes->cmac_key, 0, 989 aes->cmac_key_len); 990 ret = ccp_copy_to_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx, 991 CCP_PASSTHRU_BYTESWAP_256BIT); 992 if (ret) { 993 cmd->engine_error = cmd_q->cmd_error; 994 goto e_src; 995 } 996 } 997 998 ret = ccp_perform_aes(&op); 999 if (ret) { 1000 cmd->engine_error = cmd_q->cmd_error; 1001 goto e_src; 1002 } 1003 1004 ccp_process_data(&src, NULL, &op); 1005 } 1006 1007 /* Retrieve the AES context - convert from LE to BE using 1008 * 32-byte (256-bit) byteswapping 1009 */ 1010 ret = ccp_copy_from_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx, 1011 CCP_PASSTHRU_BYTESWAP_256BIT); 1012 if (ret) { 1013 cmd->engine_error = cmd_q->cmd_error; 1014 goto e_src; 1015 } 1016 1017 /* ...but we only need AES_BLOCK_SIZE bytes */ 1018 dm_offset = CCP_KSB_BYTES - AES_BLOCK_SIZE; 1019 ccp_get_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len); 1020 1021 e_src: 1022 ccp_free_data(&src, cmd_q); 1023 1024 e_ctx: 1025 ccp_dm_free(&ctx); 1026 1027 e_key: 1028 ccp_dm_free(&key); 1029 1030 return ret; 1031 } 1032 1033 static int ccp_run_aes_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) 1034 { 1035 struct ccp_aes_engine *aes = &cmd->u.aes; 1036 struct ccp_dm_workarea key, ctx; 1037 struct ccp_data src, dst; 1038 struct ccp_op op; 1039 unsigned int dm_offset; 1040 bool in_place = false; 1041 int ret; 1042 1043 if (aes->mode == CCP_AES_MODE_CMAC) 1044 return ccp_run_aes_cmac_cmd(cmd_q, cmd); 1045 1046 if (!((aes->key_len == AES_KEYSIZE_128) || 1047 (aes->key_len == AES_KEYSIZE_192) || 1048 (aes->key_len == AES_KEYSIZE_256))) 1049 return -EINVAL; 1050 1051 if (((aes->mode == CCP_AES_MODE_ECB) || 1052 (aes->mode == CCP_AES_MODE_CBC) || 1053 (aes->mode == CCP_AES_MODE_CFB)) && 1054 (aes->src_len & (AES_BLOCK_SIZE - 1))) 1055 return -EINVAL; 1056 1057 if (!aes->key || !aes->src || !aes->dst) 1058 return -EINVAL; 1059 1060 if (aes->mode != CCP_AES_MODE_ECB) { 1061 if (aes->iv_len != AES_BLOCK_SIZE) 1062 return -EINVAL; 1063 1064 if (!aes->iv) 1065 return -EINVAL; 1066 } 1067 1068 BUILD_BUG_ON(CCP_AES_KEY_KSB_COUNT != 1); 1069 BUILD_BUG_ON(CCP_AES_CTX_KSB_COUNT != 1); 1070 1071 ret = -EIO; 1072 memset(&op, 0, sizeof(op)); 1073 op.cmd_q = cmd_q; 1074 op.jobid = ccp_gen_jobid(cmd_q->ccp); 1075 op.ksb_key = cmd_q->ksb_key; 1076 op.ksb_ctx = cmd_q->ksb_ctx; 1077 op.init = (aes->mode == CCP_AES_MODE_ECB) ? 0 : 1; 1078 op.u.aes.type = aes->type; 1079 op.u.aes.mode = aes->mode; 1080 op.u.aes.action = aes->action; 1081 1082 /* All supported key sizes fit in a single (32-byte) KSB entry 1083 * and must be in little endian format. Use the 256-bit byte 1084 * swap passthru option to convert from big endian to little 1085 * endian. 1086 */ 1087 ret = ccp_init_dm_workarea(&key, cmd_q, 1088 CCP_AES_KEY_KSB_COUNT * CCP_KSB_BYTES, 1089 DMA_TO_DEVICE); 1090 if (ret) 1091 return ret; 1092 1093 dm_offset = CCP_KSB_BYTES - aes->key_len; 1094 ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len); 1095 ret = ccp_copy_to_ksb(cmd_q, &key, op.jobid, op.ksb_key, 1096 CCP_PASSTHRU_BYTESWAP_256BIT); 1097 if (ret) { 1098 cmd->engine_error = cmd_q->cmd_error; 1099 goto e_key; 1100 } 1101 1102 /* The AES context fits in a single (32-byte) KSB entry and 1103 * must be in little endian format. Use the 256-bit byte swap 1104 * passthru option to convert from big endian to little endian. 1105 */ 1106 ret = ccp_init_dm_workarea(&ctx, cmd_q, 1107 CCP_AES_CTX_KSB_COUNT * CCP_KSB_BYTES, 1108 DMA_BIDIRECTIONAL); 1109 if (ret) 1110 goto e_key; 1111 1112 if (aes->mode != CCP_AES_MODE_ECB) { 1113 /* Load the AES context - conver to LE */ 1114 dm_offset = CCP_KSB_BYTES - AES_BLOCK_SIZE; 1115 ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len); 1116 ret = ccp_copy_to_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx, 1117 CCP_PASSTHRU_BYTESWAP_256BIT); 1118 if (ret) { 1119 cmd->engine_error = cmd_q->cmd_error; 1120 goto e_ctx; 1121 } 1122 } 1123 1124 /* Prepare the input and output data workareas. For in-place 1125 * operations we need to set the dma direction to BIDIRECTIONAL 1126 * and copy the src workarea to the dst workarea. 1127 */ 1128 if (sg_virt(aes->src) == sg_virt(aes->dst)) 1129 in_place = true; 1130 1131 ret = ccp_init_data(&src, cmd_q, aes->src, aes->src_len, 1132 AES_BLOCK_SIZE, 1133 in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE); 1134 if (ret) 1135 goto e_ctx; 1136 1137 if (in_place) { 1138 dst = src; 1139 } else { 1140 ret = ccp_init_data(&dst, cmd_q, aes->dst, aes->src_len, 1141 AES_BLOCK_SIZE, DMA_FROM_DEVICE); 1142 if (ret) 1143 goto e_src; 1144 } 1145 1146 /* Send data to the CCP AES engine */ 1147 while (src.sg_wa.bytes_left) { 1148 ccp_prepare_data(&src, &dst, &op, AES_BLOCK_SIZE, true); 1149 if (!src.sg_wa.bytes_left) { 1150 op.eom = 1; 1151 1152 /* Since we don't retrieve the AES context in ECB 1153 * mode we have to wait for the operation to complete 1154 * on the last piece of data 1155 */ 1156 if (aes->mode == CCP_AES_MODE_ECB) 1157 op.soc = 1; 1158 } 1159 1160 ret = ccp_perform_aes(&op); 1161 if (ret) { 1162 cmd->engine_error = cmd_q->cmd_error; 1163 goto e_dst; 1164 } 1165 1166 ccp_process_data(&src, &dst, &op); 1167 } 1168 1169 if (aes->mode != CCP_AES_MODE_ECB) { 1170 /* Retrieve the AES context - convert from LE to BE using 1171 * 32-byte (256-bit) byteswapping 1172 */ 1173 ret = ccp_copy_from_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx, 1174 CCP_PASSTHRU_BYTESWAP_256BIT); 1175 if (ret) { 1176 cmd->engine_error = cmd_q->cmd_error; 1177 goto e_dst; 1178 } 1179 1180 /* ...but we only need AES_BLOCK_SIZE bytes */ 1181 dm_offset = CCP_KSB_BYTES - AES_BLOCK_SIZE; 1182 ccp_get_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len); 1183 } 1184 1185 e_dst: 1186 if (!in_place) 1187 ccp_free_data(&dst, cmd_q); 1188 1189 e_src: 1190 ccp_free_data(&src, cmd_q); 1191 1192 e_ctx: 1193 ccp_dm_free(&ctx); 1194 1195 e_key: 1196 ccp_dm_free(&key); 1197 1198 return ret; 1199 } 1200 1201 static int ccp_run_xts_aes_cmd(struct ccp_cmd_queue *cmd_q, 1202 struct ccp_cmd *cmd) 1203 { 1204 struct ccp_xts_aes_engine *xts = &cmd->u.xts; 1205 struct ccp_dm_workarea key, ctx; 1206 struct ccp_data src, dst; 1207 struct ccp_op op; 1208 unsigned int unit_size, dm_offset; 1209 bool in_place = false; 1210 int ret; 1211 1212 switch (xts->unit_size) { 1213 case CCP_XTS_AES_UNIT_SIZE_16: 1214 unit_size = 16; 1215 break; 1216 case CCP_XTS_AES_UNIT_SIZE_512: 1217 unit_size = 512; 1218 break; 1219 case CCP_XTS_AES_UNIT_SIZE_1024: 1220 unit_size = 1024; 1221 break; 1222 case CCP_XTS_AES_UNIT_SIZE_2048: 1223 unit_size = 2048; 1224 break; 1225 case CCP_XTS_AES_UNIT_SIZE_4096: 1226 unit_size = 4096; 1227 break; 1228 1229 default: 1230 return -EINVAL; 1231 } 1232 1233 if (xts->key_len != AES_KEYSIZE_128) 1234 return -EINVAL; 1235 1236 if (!xts->final && (xts->src_len & (AES_BLOCK_SIZE - 1))) 1237 return -EINVAL; 1238 1239 if (xts->iv_len != AES_BLOCK_SIZE) 1240 return -EINVAL; 1241 1242 if (!xts->key || !xts->iv || !xts->src || !xts->dst) 1243 return -EINVAL; 1244 1245 BUILD_BUG_ON(CCP_XTS_AES_KEY_KSB_COUNT != 1); 1246 BUILD_BUG_ON(CCP_XTS_AES_CTX_KSB_COUNT != 1); 1247 1248 ret = -EIO; 1249 memset(&op, 0, sizeof(op)); 1250 op.cmd_q = cmd_q; 1251 op.jobid = ccp_gen_jobid(cmd_q->ccp); 1252 op.ksb_key = cmd_q->ksb_key; 1253 op.ksb_ctx = cmd_q->ksb_ctx; 1254 op.init = 1; 1255 op.u.xts.action = xts->action; 1256 op.u.xts.unit_size = xts->unit_size; 1257 1258 /* All supported key sizes fit in a single (32-byte) KSB entry 1259 * and must be in little endian format. Use the 256-bit byte 1260 * swap passthru option to convert from big endian to little 1261 * endian. 1262 */ 1263 ret = ccp_init_dm_workarea(&key, cmd_q, 1264 CCP_XTS_AES_KEY_KSB_COUNT * CCP_KSB_BYTES, 1265 DMA_TO_DEVICE); 1266 if (ret) 1267 return ret; 1268 1269 dm_offset = CCP_KSB_BYTES - AES_KEYSIZE_128; 1270 ccp_set_dm_area(&key, dm_offset, xts->key, 0, xts->key_len); 1271 ccp_set_dm_area(&key, 0, xts->key, dm_offset, xts->key_len); 1272 ret = ccp_copy_to_ksb(cmd_q, &key, op.jobid, op.ksb_key, 1273 CCP_PASSTHRU_BYTESWAP_256BIT); 1274 if (ret) { 1275 cmd->engine_error = cmd_q->cmd_error; 1276 goto e_key; 1277 } 1278 1279 /* The AES context fits in a single (32-byte) KSB entry and 1280 * for XTS is already in little endian format so no byte swapping 1281 * is needed. 1282 */ 1283 ret = ccp_init_dm_workarea(&ctx, cmd_q, 1284 CCP_XTS_AES_CTX_KSB_COUNT * CCP_KSB_BYTES, 1285 DMA_BIDIRECTIONAL); 1286 if (ret) 1287 goto e_key; 1288 1289 ccp_set_dm_area(&ctx, 0, xts->iv, 0, xts->iv_len); 1290 ret = ccp_copy_to_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx, 1291 CCP_PASSTHRU_BYTESWAP_NOOP); 1292 if (ret) { 1293 cmd->engine_error = cmd_q->cmd_error; 1294 goto e_ctx; 1295 } 1296 1297 /* Prepare the input and output data workareas. For in-place 1298 * operations we need to set the dma direction to BIDIRECTIONAL 1299 * and copy the src workarea to the dst workarea. 1300 */ 1301 if (sg_virt(xts->src) == sg_virt(xts->dst)) 1302 in_place = true; 1303 1304 ret = ccp_init_data(&src, cmd_q, xts->src, xts->src_len, 1305 unit_size, 1306 in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE); 1307 if (ret) 1308 goto e_ctx; 1309 1310 if (in_place) { 1311 dst = src; 1312 } else { 1313 ret = ccp_init_data(&dst, cmd_q, xts->dst, xts->src_len, 1314 unit_size, DMA_FROM_DEVICE); 1315 if (ret) 1316 goto e_src; 1317 } 1318 1319 /* Send data to the CCP AES engine */ 1320 while (src.sg_wa.bytes_left) { 1321 ccp_prepare_data(&src, &dst, &op, unit_size, true); 1322 if (!src.sg_wa.bytes_left) 1323 op.eom = 1; 1324 1325 ret = ccp_perform_xts_aes(&op); 1326 if (ret) { 1327 cmd->engine_error = cmd_q->cmd_error; 1328 goto e_dst; 1329 } 1330 1331 ccp_process_data(&src, &dst, &op); 1332 } 1333 1334 /* Retrieve the AES context - convert from LE to BE using 1335 * 32-byte (256-bit) byteswapping 1336 */ 1337 ret = ccp_copy_from_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx, 1338 CCP_PASSTHRU_BYTESWAP_256BIT); 1339 if (ret) { 1340 cmd->engine_error = cmd_q->cmd_error; 1341 goto e_dst; 1342 } 1343 1344 /* ...but we only need AES_BLOCK_SIZE bytes */ 1345 dm_offset = CCP_KSB_BYTES - AES_BLOCK_SIZE; 1346 ccp_get_dm_area(&ctx, dm_offset, xts->iv, 0, xts->iv_len); 1347 1348 e_dst: 1349 if (!in_place) 1350 ccp_free_data(&dst, cmd_q); 1351 1352 e_src: 1353 ccp_free_data(&src, cmd_q); 1354 1355 e_ctx: 1356 ccp_dm_free(&ctx); 1357 1358 e_key: 1359 ccp_dm_free(&key); 1360 1361 return ret; 1362 } 1363 1364 static int ccp_run_sha_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) 1365 { 1366 struct ccp_sha_engine *sha = &cmd->u.sha; 1367 struct ccp_dm_workarea ctx; 1368 struct ccp_data src; 1369 struct ccp_op op; 1370 int ret; 1371 1372 if (sha->ctx_len != CCP_SHA_CTXSIZE) 1373 return -EINVAL; 1374 1375 if (!sha->ctx) 1376 return -EINVAL; 1377 1378 if (!sha->final && (sha->src_len & (CCP_SHA_BLOCKSIZE - 1))) 1379 return -EINVAL; 1380 1381 if (!sha->src_len) { 1382 const u8 *sha_zero; 1383 1384 /* Not final, just return */ 1385 if (!sha->final) 1386 return 0; 1387 1388 /* CCP can't do a zero length sha operation so the caller 1389 * must buffer the data. 1390 */ 1391 if (sha->msg_bits) 1392 return -EINVAL; 1393 1394 /* A sha operation for a message with a total length of zero, 1395 * return known result. 1396 */ 1397 switch (sha->type) { 1398 case CCP_SHA_TYPE_1: 1399 sha_zero = ccp_sha1_zero; 1400 break; 1401 case CCP_SHA_TYPE_224: 1402 sha_zero = ccp_sha224_zero; 1403 break; 1404 case CCP_SHA_TYPE_256: 1405 sha_zero = ccp_sha256_zero; 1406 break; 1407 default: 1408 return -EINVAL; 1409 } 1410 1411 scatterwalk_map_and_copy((void *)sha_zero, sha->ctx, 0, 1412 sha->ctx_len, 1); 1413 1414 return 0; 1415 } 1416 1417 if (!sha->src) 1418 return -EINVAL; 1419 1420 BUILD_BUG_ON(CCP_SHA_KSB_COUNT != 1); 1421 1422 memset(&op, 0, sizeof(op)); 1423 op.cmd_q = cmd_q; 1424 op.jobid = ccp_gen_jobid(cmd_q->ccp); 1425 op.ksb_ctx = cmd_q->ksb_ctx; 1426 op.u.sha.type = sha->type; 1427 op.u.sha.msg_bits = sha->msg_bits; 1428 1429 /* The SHA context fits in a single (32-byte) KSB entry and 1430 * must be in little endian format. Use the 256-bit byte swap 1431 * passthru option to convert from big endian to little endian. 1432 */ 1433 ret = ccp_init_dm_workarea(&ctx, cmd_q, 1434 CCP_SHA_KSB_COUNT * CCP_KSB_BYTES, 1435 DMA_BIDIRECTIONAL); 1436 if (ret) 1437 return ret; 1438 1439 if (sha->first) { 1440 const __be32 *init; 1441 1442 switch (sha->type) { 1443 case CCP_SHA_TYPE_1: 1444 init = ccp_sha1_init; 1445 break; 1446 case CCP_SHA_TYPE_224: 1447 init = ccp_sha224_init; 1448 break; 1449 case CCP_SHA_TYPE_256: 1450 init = ccp_sha256_init; 1451 break; 1452 default: 1453 ret = -EINVAL; 1454 goto e_ctx; 1455 } 1456 memcpy(ctx.address, init, CCP_SHA_CTXSIZE); 1457 } else { 1458 ccp_set_dm_area(&ctx, 0, sha->ctx, 0, sha->ctx_len); 1459 } 1460 1461 ret = ccp_copy_to_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx, 1462 CCP_PASSTHRU_BYTESWAP_256BIT); 1463 if (ret) { 1464 cmd->engine_error = cmd_q->cmd_error; 1465 goto e_ctx; 1466 } 1467 1468 /* Send data to the CCP SHA engine */ 1469 ret = ccp_init_data(&src, cmd_q, sha->src, sha->src_len, 1470 CCP_SHA_BLOCKSIZE, DMA_TO_DEVICE); 1471 if (ret) 1472 goto e_ctx; 1473 1474 while (src.sg_wa.bytes_left) { 1475 ccp_prepare_data(&src, NULL, &op, CCP_SHA_BLOCKSIZE, false); 1476 if (sha->final && !src.sg_wa.bytes_left) 1477 op.eom = 1; 1478 1479 ret = ccp_perform_sha(&op); 1480 if (ret) { 1481 cmd->engine_error = cmd_q->cmd_error; 1482 goto e_data; 1483 } 1484 1485 ccp_process_data(&src, NULL, &op); 1486 } 1487 1488 /* Retrieve the SHA context - convert from LE to BE using 1489 * 32-byte (256-bit) byteswapping to BE 1490 */ 1491 ret = ccp_copy_from_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx, 1492 CCP_PASSTHRU_BYTESWAP_256BIT); 1493 if (ret) { 1494 cmd->engine_error = cmd_q->cmd_error; 1495 goto e_data; 1496 } 1497 1498 ccp_get_dm_area(&ctx, 0, sha->ctx, 0, sha->ctx_len); 1499 1500 if (sha->final && sha->opad) { 1501 /* HMAC operation, recursively perform final SHA */ 1502 struct ccp_cmd hmac_cmd; 1503 struct scatterlist sg; 1504 u64 block_size, digest_size; 1505 u8 *hmac_buf; 1506 1507 switch (sha->type) { 1508 case CCP_SHA_TYPE_1: 1509 block_size = SHA1_BLOCK_SIZE; 1510 digest_size = SHA1_DIGEST_SIZE; 1511 break; 1512 case CCP_SHA_TYPE_224: 1513 block_size = SHA224_BLOCK_SIZE; 1514 digest_size = SHA224_DIGEST_SIZE; 1515 break; 1516 case CCP_SHA_TYPE_256: 1517 block_size = SHA256_BLOCK_SIZE; 1518 digest_size = SHA256_DIGEST_SIZE; 1519 break; 1520 default: 1521 ret = -EINVAL; 1522 goto e_data; 1523 } 1524 1525 if (sha->opad_len != block_size) { 1526 ret = -EINVAL; 1527 goto e_data; 1528 } 1529 1530 hmac_buf = kmalloc(block_size + digest_size, GFP_KERNEL); 1531 if (!hmac_buf) { 1532 ret = -ENOMEM; 1533 goto e_data; 1534 } 1535 sg_init_one(&sg, hmac_buf, block_size + digest_size); 1536 1537 scatterwalk_map_and_copy(hmac_buf, sha->opad, 0, block_size, 0); 1538 memcpy(hmac_buf + block_size, ctx.address, digest_size); 1539 1540 memset(&hmac_cmd, 0, sizeof(hmac_cmd)); 1541 hmac_cmd.engine = CCP_ENGINE_SHA; 1542 hmac_cmd.u.sha.type = sha->type; 1543 hmac_cmd.u.sha.ctx = sha->ctx; 1544 hmac_cmd.u.sha.ctx_len = sha->ctx_len; 1545 hmac_cmd.u.sha.src = &sg; 1546 hmac_cmd.u.sha.src_len = block_size + digest_size; 1547 hmac_cmd.u.sha.opad = NULL; 1548 hmac_cmd.u.sha.opad_len = 0; 1549 hmac_cmd.u.sha.first = 1; 1550 hmac_cmd.u.sha.final = 1; 1551 hmac_cmd.u.sha.msg_bits = (block_size + digest_size) << 3; 1552 1553 ret = ccp_run_sha_cmd(cmd_q, &hmac_cmd); 1554 if (ret) 1555 cmd->engine_error = hmac_cmd.engine_error; 1556 1557 kfree(hmac_buf); 1558 } 1559 1560 e_data: 1561 ccp_free_data(&src, cmd_q); 1562 1563 e_ctx: 1564 ccp_dm_free(&ctx); 1565 1566 return ret; 1567 } 1568 1569 static int ccp_run_rsa_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) 1570 { 1571 struct ccp_rsa_engine *rsa = &cmd->u.rsa; 1572 struct ccp_dm_workarea exp, src; 1573 struct ccp_data dst; 1574 struct ccp_op op; 1575 unsigned int ksb_count, i_len, o_len; 1576 int ret; 1577 1578 if (rsa->key_size > CCP_RSA_MAX_WIDTH) 1579 return -EINVAL; 1580 1581 if (!rsa->exp || !rsa->mod || !rsa->src || !rsa->dst) 1582 return -EINVAL; 1583 1584 /* The RSA modulus must precede the message being acted upon, so 1585 * it must be copied to a DMA area where the message and the 1586 * modulus can be concatenated. Therefore the input buffer 1587 * length required is twice the output buffer length (which 1588 * must be a multiple of 256-bits). 1589 */ 1590 o_len = ((rsa->key_size + 255) / 256) * 32; 1591 i_len = o_len * 2; 1592 1593 ksb_count = o_len / CCP_KSB_BYTES; 1594 1595 memset(&op, 0, sizeof(op)); 1596 op.cmd_q = cmd_q; 1597 op.jobid = ccp_gen_jobid(cmd_q->ccp); 1598 op.ksb_key = ccp_alloc_ksb(cmd_q->ccp, ksb_count); 1599 if (!op.ksb_key) 1600 return -EIO; 1601 1602 /* The RSA exponent may span multiple (32-byte) KSB entries and must 1603 * be in little endian format. Reverse copy each 32-byte chunk 1604 * of the exponent (En chunk to E0 chunk, E(n-1) chunk to E1 chunk) 1605 * and each byte within that chunk and do not perform any byte swap 1606 * operations on the passthru operation. 1607 */ 1608 ret = ccp_init_dm_workarea(&exp, cmd_q, o_len, DMA_TO_DEVICE); 1609 if (ret) 1610 goto e_ksb; 1611 1612 ret = ccp_reverse_set_dm_area(&exp, rsa->exp, rsa->exp_len, 1613 CCP_KSB_BYTES, false); 1614 if (ret) 1615 goto e_exp; 1616 ret = ccp_copy_to_ksb(cmd_q, &exp, op.jobid, op.ksb_key, 1617 CCP_PASSTHRU_BYTESWAP_NOOP); 1618 if (ret) { 1619 cmd->engine_error = cmd_q->cmd_error; 1620 goto e_exp; 1621 } 1622 1623 /* Concatenate the modulus and the message. Both the modulus and 1624 * the operands must be in little endian format. Since the input 1625 * is in big endian format it must be converted. 1626 */ 1627 ret = ccp_init_dm_workarea(&src, cmd_q, i_len, DMA_TO_DEVICE); 1628 if (ret) 1629 goto e_exp; 1630 1631 ret = ccp_reverse_set_dm_area(&src, rsa->mod, rsa->mod_len, 1632 CCP_KSB_BYTES, false); 1633 if (ret) 1634 goto e_src; 1635 src.address += o_len; /* Adjust the address for the copy operation */ 1636 ret = ccp_reverse_set_dm_area(&src, rsa->src, rsa->src_len, 1637 CCP_KSB_BYTES, false); 1638 if (ret) 1639 goto e_src; 1640 src.address -= o_len; /* Reset the address to original value */ 1641 1642 /* Prepare the output area for the operation */ 1643 ret = ccp_init_data(&dst, cmd_q, rsa->dst, rsa->mod_len, 1644 o_len, DMA_FROM_DEVICE); 1645 if (ret) 1646 goto e_src; 1647 1648 op.soc = 1; 1649 op.src.u.dma.address = src.dma.address; 1650 op.src.u.dma.offset = 0; 1651 op.src.u.dma.length = i_len; 1652 op.dst.u.dma.address = dst.dm_wa.dma.address; 1653 op.dst.u.dma.offset = 0; 1654 op.dst.u.dma.length = o_len; 1655 1656 op.u.rsa.mod_size = rsa->key_size; 1657 op.u.rsa.input_len = i_len; 1658 1659 ret = ccp_perform_rsa(&op); 1660 if (ret) { 1661 cmd->engine_error = cmd_q->cmd_error; 1662 goto e_dst; 1663 } 1664 1665 ccp_reverse_get_dm_area(&dst.dm_wa, rsa->dst, rsa->mod_len); 1666 1667 e_dst: 1668 ccp_free_data(&dst, cmd_q); 1669 1670 e_src: 1671 ccp_dm_free(&src); 1672 1673 e_exp: 1674 ccp_dm_free(&exp); 1675 1676 e_ksb: 1677 ccp_free_ksb(cmd_q->ccp, op.ksb_key, ksb_count); 1678 1679 return ret; 1680 } 1681 1682 static int ccp_run_passthru_cmd(struct ccp_cmd_queue *cmd_q, 1683 struct ccp_cmd *cmd) 1684 { 1685 struct ccp_passthru_engine *pt = &cmd->u.passthru; 1686 struct ccp_dm_workarea mask; 1687 struct ccp_data src, dst; 1688 struct ccp_op op; 1689 bool in_place = false; 1690 unsigned int i; 1691 int ret; 1692 1693 if (!pt->final && (pt->src_len & (CCP_PASSTHRU_BLOCKSIZE - 1))) 1694 return -EINVAL; 1695 1696 if (!pt->src || !pt->dst) 1697 return -EINVAL; 1698 1699 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) { 1700 if (pt->mask_len != CCP_PASSTHRU_MASKSIZE) 1701 return -EINVAL; 1702 if (!pt->mask) 1703 return -EINVAL; 1704 } 1705 1706 BUILD_BUG_ON(CCP_PASSTHRU_KSB_COUNT != 1); 1707 1708 memset(&op, 0, sizeof(op)); 1709 op.cmd_q = cmd_q; 1710 op.jobid = ccp_gen_jobid(cmd_q->ccp); 1711 1712 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) { 1713 /* Load the mask */ 1714 op.ksb_key = cmd_q->ksb_key; 1715 1716 ret = ccp_init_dm_workarea(&mask, cmd_q, 1717 CCP_PASSTHRU_KSB_COUNT * 1718 CCP_KSB_BYTES, 1719 DMA_TO_DEVICE); 1720 if (ret) 1721 return ret; 1722 1723 ccp_set_dm_area(&mask, 0, pt->mask, 0, pt->mask_len); 1724 ret = ccp_copy_to_ksb(cmd_q, &mask, op.jobid, op.ksb_key, 1725 CCP_PASSTHRU_BYTESWAP_NOOP); 1726 if (ret) { 1727 cmd->engine_error = cmd_q->cmd_error; 1728 goto e_mask; 1729 } 1730 } 1731 1732 /* Prepare the input and output data workareas. For in-place 1733 * operations we need to set the dma direction to BIDIRECTIONAL 1734 * and copy the src workarea to the dst workarea. 1735 */ 1736 if (sg_virt(pt->src) == sg_virt(pt->dst)) 1737 in_place = true; 1738 1739 ret = ccp_init_data(&src, cmd_q, pt->src, pt->src_len, 1740 CCP_PASSTHRU_MASKSIZE, 1741 in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE); 1742 if (ret) 1743 goto e_mask; 1744 1745 if (in_place) { 1746 dst = src; 1747 } else { 1748 ret = ccp_init_data(&dst, cmd_q, pt->dst, pt->src_len, 1749 CCP_PASSTHRU_MASKSIZE, DMA_FROM_DEVICE); 1750 if (ret) 1751 goto e_src; 1752 } 1753 1754 /* Send data to the CCP Passthru engine 1755 * Because the CCP engine works on a single source and destination 1756 * dma address at a time, each entry in the source scatterlist 1757 * (after the dma_map_sg call) must be less than or equal to the 1758 * (remaining) length in the destination scatterlist entry and the 1759 * length must be a multiple of CCP_PASSTHRU_BLOCKSIZE 1760 */ 1761 dst.sg_wa.sg_used = 0; 1762 for (i = 1; i <= src.sg_wa.dma_count; i++) { 1763 if (!dst.sg_wa.sg || 1764 (dst.sg_wa.sg->length < src.sg_wa.sg->length)) { 1765 ret = -EINVAL; 1766 goto e_dst; 1767 } 1768 1769 if (i == src.sg_wa.dma_count) { 1770 op.eom = 1; 1771 op.soc = 1; 1772 } 1773 1774 op.src.type = CCP_MEMTYPE_SYSTEM; 1775 op.src.u.dma.address = sg_dma_address(src.sg_wa.sg); 1776 op.src.u.dma.offset = 0; 1777 op.src.u.dma.length = sg_dma_len(src.sg_wa.sg); 1778 1779 op.dst.type = CCP_MEMTYPE_SYSTEM; 1780 op.dst.u.dma.address = sg_dma_address(dst.sg_wa.sg); 1781 op.dst.u.dma.offset = dst.sg_wa.sg_used; 1782 op.dst.u.dma.length = op.src.u.dma.length; 1783 1784 ret = ccp_perform_passthru(&op); 1785 if (ret) { 1786 cmd->engine_error = cmd_q->cmd_error; 1787 goto e_dst; 1788 } 1789 1790 dst.sg_wa.sg_used += src.sg_wa.sg->length; 1791 if (dst.sg_wa.sg_used == dst.sg_wa.sg->length) { 1792 dst.sg_wa.sg = sg_next(dst.sg_wa.sg); 1793 dst.sg_wa.sg_used = 0; 1794 } 1795 src.sg_wa.sg = sg_next(src.sg_wa.sg); 1796 } 1797 1798 e_dst: 1799 if (!in_place) 1800 ccp_free_data(&dst, cmd_q); 1801 1802 e_src: 1803 ccp_free_data(&src, cmd_q); 1804 1805 e_mask: 1806 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) 1807 ccp_dm_free(&mask); 1808 1809 return ret; 1810 } 1811 1812 static int ccp_run_ecc_mm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) 1813 { 1814 struct ccp_ecc_engine *ecc = &cmd->u.ecc; 1815 struct ccp_dm_workarea src, dst; 1816 struct ccp_op op; 1817 int ret; 1818 u8 *save; 1819 1820 if (!ecc->u.mm.operand_1 || 1821 (ecc->u.mm.operand_1_len > CCP_ECC_MODULUS_BYTES)) 1822 return -EINVAL; 1823 1824 if (ecc->function != CCP_ECC_FUNCTION_MINV_384BIT) 1825 if (!ecc->u.mm.operand_2 || 1826 (ecc->u.mm.operand_2_len > CCP_ECC_MODULUS_BYTES)) 1827 return -EINVAL; 1828 1829 if (!ecc->u.mm.result || 1830 (ecc->u.mm.result_len < CCP_ECC_MODULUS_BYTES)) 1831 return -EINVAL; 1832 1833 memset(&op, 0, sizeof(op)); 1834 op.cmd_q = cmd_q; 1835 op.jobid = ccp_gen_jobid(cmd_q->ccp); 1836 1837 /* Concatenate the modulus and the operands. Both the modulus and 1838 * the operands must be in little endian format. Since the input 1839 * is in big endian format it must be converted and placed in a 1840 * fixed length buffer. 1841 */ 1842 ret = ccp_init_dm_workarea(&src, cmd_q, CCP_ECC_SRC_BUF_SIZE, 1843 DMA_TO_DEVICE); 1844 if (ret) 1845 return ret; 1846 1847 /* Save the workarea address since it is updated in order to perform 1848 * the concatenation 1849 */ 1850 save = src.address; 1851 1852 /* Copy the ECC modulus */ 1853 ret = ccp_reverse_set_dm_area(&src, ecc->mod, ecc->mod_len, 1854 CCP_ECC_OPERAND_SIZE, false); 1855 if (ret) 1856 goto e_src; 1857 src.address += CCP_ECC_OPERAND_SIZE; 1858 1859 /* Copy the first operand */ 1860 ret = ccp_reverse_set_dm_area(&src, ecc->u.mm.operand_1, 1861 ecc->u.mm.operand_1_len, 1862 CCP_ECC_OPERAND_SIZE, false); 1863 if (ret) 1864 goto e_src; 1865 src.address += CCP_ECC_OPERAND_SIZE; 1866 1867 if (ecc->function != CCP_ECC_FUNCTION_MINV_384BIT) { 1868 /* Copy the second operand */ 1869 ret = ccp_reverse_set_dm_area(&src, ecc->u.mm.operand_2, 1870 ecc->u.mm.operand_2_len, 1871 CCP_ECC_OPERAND_SIZE, false); 1872 if (ret) 1873 goto e_src; 1874 src.address += CCP_ECC_OPERAND_SIZE; 1875 } 1876 1877 /* Restore the workarea address */ 1878 src.address = save; 1879 1880 /* Prepare the output area for the operation */ 1881 ret = ccp_init_dm_workarea(&dst, cmd_q, CCP_ECC_DST_BUF_SIZE, 1882 DMA_FROM_DEVICE); 1883 if (ret) 1884 goto e_src; 1885 1886 op.soc = 1; 1887 op.src.u.dma.address = src.dma.address; 1888 op.src.u.dma.offset = 0; 1889 op.src.u.dma.length = src.length; 1890 op.dst.u.dma.address = dst.dma.address; 1891 op.dst.u.dma.offset = 0; 1892 op.dst.u.dma.length = dst.length; 1893 1894 op.u.ecc.function = cmd->u.ecc.function; 1895 1896 ret = ccp_perform_ecc(&op); 1897 if (ret) { 1898 cmd->engine_error = cmd_q->cmd_error; 1899 goto e_dst; 1900 } 1901 1902 ecc->ecc_result = le16_to_cpup( 1903 (const __le16 *)(dst.address + CCP_ECC_RESULT_OFFSET)); 1904 if (!(ecc->ecc_result & CCP_ECC_RESULT_SUCCESS)) { 1905 ret = -EIO; 1906 goto e_dst; 1907 } 1908 1909 /* Save the ECC result */ 1910 ccp_reverse_get_dm_area(&dst, ecc->u.mm.result, CCP_ECC_MODULUS_BYTES); 1911 1912 e_dst: 1913 ccp_dm_free(&dst); 1914 1915 e_src: 1916 ccp_dm_free(&src); 1917 1918 return ret; 1919 } 1920 1921 static int ccp_run_ecc_pm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) 1922 { 1923 struct ccp_ecc_engine *ecc = &cmd->u.ecc; 1924 struct ccp_dm_workarea src, dst; 1925 struct ccp_op op; 1926 int ret; 1927 u8 *save; 1928 1929 if (!ecc->u.pm.point_1.x || 1930 (ecc->u.pm.point_1.x_len > CCP_ECC_MODULUS_BYTES) || 1931 !ecc->u.pm.point_1.y || 1932 (ecc->u.pm.point_1.y_len > CCP_ECC_MODULUS_BYTES)) 1933 return -EINVAL; 1934 1935 if (ecc->function == CCP_ECC_FUNCTION_PADD_384BIT) { 1936 if (!ecc->u.pm.point_2.x || 1937 (ecc->u.pm.point_2.x_len > CCP_ECC_MODULUS_BYTES) || 1938 !ecc->u.pm.point_2.y || 1939 (ecc->u.pm.point_2.y_len > CCP_ECC_MODULUS_BYTES)) 1940 return -EINVAL; 1941 } else { 1942 if (!ecc->u.pm.domain_a || 1943 (ecc->u.pm.domain_a_len > CCP_ECC_MODULUS_BYTES)) 1944 return -EINVAL; 1945 1946 if (ecc->function == CCP_ECC_FUNCTION_PMUL_384BIT) 1947 if (!ecc->u.pm.scalar || 1948 (ecc->u.pm.scalar_len > CCP_ECC_MODULUS_BYTES)) 1949 return -EINVAL; 1950 } 1951 1952 if (!ecc->u.pm.result.x || 1953 (ecc->u.pm.result.x_len < CCP_ECC_MODULUS_BYTES) || 1954 !ecc->u.pm.result.y || 1955 (ecc->u.pm.result.y_len < CCP_ECC_MODULUS_BYTES)) 1956 return -EINVAL; 1957 1958 memset(&op, 0, sizeof(op)); 1959 op.cmd_q = cmd_q; 1960 op.jobid = ccp_gen_jobid(cmd_q->ccp); 1961 1962 /* Concatenate the modulus and the operands. Both the modulus and 1963 * the operands must be in little endian format. Since the input 1964 * is in big endian format it must be converted and placed in a 1965 * fixed length buffer. 1966 */ 1967 ret = ccp_init_dm_workarea(&src, cmd_q, CCP_ECC_SRC_BUF_SIZE, 1968 DMA_TO_DEVICE); 1969 if (ret) 1970 return ret; 1971 1972 /* Save the workarea address since it is updated in order to perform 1973 * the concatenation 1974 */ 1975 save = src.address; 1976 1977 /* Copy the ECC modulus */ 1978 ret = ccp_reverse_set_dm_area(&src, ecc->mod, ecc->mod_len, 1979 CCP_ECC_OPERAND_SIZE, false); 1980 if (ret) 1981 goto e_src; 1982 src.address += CCP_ECC_OPERAND_SIZE; 1983 1984 /* Copy the first point X and Y coordinate */ 1985 ret = ccp_reverse_set_dm_area(&src, ecc->u.pm.point_1.x, 1986 ecc->u.pm.point_1.x_len, 1987 CCP_ECC_OPERAND_SIZE, false); 1988 if (ret) 1989 goto e_src; 1990 src.address += CCP_ECC_OPERAND_SIZE; 1991 ret = ccp_reverse_set_dm_area(&src, ecc->u.pm.point_1.y, 1992 ecc->u.pm.point_1.y_len, 1993 CCP_ECC_OPERAND_SIZE, false); 1994 if (ret) 1995 goto e_src; 1996 src.address += CCP_ECC_OPERAND_SIZE; 1997 1998 /* Set the first point Z coordianate to 1 */ 1999 *src.address = 0x01; 2000 src.address += CCP_ECC_OPERAND_SIZE; 2001 2002 if (ecc->function == CCP_ECC_FUNCTION_PADD_384BIT) { 2003 /* Copy the second point X and Y coordinate */ 2004 ret = ccp_reverse_set_dm_area(&src, ecc->u.pm.point_2.x, 2005 ecc->u.pm.point_2.x_len, 2006 CCP_ECC_OPERAND_SIZE, false); 2007 if (ret) 2008 goto e_src; 2009 src.address += CCP_ECC_OPERAND_SIZE; 2010 ret = ccp_reverse_set_dm_area(&src, ecc->u.pm.point_2.y, 2011 ecc->u.pm.point_2.y_len, 2012 CCP_ECC_OPERAND_SIZE, false); 2013 if (ret) 2014 goto e_src; 2015 src.address += CCP_ECC_OPERAND_SIZE; 2016 2017 /* Set the second point Z coordianate to 1 */ 2018 *src.address = 0x01; 2019 src.address += CCP_ECC_OPERAND_SIZE; 2020 } else { 2021 /* Copy the Domain "a" parameter */ 2022 ret = ccp_reverse_set_dm_area(&src, ecc->u.pm.domain_a, 2023 ecc->u.pm.domain_a_len, 2024 CCP_ECC_OPERAND_SIZE, false); 2025 if (ret) 2026 goto e_src; 2027 src.address += CCP_ECC_OPERAND_SIZE; 2028 2029 if (ecc->function == CCP_ECC_FUNCTION_PMUL_384BIT) { 2030 /* Copy the scalar value */ 2031 ret = ccp_reverse_set_dm_area(&src, ecc->u.pm.scalar, 2032 ecc->u.pm.scalar_len, 2033 CCP_ECC_OPERAND_SIZE, 2034 false); 2035 if (ret) 2036 goto e_src; 2037 src.address += CCP_ECC_OPERAND_SIZE; 2038 } 2039 } 2040 2041 /* Restore the workarea address */ 2042 src.address = save; 2043 2044 /* Prepare the output area for the operation */ 2045 ret = ccp_init_dm_workarea(&dst, cmd_q, CCP_ECC_DST_BUF_SIZE, 2046 DMA_FROM_DEVICE); 2047 if (ret) 2048 goto e_src; 2049 2050 op.soc = 1; 2051 op.src.u.dma.address = src.dma.address; 2052 op.src.u.dma.offset = 0; 2053 op.src.u.dma.length = src.length; 2054 op.dst.u.dma.address = dst.dma.address; 2055 op.dst.u.dma.offset = 0; 2056 op.dst.u.dma.length = dst.length; 2057 2058 op.u.ecc.function = cmd->u.ecc.function; 2059 2060 ret = ccp_perform_ecc(&op); 2061 if (ret) { 2062 cmd->engine_error = cmd_q->cmd_error; 2063 goto e_dst; 2064 } 2065 2066 ecc->ecc_result = le16_to_cpup( 2067 (const __le16 *)(dst.address + CCP_ECC_RESULT_OFFSET)); 2068 if (!(ecc->ecc_result & CCP_ECC_RESULT_SUCCESS)) { 2069 ret = -EIO; 2070 goto e_dst; 2071 } 2072 2073 /* Save the workarea address since it is updated as we walk through 2074 * to copy the point math result 2075 */ 2076 save = dst.address; 2077 2078 /* Save the ECC result X and Y coordinates */ 2079 ccp_reverse_get_dm_area(&dst, ecc->u.pm.result.x, 2080 CCP_ECC_MODULUS_BYTES); 2081 dst.address += CCP_ECC_OUTPUT_SIZE; 2082 ccp_reverse_get_dm_area(&dst, ecc->u.pm.result.y, 2083 CCP_ECC_MODULUS_BYTES); 2084 dst.address += CCP_ECC_OUTPUT_SIZE; 2085 2086 /* Restore the workarea address */ 2087 dst.address = save; 2088 2089 e_dst: 2090 ccp_dm_free(&dst); 2091 2092 e_src: 2093 ccp_dm_free(&src); 2094 2095 return ret; 2096 } 2097 2098 static int ccp_run_ecc_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) 2099 { 2100 struct ccp_ecc_engine *ecc = &cmd->u.ecc; 2101 2102 ecc->ecc_result = 0; 2103 2104 if (!ecc->mod || 2105 (ecc->mod_len > CCP_ECC_MODULUS_BYTES)) 2106 return -EINVAL; 2107 2108 switch (ecc->function) { 2109 case CCP_ECC_FUNCTION_MMUL_384BIT: 2110 case CCP_ECC_FUNCTION_MADD_384BIT: 2111 case CCP_ECC_FUNCTION_MINV_384BIT: 2112 return ccp_run_ecc_mm_cmd(cmd_q, cmd); 2113 2114 case CCP_ECC_FUNCTION_PADD_384BIT: 2115 case CCP_ECC_FUNCTION_PMUL_384BIT: 2116 case CCP_ECC_FUNCTION_PDBL_384BIT: 2117 return ccp_run_ecc_pm_cmd(cmd_q, cmd); 2118 2119 default: 2120 return -EINVAL; 2121 } 2122 } 2123 2124 int ccp_run_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) 2125 { 2126 int ret; 2127 2128 cmd->engine_error = 0; 2129 cmd_q->cmd_error = 0; 2130 cmd_q->int_rcvd = 0; 2131 cmd_q->free_slots = CMD_Q_DEPTH(ioread32(cmd_q->reg_status)); 2132 2133 switch (cmd->engine) { 2134 case CCP_ENGINE_AES: 2135 ret = ccp_run_aes_cmd(cmd_q, cmd); 2136 break; 2137 case CCP_ENGINE_XTS_AES_128: 2138 ret = ccp_run_xts_aes_cmd(cmd_q, cmd); 2139 break; 2140 case CCP_ENGINE_SHA: 2141 ret = ccp_run_sha_cmd(cmd_q, cmd); 2142 break; 2143 case CCP_ENGINE_RSA: 2144 ret = ccp_run_rsa_cmd(cmd_q, cmd); 2145 break; 2146 case CCP_ENGINE_PASSTHRU: 2147 ret = ccp_run_passthru_cmd(cmd_q, cmd); 2148 break; 2149 case CCP_ENGINE_ECC: 2150 ret = ccp_run_ecc_cmd(cmd_q, cmd); 2151 break; 2152 default: 2153 ret = -EINVAL; 2154 } 2155 2156 return ret; 2157 } 2158