1 /* 2 * AMD Cryptographic Coprocessor (CCP) driver 3 * 4 * Copyright (C) 2013 Advanced Micro Devices, Inc. 5 * 6 * Author: Tom Lendacky <thomas.lendacky@amd.com> 7 * 8 * This program is free software; you can redistribute it and/or modify 9 * it under the terms of the GNU General Public License version 2 as 10 * published by the Free Software Foundation. 11 */ 12 13 #include <linux/module.h> 14 #include <linux/kernel.h> 15 #include <linux/pci.h> 16 #include <linux/pci_ids.h> 17 #include <linux/kthread.h> 18 #include <linux/sched.h> 19 #include <linux/interrupt.h> 20 #include <linux/spinlock.h> 21 #include <linux/mutex.h> 22 #include <linux/delay.h> 23 #include <linux/ccp.h> 24 #include <linux/scatterlist.h> 25 #include <crypto/scatterwalk.h> 26 #include <crypto/sha.h> 27 28 #include "ccp-dev.h" 29 30 enum ccp_memtype { 31 CCP_MEMTYPE_SYSTEM = 0, 32 CCP_MEMTYPE_KSB, 33 CCP_MEMTYPE_LOCAL, 34 CCP_MEMTYPE__LAST, 35 }; 36 37 struct ccp_dma_info { 38 dma_addr_t address; 39 unsigned int offset; 40 unsigned int length; 41 enum dma_data_direction dir; 42 }; 43 44 struct ccp_dm_workarea { 45 struct device *dev; 46 struct dma_pool *dma_pool; 47 unsigned int length; 48 49 u8 *address; 50 struct ccp_dma_info dma; 51 }; 52 53 struct ccp_sg_workarea { 54 struct scatterlist *sg; 55 int nents; 56 57 struct scatterlist *dma_sg; 58 struct device *dma_dev; 59 unsigned int dma_count; 60 enum dma_data_direction dma_dir; 61 62 unsigned int sg_used; 63 64 u64 bytes_left; 65 }; 66 67 struct ccp_data { 68 struct ccp_sg_workarea sg_wa; 69 struct ccp_dm_workarea dm_wa; 70 }; 71 72 struct ccp_mem { 73 enum ccp_memtype type; 74 union { 75 struct ccp_dma_info dma; 76 u32 ksb; 77 } u; 78 }; 79 80 struct ccp_aes_op { 81 enum ccp_aes_type type; 82 enum ccp_aes_mode mode; 83 enum ccp_aes_action action; 84 }; 85 86 struct ccp_xts_aes_op { 87 enum ccp_aes_action action; 88 enum ccp_xts_aes_unit_size unit_size; 89 }; 90 91 struct ccp_sha_op { 92 enum ccp_sha_type type; 93 u64 msg_bits; 94 }; 95 96 struct ccp_rsa_op { 97 u32 mod_size; 98 u32 input_len; 99 }; 100 101 struct ccp_passthru_op { 102 enum ccp_passthru_bitwise bit_mod; 103 enum ccp_passthru_byteswap byte_swap; 104 }; 105 106 struct ccp_ecc_op { 107 enum ccp_ecc_function function; 108 }; 109 110 struct ccp_op { 111 struct ccp_cmd_queue *cmd_q; 112 113 u32 jobid; 114 u32 ioc; 115 u32 soc; 116 u32 ksb_key; 117 u32 ksb_ctx; 118 u32 init; 119 u32 eom; 120 121 struct ccp_mem src; 122 struct ccp_mem dst; 123 124 union { 125 struct ccp_aes_op aes; 126 struct ccp_xts_aes_op xts; 127 struct ccp_sha_op sha; 128 struct ccp_rsa_op rsa; 129 struct ccp_passthru_op passthru; 130 struct ccp_ecc_op ecc; 131 } u; 132 }; 133 134 /* SHA initial context values */ 135 static const __be32 ccp_sha1_init[CCP_SHA_CTXSIZE / sizeof(__be32)] = { 136 cpu_to_be32(SHA1_H0), cpu_to_be32(SHA1_H1), 137 cpu_to_be32(SHA1_H2), cpu_to_be32(SHA1_H3), 138 cpu_to_be32(SHA1_H4), 0, 0, 0, 139 }; 140 141 static const __be32 ccp_sha224_init[CCP_SHA_CTXSIZE / sizeof(__be32)] = { 142 cpu_to_be32(SHA224_H0), cpu_to_be32(SHA224_H1), 143 cpu_to_be32(SHA224_H2), cpu_to_be32(SHA224_H3), 144 cpu_to_be32(SHA224_H4), cpu_to_be32(SHA224_H5), 145 cpu_to_be32(SHA224_H6), cpu_to_be32(SHA224_H7), 146 }; 147 148 static const __be32 ccp_sha256_init[CCP_SHA_CTXSIZE / sizeof(__be32)] = { 149 cpu_to_be32(SHA256_H0), cpu_to_be32(SHA256_H1), 150 cpu_to_be32(SHA256_H2), cpu_to_be32(SHA256_H3), 151 cpu_to_be32(SHA256_H4), cpu_to_be32(SHA256_H5), 152 cpu_to_be32(SHA256_H6), cpu_to_be32(SHA256_H7), 153 }; 154 155 /* The CCP cannot perform zero-length sha operations so the caller 156 * is required to buffer data for the final operation. However, a 157 * sha operation for a message with a total length of zero is valid 158 * so known values are required to supply the result. 159 */ 160 static const u8 ccp_sha1_zero[CCP_SHA_CTXSIZE] = { 161 0xda, 0x39, 0xa3, 0xee, 0x5e, 0x6b, 0x4b, 0x0d, 162 0x32, 0x55, 0xbf, 0xef, 0x95, 0x60, 0x18, 0x90, 163 0xaf, 0xd8, 0x07, 0x09, 0x00, 0x00, 0x00, 0x00, 164 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 165 }; 166 167 static const u8 ccp_sha224_zero[CCP_SHA_CTXSIZE] = { 168 0xd1, 0x4a, 0x02, 0x8c, 0x2a, 0x3a, 0x2b, 0xc9, 169 0x47, 0x61, 0x02, 0xbb, 0x28, 0x82, 0x34, 0xc4, 170 0x15, 0xa2, 0xb0, 0x1f, 0x82, 0x8e, 0xa6, 0x2a, 171 0xc5, 0xb3, 0xe4, 0x2f, 0x00, 0x00, 0x00, 0x00, 172 }; 173 174 static const u8 ccp_sha256_zero[CCP_SHA_CTXSIZE] = { 175 0xe3, 0xb0, 0xc4, 0x42, 0x98, 0xfc, 0x1c, 0x14, 176 0x9a, 0xfb, 0xf4, 0xc8, 0x99, 0x6f, 0xb9, 0x24, 177 0x27, 0xae, 0x41, 0xe4, 0x64, 0x9b, 0x93, 0x4c, 178 0xa4, 0x95, 0x99, 0x1b, 0x78, 0x52, 0xb8, 0x55, 179 }; 180 181 static u32 ccp_addr_lo(struct ccp_dma_info *info) 182 { 183 return lower_32_bits(info->address + info->offset); 184 } 185 186 static u32 ccp_addr_hi(struct ccp_dma_info *info) 187 { 188 return upper_32_bits(info->address + info->offset) & 0x0000ffff; 189 } 190 191 static int ccp_do_cmd(struct ccp_op *op, u32 *cr, unsigned int cr_count) 192 { 193 struct ccp_cmd_queue *cmd_q = op->cmd_q; 194 struct ccp_device *ccp = cmd_q->ccp; 195 void __iomem *cr_addr; 196 u32 cr0, cmd; 197 unsigned int i; 198 int ret = 0; 199 200 /* We could read a status register to see how many free slots 201 * are actually available, but reading that register resets it 202 * and you could lose some error information. 203 */ 204 cmd_q->free_slots--; 205 206 cr0 = (cmd_q->id << REQ0_CMD_Q_SHIFT) 207 | (op->jobid << REQ0_JOBID_SHIFT) 208 | REQ0_WAIT_FOR_WRITE; 209 210 if (op->soc) 211 cr0 |= REQ0_STOP_ON_COMPLETE 212 | REQ0_INT_ON_COMPLETE; 213 214 if (op->ioc || !cmd_q->free_slots) 215 cr0 |= REQ0_INT_ON_COMPLETE; 216 217 /* Start at CMD_REQ1 */ 218 cr_addr = ccp->io_regs + CMD_REQ0 + CMD_REQ_INCR; 219 220 mutex_lock(&ccp->req_mutex); 221 222 /* Write CMD_REQ1 through CMD_REQx first */ 223 for (i = 0; i < cr_count; i++, cr_addr += CMD_REQ_INCR) 224 iowrite32(*(cr + i), cr_addr); 225 226 /* Tell the CCP to start */ 227 wmb(); 228 iowrite32(cr0, ccp->io_regs + CMD_REQ0); 229 230 mutex_unlock(&ccp->req_mutex); 231 232 if (cr0 & REQ0_INT_ON_COMPLETE) { 233 /* Wait for the job to complete */ 234 ret = wait_event_interruptible(cmd_q->int_queue, 235 cmd_q->int_rcvd); 236 if (ret || cmd_q->cmd_error) { 237 /* On error delete all related jobs from the queue */ 238 cmd = (cmd_q->id << DEL_Q_ID_SHIFT) 239 | op->jobid; 240 241 iowrite32(cmd, ccp->io_regs + DEL_CMD_Q_JOB); 242 243 if (!ret) 244 ret = -EIO; 245 } else if (op->soc) { 246 /* Delete just head job from the queue on SoC */ 247 cmd = DEL_Q_ACTIVE 248 | (cmd_q->id << DEL_Q_ID_SHIFT) 249 | op->jobid; 250 251 iowrite32(cmd, ccp->io_regs + DEL_CMD_Q_JOB); 252 } 253 254 cmd_q->free_slots = CMD_Q_DEPTH(cmd_q->q_status); 255 256 cmd_q->int_rcvd = 0; 257 } 258 259 return ret; 260 } 261 262 static int ccp_perform_aes(struct ccp_op *op) 263 { 264 u32 cr[6]; 265 266 /* Fill out the register contents for REQ1 through REQ6 */ 267 cr[0] = (CCP_ENGINE_AES << REQ1_ENGINE_SHIFT) 268 | (op->u.aes.type << REQ1_AES_TYPE_SHIFT) 269 | (op->u.aes.mode << REQ1_AES_MODE_SHIFT) 270 | (op->u.aes.action << REQ1_AES_ACTION_SHIFT) 271 | (op->ksb_key << REQ1_KEY_KSB_SHIFT); 272 cr[1] = op->src.u.dma.length - 1; 273 cr[2] = ccp_addr_lo(&op->src.u.dma); 274 cr[3] = (op->ksb_ctx << REQ4_KSB_SHIFT) 275 | (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT) 276 | ccp_addr_hi(&op->src.u.dma); 277 cr[4] = ccp_addr_lo(&op->dst.u.dma); 278 cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT) 279 | ccp_addr_hi(&op->dst.u.dma); 280 281 if (op->u.aes.mode == CCP_AES_MODE_CFB) 282 cr[0] |= ((0x7f) << REQ1_AES_CFB_SIZE_SHIFT); 283 284 if (op->eom) 285 cr[0] |= REQ1_EOM; 286 287 if (op->init) 288 cr[0] |= REQ1_INIT; 289 290 return ccp_do_cmd(op, cr, ARRAY_SIZE(cr)); 291 } 292 293 static int ccp_perform_xts_aes(struct ccp_op *op) 294 { 295 u32 cr[6]; 296 297 /* Fill out the register contents for REQ1 through REQ6 */ 298 cr[0] = (CCP_ENGINE_XTS_AES_128 << REQ1_ENGINE_SHIFT) 299 | (op->u.xts.action << REQ1_AES_ACTION_SHIFT) 300 | (op->u.xts.unit_size << REQ1_XTS_AES_SIZE_SHIFT) 301 | (op->ksb_key << REQ1_KEY_KSB_SHIFT); 302 cr[1] = op->src.u.dma.length - 1; 303 cr[2] = ccp_addr_lo(&op->src.u.dma); 304 cr[3] = (op->ksb_ctx << REQ4_KSB_SHIFT) 305 | (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT) 306 | ccp_addr_hi(&op->src.u.dma); 307 cr[4] = ccp_addr_lo(&op->dst.u.dma); 308 cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT) 309 | ccp_addr_hi(&op->dst.u.dma); 310 311 if (op->eom) 312 cr[0] |= REQ1_EOM; 313 314 if (op->init) 315 cr[0] |= REQ1_INIT; 316 317 return ccp_do_cmd(op, cr, ARRAY_SIZE(cr)); 318 } 319 320 static int ccp_perform_sha(struct ccp_op *op) 321 { 322 u32 cr[6]; 323 324 /* Fill out the register contents for REQ1 through REQ6 */ 325 cr[0] = (CCP_ENGINE_SHA << REQ1_ENGINE_SHIFT) 326 | (op->u.sha.type << REQ1_SHA_TYPE_SHIFT) 327 | REQ1_INIT; 328 cr[1] = op->src.u.dma.length - 1; 329 cr[2] = ccp_addr_lo(&op->src.u.dma); 330 cr[3] = (op->ksb_ctx << REQ4_KSB_SHIFT) 331 | (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT) 332 | ccp_addr_hi(&op->src.u.dma); 333 334 if (op->eom) { 335 cr[0] |= REQ1_EOM; 336 cr[4] = lower_32_bits(op->u.sha.msg_bits); 337 cr[5] = upper_32_bits(op->u.sha.msg_bits); 338 } else { 339 cr[4] = 0; 340 cr[5] = 0; 341 } 342 343 return ccp_do_cmd(op, cr, ARRAY_SIZE(cr)); 344 } 345 346 static int ccp_perform_rsa(struct ccp_op *op) 347 { 348 u32 cr[6]; 349 350 /* Fill out the register contents for REQ1 through REQ6 */ 351 cr[0] = (CCP_ENGINE_RSA << REQ1_ENGINE_SHIFT) 352 | (op->u.rsa.mod_size << REQ1_RSA_MOD_SIZE_SHIFT) 353 | (op->ksb_key << REQ1_KEY_KSB_SHIFT) 354 | REQ1_EOM; 355 cr[1] = op->u.rsa.input_len - 1; 356 cr[2] = ccp_addr_lo(&op->src.u.dma); 357 cr[3] = (op->ksb_ctx << REQ4_KSB_SHIFT) 358 | (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT) 359 | ccp_addr_hi(&op->src.u.dma); 360 cr[4] = ccp_addr_lo(&op->dst.u.dma); 361 cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT) 362 | ccp_addr_hi(&op->dst.u.dma); 363 364 return ccp_do_cmd(op, cr, ARRAY_SIZE(cr)); 365 } 366 367 static int ccp_perform_passthru(struct ccp_op *op) 368 { 369 u32 cr[6]; 370 371 /* Fill out the register contents for REQ1 through REQ6 */ 372 cr[0] = (CCP_ENGINE_PASSTHRU << REQ1_ENGINE_SHIFT) 373 | (op->u.passthru.bit_mod << REQ1_PT_BW_SHIFT) 374 | (op->u.passthru.byte_swap << REQ1_PT_BS_SHIFT); 375 376 if (op->src.type == CCP_MEMTYPE_SYSTEM) 377 cr[1] = op->src.u.dma.length - 1; 378 else 379 cr[1] = op->dst.u.dma.length - 1; 380 381 if (op->src.type == CCP_MEMTYPE_SYSTEM) { 382 cr[2] = ccp_addr_lo(&op->src.u.dma); 383 cr[3] = (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT) 384 | ccp_addr_hi(&op->src.u.dma); 385 386 if (op->u.passthru.bit_mod != CCP_PASSTHRU_BITWISE_NOOP) 387 cr[3] |= (op->ksb_key << REQ4_KSB_SHIFT); 388 } else { 389 cr[2] = op->src.u.ksb * CCP_KSB_BYTES; 390 cr[3] = (CCP_MEMTYPE_KSB << REQ4_MEMTYPE_SHIFT); 391 } 392 393 if (op->dst.type == CCP_MEMTYPE_SYSTEM) { 394 cr[4] = ccp_addr_lo(&op->dst.u.dma); 395 cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT) 396 | ccp_addr_hi(&op->dst.u.dma); 397 } else { 398 cr[4] = op->dst.u.ksb * CCP_KSB_BYTES; 399 cr[5] = (CCP_MEMTYPE_KSB << REQ6_MEMTYPE_SHIFT); 400 } 401 402 if (op->eom) 403 cr[0] |= REQ1_EOM; 404 405 return ccp_do_cmd(op, cr, ARRAY_SIZE(cr)); 406 } 407 408 static int ccp_perform_ecc(struct ccp_op *op) 409 { 410 u32 cr[6]; 411 412 /* Fill out the register contents for REQ1 through REQ6 */ 413 cr[0] = REQ1_ECC_AFFINE_CONVERT 414 | (CCP_ENGINE_ECC << REQ1_ENGINE_SHIFT) 415 | (op->u.ecc.function << REQ1_ECC_FUNCTION_SHIFT) 416 | REQ1_EOM; 417 cr[1] = op->src.u.dma.length - 1; 418 cr[2] = ccp_addr_lo(&op->src.u.dma); 419 cr[3] = (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT) 420 | ccp_addr_hi(&op->src.u.dma); 421 cr[4] = ccp_addr_lo(&op->dst.u.dma); 422 cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT) 423 | ccp_addr_hi(&op->dst.u.dma); 424 425 return ccp_do_cmd(op, cr, ARRAY_SIZE(cr)); 426 } 427 428 static u32 ccp_alloc_ksb(struct ccp_device *ccp, unsigned int count) 429 { 430 int start; 431 432 for (;;) { 433 mutex_lock(&ccp->ksb_mutex); 434 435 start = (u32)bitmap_find_next_zero_area(ccp->ksb, 436 ccp->ksb_count, 437 ccp->ksb_start, 438 count, 0); 439 if (start <= ccp->ksb_count) { 440 bitmap_set(ccp->ksb, start, count); 441 442 mutex_unlock(&ccp->ksb_mutex); 443 break; 444 } 445 446 ccp->ksb_avail = 0; 447 448 mutex_unlock(&ccp->ksb_mutex); 449 450 /* Wait for KSB entries to become available */ 451 if (wait_event_interruptible(ccp->ksb_queue, ccp->ksb_avail)) 452 return 0; 453 } 454 455 return KSB_START + start; 456 } 457 458 static void ccp_free_ksb(struct ccp_device *ccp, unsigned int start, 459 unsigned int count) 460 { 461 if (!start) 462 return; 463 464 mutex_lock(&ccp->ksb_mutex); 465 466 bitmap_clear(ccp->ksb, start - KSB_START, count); 467 468 ccp->ksb_avail = 1; 469 470 mutex_unlock(&ccp->ksb_mutex); 471 472 wake_up_interruptible_all(&ccp->ksb_queue); 473 } 474 475 static u32 ccp_gen_jobid(struct ccp_device *ccp) 476 { 477 return atomic_inc_return(&ccp->current_id) & CCP_JOBID_MASK; 478 } 479 480 static void ccp_sg_free(struct ccp_sg_workarea *wa) 481 { 482 if (wa->dma_count) 483 dma_unmap_sg(wa->dma_dev, wa->dma_sg, wa->nents, wa->dma_dir); 484 485 wa->dma_count = 0; 486 } 487 488 static int ccp_init_sg_workarea(struct ccp_sg_workarea *wa, struct device *dev, 489 struct scatterlist *sg, u64 len, 490 enum dma_data_direction dma_dir) 491 { 492 memset(wa, 0, sizeof(*wa)); 493 494 wa->sg = sg; 495 if (!sg) 496 return 0; 497 498 wa->nents = sg_nents_for_len(sg, len); 499 if (wa->nents < 0) 500 return wa->nents; 501 502 wa->bytes_left = len; 503 wa->sg_used = 0; 504 505 if (len == 0) 506 return 0; 507 508 if (dma_dir == DMA_NONE) 509 return 0; 510 511 wa->dma_sg = sg; 512 wa->dma_dev = dev; 513 wa->dma_dir = dma_dir; 514 wa->dma_count = dma_map_sg(dev, sg, wa->nents, dma_dir); 515 if (!wa->dma_count) 516 return -ENOMEM; 517 518 return 0; 519 } 520 521 static void ccp_update_sg_workarea(struct ccp_sg_workarea *wa, unsigned int len) 522 { 523 unsigned int nbytes = min_t(u64, len, wa->bytes_left); 524 525 if (!wa->sg) 526 return; 527 528 wa->sg_used += nbytes; 529 wa->bytes_left -= nbytes; 530 if (wa->sg_used == wa->sg->length) { 531 wa->sg = sg_next(wa->sg); 532 wa->sg_used = 0; 533 } 534 } 535 536 static void ccp_dm_free(struct ccp_dm_workarea *wa) 537 { 538 if (wa->length <= CCP_DMAPOOL_MAX_SIZE) { 539 if (wa->address) 540 dma_pool_free(wa->dma_pool, wa->address, 541 wa->dma.address); 542 } else { 543 if (wa->dma.address) 544 dma_unmap_single(wa->dev, wa->dma.address, wa->length, 545 wa->dma.dir); 546 kfree(wa->address); 547 } 548 549 wa->address = NULL; 550 wa->dma.address = 0; 551 } 552 553 static int ccp_init_dm_workarea(struct ccp_dm_workarea *wa, 554 struct ccp_cmd_queue *cmd_q, 555 unsigned int len, 556 enum dma_data_direction dir) 557 { 558 memset(wa, 0, sizeof(*wa)); 559 560 if (!len) 561 return 0; 562 563 wa->dev = cmd_q->ccp->dev; 564 wa->length = len; 565 566 if (len <= CCP_DMAPOOL_MAX_SIZE) { 567 wa->dma_pool = cmd_q->dma_pool; 568 569 wa->address = dma_pool_alloc(wa->dma_pool, GFP_KERNEL, 570 &wa->dma.address); 571 if (!wa->address) 572 return -ENOMEM; 573 574 wa->dma.length = CCP_DMAPOOL_MAX_SIZE; 575 576 memset(wa->address, 0, CCP_DMAPOOL_MAX_SIZE); 577 } else { 578 wa->address = kzalloc(len, GFP_KERNEL); 579 if (!wa->address) 580 return -ENOMEM; 581 582 wa->dma.address = dma_map_single(wa->dev, wa->address, len, 583 dir); 584 if (!wa->dma.address) 585 return -ENOMEM; 586 587 wa->dma.length = len; 588 } 589 wa->dma.dir = dir; 590 591 return 0; 592 } 593 594 static void ccp_set_dm_area(struct ccp_dm_workarea *wa, unsigned int wa_offset, 595 struct scatterlist *sg, unsigned int sg_offset, 596 unsigned int len) 597 { 598 WARN_ON(!wa->address); 599 600 scatterwalk_map_and_copy(wa->address + wa_offset, sg, sg_offset, len, 601 0); 602 } 603 604 static void ccp_get_dm_area(struct ccp_dm_workarea *wa, unsigned int wa_offset, 605 struct scatterlist *sg, unsigned int sg_offset, 606 unsigned int len) 607 { 608 WARN_ON(!wa->address); 609 610 scatterwalk_map_and_copy(wa->address + wa_offset, sg, sg_offset, len, 611 1); 612 } 613 614 static void ccp_reverse_set_dm_area(struct ccp_dm_workarea *wa, 615 struct scatterlist *sg, 616 unsigned int len, unsigned int se_len, 617 bool sign_extend) 618 { 619 unsigned int nbytes, sg_offset, dm_offset, ksb_len, i; 620 u8 buffer[CCP_REVERSE_BUF_SIZE]; 621 622 BUG_ON(se_len > sizeof(buffer)); 623 624 sg_offset = len; 625 dm_offset = 0; 626 nbytes = len; 627 while (nbytes) { 628 ksb_len = min_t(unsigned int, nbytes, se_len); 629 sg_offset -= ksb_len; 630 631 scatterwalk_map_and_copy(buffer, sg, sg_offset, ksb_len, 0); 632 for (i = 0; i < ksb_len; i++) 633 wa->address[dm_offset + i] = buffer[ksb_len - i - 1]; 634 635 dm_offset += ksb_len; 636 nbytes -= ksb_len; 637 638 if ((ksb_len != se_len) && sign_extend) { 639 /* Must sign-extend to nearest sign-extend length */ 640 if (wa->address[dm_offset - 1] & 0x80) 641 memset(wa->address + dm_offset, 0xff, 642 se_len - ksb_len); 643 } 644 } 645 } 646 647 static void ccp_reverse_get_dm_area(struct ccp_dm_workarea *wa, 648 struct scatterlist *sg, 649 unsigned int len) 650 { 651 unsigned int nbytes, sg_offset, dm_offset, ksb_len, i; 652 u8 buffer[CCP_REVERSE_BUF_SIZE]; 653 654 sg_offset = 0; 655 dm_offset = len; 656 nbytes = len; 657 while (nbytes) { 658 ksb_len = min_t(unsigned int, nbytes, sizeof(buffer)); 659 dm_offset -= ksb_len; 660 661 for (i = 0; i < ksb_len; i++) 662 buffer[ksb_len - i - 1] = wa->address[dm_offset + i]; 663 scatterwalk_map_and_copy(buffer, sg, sg_offset, ksb_len, 1); 664 665 sg_offset += ksb_len; 666 nbytes -= ksb_len; 667 } 668 } 669 670 static void ccp_free_data(struct ccp_data *data, struct ccp_cmd_queue *cmd_q) 671 { 672 ccp_dm_free(&data->dm_wa); 673 ccp_sg_free(&data->sg_wa); 674 } 675 676 static int ccp_init_data(struct ccp_data *data, struct ccp_cmd_queue *cmd_q, 677 struct scatterlist *sg, u64 sg_len, 678 unsigned int dm_len, 679 enum dma_data_direction dir) 680 { 681 int ret; 682 683 memset(data, 0, sizeof(*data)); 684 685 ret = ccp_init_sg_workarea(&data->sg_wa, cmd_q->ccp->dev, sg, sg_len, 686 dir); 687 if (ret) 688 goto e_err; 689 690 ret = ccp_init_dm_workarea(&data->dm_wa, cmd_q, dm_len, dir); 691 if (ret) 692 goto e_err; 693 694 return 0; 695 696 e_err: 697 ccp_free_data(data, cmd_q); 698 699 return ret; 700 } 701 702 static unsigned int ccp_queue_buf(struct ccp_data *data, unsigned int from) 703 { 704 struct ccp_sg_workarea *sg_wa = &data->sg_wa; 705 struct ccp_dm_workarea *dm_wa = &data->dm_wa; 706 unsigned int buf_count, nbytes; 707 708 /* Clear the buffer if setting it */ 709 if (!from) 710 memset(dm_wa->address, 0, dm_wa->length); 711 712 if (!sg_wa->sg) 713 return 0; 714 715 /* Perform the copy operation 716 * nbytes will always be <= UINT_MAX because dm_wa->length is 717 * an unsigned int 718 */ 719 nbytes = min_t(u64, sg_wa->bytes_left, dm_wa->length); 720 scatterwalk_map_and_copy(dm_wa->address, sg_wa->sg, sg_wa->sg_used, 721 nbytes, from); 722 723 /* Update the structures and generate the count */ 724 buf_count = 0; 725 while (sg_wa->bytes_left && (buf_count < dm_wa->length)) { 726 nbytes = min(sg_wa->sg->length - sg_wa->sg_used, 727 dm_wa->length - buf_count); 728 nbytes = min_t(u64, sg_wa->bytes_left, nbytes); 729 730 buf_count += nbytes; 731 ccp_update_sg_workarea(sg_wa, nbytes); 732 } 733 734 return buf_count; 735 } 736 737 static unsigned int ccp_fill_queue_buf(struct ccp_data *data) 738 { 739 return ccp_queue_buf(data, 0); 740 } 741 742 static unsigned int ccp_empty_queue_buf(struct ccp_data *data) 743 { 744 return ccp_queue_buf(data, 1); 745 } 746 747 static void ccp_prepare_data(struct ccp_data *src, struct ccp_data *dst, 748 struct ccp_op *op, unsigned int block_size, 749 bool blocksize_op) 750 { 751 unsigned int sg_src_len, sg_dst_len, op_len; 752 753 /* The CCP can only DMA from/to one address each per operation. This 754 * requires that we find the smallest DMA area between the source 755 * and destination. The resulting len values will always be <= UINT_MAX 756 * because the dma length is an unsigned int. 757 */ 758 sg_src_len = sg_dma_len(src->sg_wa.sg) - src->sg_wa.sg_used; 759 sg_src_len = min_t(u64, src->sg_wa.bytes_left, sg_src_len); 760 761 if (dst) { 762 sg_dst_len = sg_dma_len(dst->sg_wa.sg) - dst->sg_wa.sg_used; 763 sg_dst_len = min_t(u64, src->sg_wa.bytes_left, sg_dst_len); 764 op_len = min(sg_src_len, sg_dst_len); 765 } else { 766 op_len = sg_src_len; 767 } 768 769 /* The data operation length will be at least block_size in length 770 * or the smaller of available sg room remaining for the source or 771 * the destination 772 */ 773 op_len = max(op_len, block_size); 774 775 /* Unless we have to buffer data, there's no reason to wait */ 776 op->soc = 0; 777 778 if (sg_src_len < block_size) { 779 /* Not enough data in the sg element, so it 780 * needs to be buffered into a blocksize chunk 781 */ 782 int cp_len = ccp_fill_queue_buf(src); 783 784 op->soc = 1; 785 op->src.u.dma.address = src->dm_wa.dma.address; 786 op->src.u.dma.offset = 0; 787 op->src.u.dma.length = (blocksize_op) ? block_size : cp_len; 788 } else { 789 /* Enough data in the sg element, but we need to 790 * adjust for any previously copied data 791 */ 792 op->src.u.dma.address = sg_dma_address(src->sg_wa.sg); 793 op->src.u.dma.offset = src->sg_wa.sg_used; 794 op->src.u.dma.length = op_len & ~(block_size - 1); 795 796 ccp_update_sg_workarea(&src->sg_wa, op->src.u.dma.length); 797 } 798 799 if (dst) { 800 if (sg_dst_len < block_size) { 801 /* Not enough room in the sg element or we're on the 802 * last piece of data (when using padding), so the 803 * output needs to be buffered into a blocksize chunk 804 */ 805 op->soc = 1; 806 op->dst.u.dma.address = dst->dm_wa.dma.address; 807 op->dst.u.dma.offset = 0; 808 op->dst.u.dma.length = op->src.u.dma.length; 809 } else { 810 /* Enough room in the sg element, but we need to 811 * adjust for any previously used area 812 */ 813 op->dst.u.dma.address = sg_dma_address(dst->sg_wa.sg); 814 op->dst.u.dma.offset = dst->sg_wa.sg_used; 815 op->dst.u.dma.length = op->src.u.dma.length; 816 } 817 } 818 } 819 820 static void ccp_process_data(struct ccp_data *src, struct ccp_data *dst, 821 struct ccp_op *op) 822 { 823 op->init = 0; 824 825 if (dst) { 826 if (op->dst.u.dma.address == dst->dm_wa.dma.address) 827 ccp_empty_queue_buf(dst); 828 else 829 ccp_update_sg_workarea(&dst->sg_wa, 830 op->dst.u.dma.length); 831 } 832 } 833 834 static int ccp_copy_to_from_ksb(struct ccp_cmd_queue *cmd_q, 835 struct ccp_dm_workarea *wa, u32 jobid, u32 ksb, 836 u32 byte_swap, bool from) 837 { 838 struct ccp_op op; 839 840 memset(&op, 0, sizeof(op)); 841 842 op.cmd_q = cmd_q; 843 op.jobid = jobid; 844 op.eom = 1; 845 846 if (from) { 847 op.soc = 1; 848 op.src.type = CCP_MEMTYPE_KSB; 849 op.src.u.ksb = ksb; 850 op.dst.type = CCP_MEMTYPE_SYSTEM; 851 op.dst.u.dma.address = wa->dma.address; 852 op.dst.u.dma.length = wa->length; 853 } else { 854 op.src.type = CCP_MEMTYPE_SYSTEM; 855 op.src.u.dma.address = wa->dma.address; 856 op.src.u.dma.length = wa->length; 857 op.dst.type = CCP_MEMTYPE_KSB; 858 op.dst.u.ksb = ksb; 859 } 860 861 op.u.passthru.byte_swap = byte_swap; 862 863 return ccp_perform_passthru(&op); 864 } 865 866 static int ccp_copy_to_ksb(struct ccp_cmd_queue *cmd_q, 867 struct ccp_dm_workarea *wa, u32 jobid, u32 ksb, 868 u32 byte_swap) 869 { 870 return ccp_copy_to_from_ksb(cmd_q, wa, jobid, ksb, byte_swap, false); 871 } 872 873 static int ccp_copy_from_ksb(struct ccp_cmd_queue *cmd_q, 874 struct ccp_dm_workarea *wa, u32 jobid, u32 ksb, 875 u32 byte_swap) 876 { 877 return ccp_copy_to_from_ksb(cmd_q, wa, jobid, ksb, byte_swap, true); 878 } 879 880 static int ccp_run_aes_cmac_cmd(struct ccp_cmd_queue *cmd_q, 881 struct ccp_cmd *cmd) 882 { 883 struct ccp_aes_engine *aes = &cmd->u.aes; 884 struct ccp_dm_workarea key, ctx; 885 struct ccp_data src; 886 struct ccp_op op; 887 unsigned int dm_offset; 888 int ret; 889 890 if (!((aes->key_len == AES_KEYSIZE_128) || 891 (aes->key_len == AES_KEYSIZE_192) || 892 (aes->key_len == AES_KEYSIZE_256))) 893 return -EINVAL; 894 895 if (aes->src_len & (AES_BLOCK_SIZE - 1)) 896 return -EINVAL; 897 898 if (aes->iv_len != AES_BLOCK_SIZE) 899 return -EINVAL; 900 901 if (!aes->key || !aes->iv || !aes->src) 902 return -EINVAL; 903 904 if (aes->cmac_final) { 905 if (aes->cmac_key_len != AES_BLOCK_SIZE) 906 return -EINVAL; 907 908 if (!aes->cmac_key) 909 return -EINVAL; 910 } 911 912 BUILD_BUG_ON(CCP_AES_KEY_KSB_COUNT != 1); 913 BUILD_BUG_ON(CCP_AES_CTX_KSB_COUNT != 1); 914 915 ret = -EIO; 916 memset(&op, 0, sizeof(op)); 917 op.cmd_q = cmd_q; 918 op.jobid = ccp_gen_jobid(cmd_q->ccp); 919 op.ksb_key = cmd_q->ksb_key; 920 op.ksb_ctx = cmd_q->ksb_ctx; 921 op.init = 1; 922 op.u.aes.type = aes->type; 923 op.u.aes.mode = aes->mode; 924 op.u.aes.action = aes->action; 925 926 /* All supported key sizes fit in a single (32-byte) KSB entry 927 * and must be in little endian format. Use the 256-bit byte 928 * swap passthru option to convert from big endian to little 929 * endian. 930 */ 931 ret = ccp_init_dm_workarea(&key, cmd_q, 932 CCP_AES_KEY_KSB_COUNT * CCP_KSB_BYTES, 933 DMA_TO_DEVICE); 934 if (ret) 935 return ret; 936 937 dm_offset = CCP_KSB_BYTES - aes->key_len; 938 ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len); 939 ret = ccp_copy_to_ksb(cmd_q, &key, op.jobid, op.ksb_key, 940 CCP_PASSTHRU_BYTESWAP_256BIT); 941 if (ret) { 942 cmd->engine_error = cmd_q->cmd_error; 943 goto e_key; 944 } 945 946 /* The AES context fits in a single (32-byte) KSB entry and 947 * must be in little endian format. Use the 256-bit byte swap 948 * passthru option to convert from big endian to little endian. 949 */ 950 ret = ccp_init_dm_workarea(&ctx, cmd_q, 951 CCP_AES_CTX_KSB_COUNT * CCP_KSB_BYTES, 952 DMA_BIDIRECTIONAL); 953 if (ret) 954 goto e_key; 955 956 dm_offset = CCP_KSB_BYTES - AES_BLOCK_SIZE; 957 ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len); 958 ret = ccp_copy_to_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx, 959 CCP_PASSTHRU_BYTESWAP_256BIT); 960 if (ret) { 961 cmd->engine_error = cmd_q->cmd_error; 962 goto e_ctx; 963 } 964 965 /* Send data to the CCP AES engine */ 966 ret = ccp_init_data(&src, cmd_q, aes->src, aes->src_len, 967 AES_BLOCK_SIZE, DMA_TO_DEVICE); 968 if (ret) 969 goto e_ctx; 970 971 while (src.sg_wa.bytes_left) { 972 ccp_prepare_data(&src, NULL, &op, AES_BLOCK_SIZE, true); 973 if (aes->cmac_final && !src.sg_wa.bytes_left) { 974 op.eom = 1; 975 976 /* Push the K1/K2 key to the CCP now */ 977 ret = ccp_copy_from_ksb(cmd_q, &ctx, op.jobid, 978 op.ksb_ctx, 979 CCP_PASSTHRU_BYTESWAP_256BIT); 980 if (ret) { 981 cmd->engine_error = cmd_q->cmd_error; 982 goto e_src; 983 } 984 985 ccp_set_dm_area(&ctx, 0, aes->cmac_key, 0, 986 aes->cmac_key_len); 987 ret = ccp_copy_to_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx, 988 CCP_PASSTHRU_BYTESWAP_256BIT); 989 if (ret) { 990 cmd->engine_error = cmd_q->cmd_error; 991 goto e_src; 992 } 993 } 994 995 ret = ccp_perform_aes(&op); 996 if (ret) { 997 cmd->engine_error = cmd_q->cmd_error; 998 goto e_src; 999 } 1000 1001 ccp_process_data(&src, NULL, &op); 1002 } 1003 1004 /* Retrieve the AES context - convert from LE to BE using 1005 * 32-byte (256-bit) byteswapping 1006 */ 1007 ret = ccp_copy_from_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx, 1008 CCP_PASSTHRU_BYTESWAP_256BIT); 1009 if (ret) { 1010 cmd->engine_error = cmd_q->cmd_error; 1011 goto e_src; 1012 } 1013 1014 /* ...but we only need AES_BLOCK_SIZE bytes */ 1015 dm_offset = CCP_KSB_BYTES - AES_BLOCK_SIZE; 1016 ccp_get_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len); 1017 1018 e_src: 1019 ccp_free_data(&src, cmd_q); 1020 1021 e_ctx: 1022 ccp_dm_free(&ctx); 1023 1024 e_key: 1025 ccp_dm_free(&key); 1026 1027 return ret; 1028 } 1029 1030 static int ccp_run_aes_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) 1031 { 1032 struct ccp_aes_engine *aes = &cmd->u.aes; 1033 struct ccp_dm_workarea key, ctx; 1034 struct ccp_data src, dst; 1035 struct ccp_op op; 1036 unsigned int dm_offset; 1037 bool in_place = false; 1038 int ret; 1039 1040 if (aes->mode == CCP_AES_MODE_CMAC) 1041 return ccp_run_aes_cmac_cmd(cmd_q, cmd); 1042 1043 if (!((aes->key_len == AES_KEYSIZE_128) || 1044 (aes->key_len == AES_KEYSIZE_192) || 1045 (aes->key_len == AES_KEYSIZE_256))) 1046 return -EINVAL; 1047 1048 if (((aes->mode == CCP_AES_MODE_ECB) || 1049 (aes->mode == CCP_AES_MODE_CBC) || 1050 (aes->mode == CCP_AES_MODE_CFB)) && 1051 (aes->src_len & (AES_BLOCK_SIZE - 1))) 1052 return -EINVAL; 1053 1054 if (!aes->key || !aes->src || !aes->dst) 1055 return -EINVAL; 1056 1057 if (aes->mode != CCP_AES_MODE_ECB) { 1058 if (aes->iv_len != AES_BLOCK_SIZE) 1059 return -EINVAL; 1060 1061 if (!aes->iv) 1062 return -EINVAL; 1063 } 1064 1065 BUILD_BUG_ON(CCP_AES_KEY_KSB_COUNT != 1); 1066 BUILD_BUG_ON(CCP_AES_CTX_KSB_COUNT != 1); 1067 1068 ret = -EIO; 1069 memset(&op, 0, sizeof(op)); 1070 op.cmd_q = cmd_q; 1071 op.jobid = ccp_gen_jobid(cmd_q->ccp); 1072 op.ksb_key = cmd_q->ksb_key; 1073 op.ksb_ctx = cmd_q->ksb_ctx; 1074 op.init = (aes->mode == CCP_AES_MODE_ECB) ? 0 : 1; 1075 op.u.aes.type = aes->type; 1076 op.u.aes.mode = aes->mode; 1077 op.u.aes.action = aes->action; 1078 1079 /* All supported key sizes fit in a single (32-byte) KSB entry 1080 * and must be in little endian format. Use the 256-bit byte 1081 * swap passthru option to convert from big endian to little 1082 * endian. 1083 */ 1084 ret = ccp_init_dm_workarea(&key, cmd_q, 1085 CCP_AES_KEY_KSB_COUNT * CCP_KSB_BYTES, 1086 DMA_TO_DEVICE); 1087 if (ret) 1088 return ret; 1089 1090 dm_offset = CCP_KSB_BYTES - aes->key_len; 1091 ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len); 1092 ret = ccp_copy_to_ksb(cmd_q, &key, op.jobid, op.ksb_key, 1093 CCP_PASSTHRU_BYTESWAP_256BIT); 1094 if (ret) { 1095 cmd->engine_error = cmd_q->cmd_error; 1096 goto e_key; 1097 } 1098 1099 /* The AES context fits in a single (32-byte) KSB entry and 1100 * must be in little endian format. Use the 256-bit byte swap 1101 * passthru option to convert from big endian to little endian. 1102 */ 1103 ret = ccp_init_dm_workarea(&ctx, cmd_q, 1104 CCP_AES_CTX_KSB_COUNT * CCP_KSB_BYTES, 1105 DMA_BIDIRECTIONAL); 1106 if (ret) 1107 goto e_key; 1108 1109 if (aes->mode != CCP_AES_MODE_ECB) { 1110 /* Load the AES context - conver to LE */ 1111 dm_offset = CCP_KSB_BYTES - AES_BLOCK_SIZE; 1112 ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len); 1113 ret = ccp_copy_to_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx, 1114 CCP_PASSTHRU_BYTESWAP_256BIT); 1115 if (ret) { 1116 cmd->engine_error = cmd_q->cmd_error; 1117 goto e_ctx; 1118 } 1119 } 1120 1121 /* Prepare the input and output data workareas. For in-place 1122 * operations we need to set the dma direction to BIDIRECTIONAL 1123 * and copy the src workarea to the dst workarea. 1124 */ 1125 if (sg_virt(aes->src) == sg_virt(aes->dst)) 1126 in_place = true; 1127 1128 ret = ccp_init_data(&src, cmd_q, aes->src, aes->src_len, 1129 AES_BLOCK_SIZE, 1130 in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE); 1131 if (ret) 1132 goto e_ctx; 1133 1134 if (in_place) { 1135 dst = src; 1136 } else { 1137 ret = ccp_init_data(&dst, cmd_q, aes->dst, aes->src_len, 1138 AES_BLOCK_SIZE, DMA_FROM_DEVICE); 1139 if (ret) 1140 goto e_src; 1141 } 1142 1143 /* Send data to the CCP AES engine */ 1144 while (src.sg_wa.bytes_left) { 1145 ccp_prepare_data(&src, &dst, &op, AES_BLOCK_SIZE, true); 1146 if (!src.sg_wa.bytes_left) { 1147 op.eom = 1; 1148 1149 /* Since we don't retrieve the AES context in ECB 1150 * mode we have to wait for the operation to complete 1151 * on the last piece of data 1152 */ 1153 if (aes->mode == CCP_AES_MODE_ECB) 1154 op.soc = 1; 1155 } 1156 1157 ret = ccp_perform_aes(&op); 1158 if (ret) { 1159 cmd->engine_error = cmd_q->cmd_error; 1160 goto e_dst; 1161 } 1162 1163 ccp_process_data(&src, &dst, &op); 1164 } 1165 1166 if (aes->mode != CCP_AES_MODE_ECB) { 1167 /* Retrieve the AES context - convert from LE to BE using 1168 * 32-byte (256-bit) byteswapping 1169 */ 1170 ret = ccp_copy_from_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx, 1171 CCP_PASSTHRU_BYTESWAP_256BIT); 1172 if (ret) { 1173 cmd->engine_error = cmd_q->cmd_error; 1174 goto e_dst; 1175 } 1176 1177 /* ...but we only need AES_BLOCK_SIZE bytes */ 1178 dm_offset = CCP_KSB_BYTES - AES_BLOCK_SIZE; 1179 ccp_get_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len); 1180 } 1181 1182 e_dst: 1183 if (!in_place) 1184 ccp_free_data(&dst, cmd_q); 1185 1186 e_src: 1187 ccp_free_data(&src, cmd_q); 1188 1189 e_ctx: 1190 ccp_dm_free(&ctx); 1191 1192 e_key: 1193 ccp_dm_free(&key); 1194 1195 return ret; 1196 } 1197 1198 static int ccp_run_xts_aes_cmd(struct ccp_cmd_queue *cmd_q, 1199 struct ccp_cmd *cmd) 1200 { 1201 struct ccp_xts_aes_engine *xts = &cmd->u.xts; 1202 struct ccp_dm_workarea key, ctx; 1203 struct ccp_data src, dst; 1204 struct ccp_op op; 1205 unsigned int unit_size, dm_offset; 1206 bool in_place = false; 1207 int ret; 1208 1209 switch (xts->unit_size) { 1210 case CCP_XTS_AES_UNIT_SIZE_16: 1211 unit_size = 16; 1212 break; 1213 case CCP_XTS_AES_UNIT_SIZE_512: 1214 unit_size = 512; 1215 break; 1216 case CCP_XTS_AES_UNIT_SIZE_1024: 1217 unit_size = 1024; 1218 break; 1219 case CCP_XTS_AES_UNIT_SIZE_2048: 1220 unit_size = 2048; 1221 break; 1222 case CCP_XTS_AES_UNIT_SIZE_4096: 1223 unit_size = 4096; 1224 break; 1225 1226 default: 1227 return -EINVAL; 1228 } 1229 1230 if (xts->key_len != AES_KEYSIZE_128) 1231 return -EINVAL; 1232 1233 if (!xts->final && (xts->src_len & (AES_BLOCK_SIZE - 1))) 1234 return -EINVAL; 1235 1236 if (xts->iv_len != AES_BLOCK_SIZE) 1237 return -EINVAL; 1238 1239 if (!xts->key || !xts->iv || !xts->src || !xts->dst) 1240 return -EINVAL; 1241 1242 BUILD_BUG_ON(CCP_XTS_AES_KEY_KSB_COUNT != 1); 1243 BUILD_BUG_ON(CCP_XTS_AES_CTX_KSB_COUNT != 1); 1244 1245 ret = -EIO; 1246 memset(&op, 0, sizeof(op)); 1247 op.cmd_q = cmd_q; 1248 op.jobid = ccp_gen_jobid(cmd_q->ccp); 1249 op.ksb_key = cmd_q->ksb_key; 1250 op.ksb_ctx = cmd_q->ksb_ctx; 1251 op.init = 1; 1252 op.u.xts.action = xts->action; 1253 op.u.xts.unit_size = xts->unit_size; 1254 1255 /* All supported key sizes fit in a single (32-byte) KSB entry 1256 * and must be in little endian format. Use the 256-bit byte 1257 * swap passthru option to convert from big endian to little 1258 * endian. 1259 */ 1260 ret = ccp_init_dm_workarea(&key, cmd_q, 1261 CCP_XTS_AES_KEY_KSB_COUNT * CCP_KSB_BYTES, 1262 DMA_TO_DEVICE); 1263 if (ret) 1264 return ret; 1265 1266 dm_offset = CCP_KSB_BYTES - AES_KEYSIZE_128; 1267 ccp_set_dm_area(&key, dm_offset, xts->key, 0, xts->key_len); 1268 ccp_set_dm_area(&key, 0, xts->key, dm_offset, xts->key_len); 1269 ret = ccp_copy_to_ksb(cmd_q, &key, op.jobid, op.ksb_key, 1270 CCP_PASSTHRU_BYTESWAP_256BIT); 1271 if (ret) { 1272 cmd->engine_error = cmd_q->cmd_error; 1273 goto e_key; 1274 } 1275 1276 /* The AES context fits in a single (32-byte) KSB entry and 1277 * for XTS is already in little endian format so no byte swapping 1278 * is needed. 1279 */ 1280 ret = ccp_init_dm_workarea(&ctx, cmd_q, 1281 CCP_XTS_AES_CTX_KSB_COUNT * CCP_KSB_BYTES, 1282 DMA_BIDIRECTIONAL); 1283 if (ret) 1284 goto e_key; 1285 1286 ccp_set_dm_area(&ctx, 0, xts->iv, 0, xts->iv_len); 1287 ret = ccp_copy_to_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx, 1288 CCP_PASSTHRU_BYTESWAP_NOOP); 1289 if (ret) { 1290 cmd->engine_error = cmd_q->cmd_error; 1291 goto e_ctx; 1292 } 1293 1294 /* Prepare the input and output data workareas. For in-place 1295 * operations we need to set the dma direction to BIDIRECTIONAL 1296 * and copy the src workarea to the dst workarea. 1297 */ 1298 if (sg_virt(xts->src) == sg_virt(xts->dst)) 1299 in_place = true; 1300 1301 ret = ccp_init_data(&src, cmd_q, xts->src, xts->src_len, 1302 unit_size, 1303 in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE); 1304 if (ret) 1305 goto e_ctx; 1306 1307 if (in_place) { 1308 dst = src; 1309 } else { 1310 ret = ccp_init_data(&dst, cmd_q, xts->dst, xts->src_len, 1311 unit_size, DMA_FROM_DEVICE); 1312 if (ret) 1313 goto e_src; 1314 } 1315 1316 /* Send data to the CCP AES engine */ 1317 while (src.sg_wa.bytes_left) { 1318 ccp_prepare_data(&src, &dst, &op, unit_size, true); 1319 if (!src.sg_wa.bytes_left) 1320 op.eom = 1; 1321 1322 ret = ccp_perform_xts_aes(&op); 1323 if (ret) { 1324 cmd->engine_error = cmd_q->cmd_error; 1325 goto e_dst; 1326 } 1327 1328 ccp_process_data(&src, &dst, &op); 1329 } 1330 1331 /* Retrieve the AES context - convert from LE to BE using 1332 * 32-byte (256-bit) byteswapping 1333 */ 1334 ret = ccp_copy_from_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx, 1335 CCP_PASSTHRU_BYTESWAP_256BIT); 1336 if (ret) { 1337 cmd->engine_error = cmd_q->cmd_error; 1338 goto e_dst; 1339 } 1340 1341 /* ...but we only need AES_BLOCK_SIZE bytes */ 1342 dm_offset = CCP_KSB_BYTES - AES_BLOCK_SIZE; 1343 ccp_get_dm_area(&ctx, dm_offset, xts->iv, 0, xts->iv_len); 1344 1345 e_dst: 1346 if (!in_place) 1347 ccp_free_data(&dst, cmd_q); 1348 1349 e_src: 1350 ccp_free_data(&src, cmd_q); 1351 1352 e_ctx: 1353 ccp_dm_free(&ctx); 1354 1355 e_key: 1356 ccp_dm_free(&key); 1357 1358 return ret; 1359 } 1360 1361 static int ccp_run_sha_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) 1362 { 1363 struct ccp_sha_engine *sha = &cmd->u.sha; 1364 struct ccp_dm_workarea ctx; 1365 struct ccp_data src; 1366 struct ccp_op op; 1367 int ret; 1368 1369 if (sha->ctx_len != CCP_SHA_CTXSIZE) 1370 return -EINVAL; 1371 1372 if (!sha->ctx) 1373 return -EINVAL; 1374 1375 if (!sha->final && (sha->src_len & (CCP_SHA_BLOCKSIZE - 1))) 1376 return -EINVAL; 1377 1378 if (!sha->src_len) { 1379 const u8 *sha_zero; 1380 1381 /* Not final, just return */ 1382 if (!sha->final) 1383 return 0; 1384 1385 /* CCP can't do a zero length sha operation so the caller 1386 * must buffer the data. 1387 */ 1388 if (sha->msg_bits) 1389 return -EINVAL; 1390 1391 /* A sha operation for a message with a total length of zero, 1392 * return known result. 1393 */ 1394 switch (sha->type) { 1395 case CCP_SHA_TYPE_1: 1396 sha_zero = ccp_sha1_zero; 1397 break; 1398 case CCP_SHA_TYPE_224: 1399 sha_zero = ccp_sha224_zero; 1400 break; 1401 case CCP_SHA_TYPE_256: 1402 sha_zero = ccp_sha256_zero; 1403 break; 1404 default: 1405 return -EINVAL; 1406 } 1407 1408 scatterwalk_map_and_copy((void *)sha_zero, sha->ctx, 0, 1409 sha->ctx_len, 1); 1410 1411 return 0; 1412 } 1413 1414 if (!sha->src) 1415 return -EINVAL; 1416 1417 BUILD_BUG_ON(CCP_SHA_KSB_COUNT != 1); 1418 1419 memset(&op, 0, sizeof(op)); 1420 op.cmd_q = cmd_q; 1421 op.jobid = ccp_gen_jobid(cmd_q->ccp); 1422 op.ksb_ctx = cmd_q->ksb_ctx; 1423 op.u.sha.type = sha->type; 1424 op.u.sha.msg_bits = sha->msg_bits; 1425 1426 /* The SHA context fits in a single (32-byte) KSB entry and 1427 * must be in little endian format. Use the 256-bit byte swap 1428 * passthru option to convert from big endian to little endian. 1429 */ 1430 ret = ccp_init_dm_workarea(&ctx, cmd_q, 1431 CCP_SHA_KSB_COUNT * CCP_KSB_BYTES, 1432 DMA_BIDIRECTIONAL); 1433 if (ret) 1434 return ret; 1435 1436 if (sha->first) { 1437 const __be32 *init; 1438 1439 switch (sha->type) { 1440 case CCP_SHA_TYPE_1: 1441 init = ccp_sha1_init; 1442 break; 1443 case CCP_SHA_TYPE_224: 1444 init = ccp_sha224_init; 1445 break; 1446 case CCP_SHA_TYPE_256: 1447 init = ccp_sha256_init; 1448 break; 1449 default: 1450 ret = -EINVAL; 1451 goto e_ctx; 1452 } 1453 memcpy(ctx.address, init, CCP_SHA_CTXSIZE); 1454 } else { 1455 ccp_set_dm_area(&ctx, 0, sha->ctx, 0, sha->ctx_len); 1456 } 1457 1458 ret = ccp_copy_to_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx, 1459 CCP_PASSTHRU_BYTESWAP_256BIT); 1460 if (ret) { 1461 cmd->engine_error = cmd_q->cmd_error; 1462 goto e_ctx; 1463 } 1464 1465 /* Send data to the CCP SHA engine */ 1466 ret = ccp_init_data(&src, cmd_q, sha->src, sha->src_len, 1467 CCP_SHA_BLOCKSIZE, DMA_TO_DEVICE); 1468 if (ret) 1469 goto e_ctx; 1470 1471 while (src.sg_wa.bytes_left) { 1472 ccp_prepare_data(&src, NULL, &op, CCP_SHA_BLOCKSIZE, false); 1473 if (sha->final && !src.sg_wa.bytes_left) 1474 op.eom = 1; 1475 1476 ret = ccp_perform_sha(&op); 1477 if (ret) { 1478 cmd->engine_error = cmd_q->cmd_error; 1479 goto e_data; 1480 } 1481 1482 ccp_process_data(&src, NULL, &op); 1483 } 1484 1485 /* Retrieve the SHA context - convert from LE to BE using 1486 * 32-byte (256-bit) byteswapping to BE 1487 */ 1488 ret = ccp_copy_from_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx, 1489 CCP_PASSTHRU_BYTESWAP_256BIT); 1490 if (ret) { 1491 cmd->engine_error = cmd_q->cmd_error; 1492 goto e_data; 1493 } 1494 1495 ccp_get_dm_area(&ctx, 0, sha->ctx, 0, sha->ctx_len); 1496 1497 if (sha->final && sha->opad) { 1498 /* HMAC operation, recursively perform final SHA */ 1499 struct ccp_cmd hmac_cmd; 1500 struct scatterlist sg; 1501 u64 block_size, digest_size; 1502 u8 *hmac_buf; 1503 1504 switch (sha->type) { 1505 case CCP_SHA_TYPE_1: 1506 block_size = SHA1_BLOCK_SIZE; 1507 digest_size = SHA1_DIGEST_SIZE; 1508 break; 1509 case CCP_SHA_TYPE_224: 1510 block_size = SHA224_BLOCK_SIZE; 1511 digest_size = SHA224_DIGEST_SIZE; 1512 break; 1513 case CCP_SHA_TYPE_256: 1514 block_size = SHA256_BLOCK_SIZE; 1515 digest_size = SHA256_DIGEST_SIZE; 1516 break; 1517 default: 1518 ret = -EINVAL; 1519 goto e_data; 1520 } 1521 1522 if (sha->opad_len != block_size) { 1523 ret = -EINVAL; 1524 goto e_data; 1525 } 1526 1527 hmac_buf = kmalloc(block_size + digest_size, GFP_KERNEL); 1528 if (!hmac_buf) { 1529 ret = -ENOMEM; 1530 goto e_data; 1531 } 1532 sg_init_one(&sg, hmac_buf, block_size + digest_size); 1533 1534 scatterwalk_map_and_copy(hmac_buf, sha->opad, 0, block_size, 0); 1535 memcpy(hmac_buf + block_size, ctx.address, digest_size); 1536 1537 memset(&hmac_cmd, 0, sizeof(hmac_cmd)); 1538 hmac_cmd.engine = CCP_ENGINE_SHA; 1539 hmac_cmd.u.sha.type = sha->type; 1540 hmac_cmd.u.sha.ctx = sha->ctx; 1541 hmac_cmd.u.sha.ctx_len = sha->ctx_len; 1542 hmac_cmd.u.sha.src = &sg; 1543 hmac_cmd.u.sha.src_len = block_size + digest_size; 1544 hmac_cmd.u.sha.opad = NULL; 1545 hmac_cmd.u.sha.opad_len = 0; 1546 hmac_cmd.u.sha.first = 1; 1547 hmac_cmd.u.sha.final = 1; 1548 hmac_cmd.u.sha.msg_bits = (block_size + digest_size) << 3; 1549 1550 ret = ccp_run_sha_cmd(cmd_q, &hmac_cmd); 1551 if (ret) 1552 cmd->engine_error = hmac_cmd.engine_error; 1553 1554 kfree(hmac_buf); 1555 } 1556 1557 e_data: 1558 ccp_free_data(&src, cmd_q); 1559 1560 e_ctx: 1561 ccp_dm_free(&ctx); 1562 1563 return ret; 1564 } 1565 1566 static int ccp_run_rsa_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) 1567 { 1568 struct ccp_rsa_engine *rsa = &cmd->u.rsa; 1569 struct ccp_dm_workarea exp, src; 1570 struct ccp_data dst; 1571 struct ccp_op op; 1572 unsigned int ksb_count, i_len, o_len; 1573 int ret; 1574 1575 if (rsa->key_size > CCP_RSA_MAX_WIDTH) 1576 return -EINVAL; 1577 1578 if (!rsa->exp || !rsa->mod || !rsa->src || !rsa->dst) 1579 return -EINVAL; 1580 1581 /* The RSA modulus must precede the message being acted upon, so 1582 * it must be copied to a DMA area where the message and the 1583 * modulus can be concatenated. Therefore the input buffer 1584 * length required is twice the output buffer length (which 1585 * must be a multiple of 256-bits). 1586 */ 1587 o_len = ((rsa->key_size + 255) / 256) * 32; 1588 i_len = o_len * 2; 1589 1590 ksb_count = o_len / CCP_KSB_BYTES; 1591 1592 memset(&op, 0, sizeof(op)); 1593 op.cmd_q = cmd_q; 1594 op.jobid = ccp_gen_jobid(cmd_q->ccp); 1595 op.ksb_key = ccp_alloc_ksb(cmd_q->ccp, ksb_count); 1596 if (!op.ksb_key) 1597 return -EIO; 1598 1599 /* The RSA exponent may span multiple (32-byte) KSB entries and must 1600 * be in little endian format. Reverse copy each 32-byte chunk 1601 * of the exponent (En chunk to E0 chunk, E(n-1) chunk to E1 chunk) 1602 * and each byte within that chunk and do not perform any byte swap 1603 * operations on the passthru operation. 1604 */ 1605 ret = ccp_init_dm_workarea(&exp, cmd_q, o_len, DMA_TO_DEVICE); 1606 if (ret) 1607 goto e_ksb; 1608 1609 ccp_reverse_set_dm_area(&exp, rsa->exp, rsa->exp_len, CCP_KSB_BYTES, 1610 false); 1611 ret = ccp_copy_to_ksb(cmd_q, &exp, op.jobid, op.ksb_key, 1612 CCP_PASSTHRU_BYTESWAP_NOOP); 1613 if (ret) { 1614 cmd->engine_error = cmd_q->cmd_error; 1615 goto e_exp; 1616 } 1617 1618 /* Concatenate the modulus and the message. Both the modulus and 1619 * the operands must be in little endian format. Since the input 1620 * is in big endian format it must be converted. 1621 */ 1622 ret = ccp_init_dm_workarea(&src, cmd_q, i_len, DMA_TO_DEVICE); 1623 if (ret) 1624 goto e_exp; 1625 1626 ccp_reverse_set_dm_area(&src, rsa->mod, rsa->mod_len, CCP_KSB_BYTES, 1627 false); 1628 src.address += o_len; /* Adjust the address for the copy operation */ 1629 ccp_reverse_set_dm_area(&src, rsa->src, rsa->src_len, CCP_KSB_BYTES, 1630 false); 1631 src.address -= o_len; /* Reset the address to original value */ 1632 1633 /* Prepare the output area for the operation */ 1634 ret = ccp_init_data(&dst, cmd_q, rsa->dst, rsa->mod_len, 1635 o_len, DMA_FROM_DEVICE); 1636 if (ret) 1637 goto e_src; 1638 1639 op.soc = 1; 1640 op.src.u.dma.address = src.dma.address; 1641 op.src.u.dma.offset = 0; 1642 op.src.u.dma.length = i_len; 1643 op.dst.u.dma.address = dst.dm_wa.dma.address; 1644 op.dst.u.dma.offset = 0; 1645 op.dst.u.dma.length = o_len; 1646 1647 op.u.rsa.mod_size = rsa->key_size; 1648 op.u.rsa.input_len = i_len; 1649 1650 ret = ccp_perform_rsa(&op); 1651 if (ret) { 1652 cmd->engine_error = cmd_q->cmd_error; 1653 goto e_dst; 1654 } 1655 1656 ccp_reverse_get_dm_area(&dst.dm_wa, rsa->dst, rsa->mod_len); 1657 1658 e_dst: 1659 ccp_free_data(&dst, cmd_q); 1660 1661 e_src: 1662 ccp_dm_free(&src); 1663 1664 e_exp: 1665 ccp_dm_free(&exp); 1666 1667 e_ksb: 1668 ccp_free_ksb(cmd_q->ccp, op.ksb_key, ksb_count); 1669 1670 return ret; 1671 } 1672 1673 static int ccp_run_passthru_cmd(struct ccp_cmd_queue *cmd_q, 1674 struct ccp_cmd *cmd) 1675 { 1676 struct ccp_passthru_engine *pt = &cmd->u.passthru; 1677 struct ccp_dm_workarea mask; 1678 struct ccp_data src, dst; 1679 struct ccp_op op; 1680 bool in_place = false; 1681 unsigned int i; 1682 int ret; 1683 1684 if (!pt->final && (pt->src_len & (CCP_PASSTHRU_BLOCKSIZE - 1))) 1685 return -EINVAL; 1686 1687 if (!pt->src || !pt->dst) 1688 return -EINVAL; 1689 1690 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) { 1691 if (pt->mask_len != CCP_PASSTHRU_MASKSIZE) 1692 return -EINVAL; 1693 if (!pt->mask) 1694 return -EINVAL; 1695 } 1696 1697 BUILD_BUG_ON(CCP_PASSTHRU_KSB_COUNT != 1); 1698 1699 memset(&op, 0, sizeof(op)); 1700 op.cmd_q = cmd_q; 1701 op.jobid = ccp_gen_jobid(cmd_q->ccp); 1702 1703 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) { 1704 /* Load the mask */ 1705 op.ksb_key = cmd_q->ksb_key; 1706 1707 ret = ccp_init_dm_workarea(&mask, cmd_q, 1708 CCP_PASSTHRU_KSB_COUNT * 1709 CCP_KSB_BYTES, 1710 DMA_TO_DEVICE); 1711 if (ret) 1712 return ret; 1713 1714 ccp_set_dm_area(&mask, 0, pt->mask, 0, pt->mask_len); 1715 ret = ccp_copy_to_ksb(cmd_q, &mask, op.jobid, op.ksb_key, 1716 CCP_PASSTHRU_BYTESWAP_NOOP); 1717 if (ret) { 1718 cmd->engine_error = cmd_q->cmd_error; 1719 goto e_mask; 1720 } 1721 } 1722 1723 /* Prepare the input and output data workareas. For in-place 1724 * operations we need to set the dma direction to BIDIRECTIONAL 1725 * and copy the src workarea to the dst workarea. 1726 */ 1727 if (sg_virt(pt->src) == sg_virt(pt->dst)) 1728 in_place = true; 1729 1730 ret = ccp_init_data(&src, cmd_q, pt->src, pt->src_len, 1731 CCP_PASSTHRU_MASKSIZE, 1732 in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE); 1733 if (ret) 1734 goto e_mask; 1735 1736 if (in_place) { 1737 dst = src; 1738 } else { 1739 ret = ccp_init_data(&dst, cmd_q, pt->dst, pt->src_len, 1740 CCP_PASSTHRU_MASKSIZE, DMA_FROM_DEVICE); 1741 if (ret) 1742 goto e_src; 1743 } 1744 1745 /* Send data to the CCP Passthru engine 1746 * Because the CCP engine works on a single source and destination 1747 * dma address at a time, each entry in the source scatterlist 1748 * (after the dma_map_sg call) must be less than or equal to the 1749 * (remaining) length in the destination scatterlist entry and the 1750 * length must be a multiple of CCP_PASSTHRU_BLOCKSIZE 1751 */ 1752 dst.sg_wa.sg_used = 0; 1753 for (i = 1; i <= src.sg_wa.dma_count; i++) { 1754 if (!dst.sg_wa.sg || 1755 (dst.sg_wa.sg->length < src.sg_wa.sg->length)) { 1756 ret = -EINVAL; 1757 goto e_dst; 1758 } 1759 1760 if (i == src.sg_wa.dma_count) { 1761 op.eom = 1; 1762 op.soc = 1; 1763 } 1764 1765 op.src.type = CCP_MEMTYPE_SYSTEM; 1766 op.src.u.dma.address = sg_dma_address(src.sg_wa.sg); 1767 op.src.u.dma.offset = 0; 1768 op.src.u.dma.length = sg_dma_len(src.sg_wa.sg); 1769 1770 op.dst.type = CCP_MEMTYPE_SYSTEM; 1771 op.dst.u.dma.address = sg_dma_address(dst.sg_wa.sg); 1772 op.dst.u.dma.offset = dst.sg_wa.sg_used; 1773 op.dst.u.dma.length = op.src.u.dma.length; 1774 1775 ret = ccp_perform_passthru(&op); 1776 if (ret) { 1777 cmd->engine_error = cmd_q->cmd_error; 1778 goto e_dst; 1779 } 1780 1781 dst.sg_wa.sg_used += src.sg_wa.sg->length; 1782 if (dst.sg_wa.sg_used == dst.sg_wa.sg->length) { 1783 dst.sg_wa.sg = sg_next(dst.sg_wa.sg); 1784 dst.sg_wa.sg_used = 0; 1785 } 1786 src.sg_wa.sg = sg_next(src.sg_wa.sg); 1787 } 1788 1789 e_dst: 1790 if (!in_place) 1791 ccp_free_data(&dst, cmd_q); 1792 1793 e_src: 1794 ccp_free_data(&src, cmd_q); 1795 1796 e_mask: 1797 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) 1798 ccp_dm_free(&mask); 1799 1800 return ret; 1801 } 1802 1803 static int ccp_run_ecc_mm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) 1804 { 1805 struct ccp_ecc_engine *ecc = &cmd->u.ecc; 1806 struct ccp_dm_workarea src, dst; 1807 struct ccp_op op; 1808 int ret; 1809 u8 *save; 1810 1811 if (!ecc->u.mm.operand_1 || 1812 (ecc->u.mm.operand_1_len > CCP_ECC_MODULUS_BYTES)) 1813 return -EINVAL; 1814 1815 if (ecc->function != CCP_ECC_FUNCTION_MINV_384BIT) 1816 if (!ecc->u.mm.operand_2 || 1817 (ecc->u.mm.operand_2_len > CCP_ECC_MODULUS_BYTES)) 1818 return -EINVAL; 1819 1820 if (!ecc->u.mm.result || 1821 (ecc->u.mm.result_len < CCP_ECC_MODULUS_BYTES)) 1822 return -EINVAL; 1823 1824 memset(&op, 0, sizeof(op)); 1825 op.cmd_q = cmd_q; 1826 op.jobid = ccp_gen_jobid(cmd_q->ccp); 1827 1828 /* Concatenate the modulus and the operands. Both the modulus and 1829 * the operands must be in little endian format. Since the input 1830 * is in big endian format it must be converted and placed in a 1831 * fixed length buffer. 1832 */ 1833 ret = ccp_init_dm_workarea(&src, cmd_q, CCP_ECC_SRC_BUF_SIZE, 1834 DMA_TO_DEVICE); 1835 if (ret) 1836 return ret; 1837 1838 /* Save the workarea address since it is updated in order to perform 1839 * the concatenation 1840 */ 1841 save = src.address; 1842 1843 /* Copy the ECC modulus */ 1844 ccp_reverse_set_dm_area(&src, ecc->mod, ecc->mod_len, 1845 CCP_ECC_OPERAND_SIZE, false); 1846 src.address += CCP_ECC_OPERAND_SIZE; 1847 1848 /* Copy the first operand */ 1849 ccp_reverse_set_dm_area(&src, ecc->u.mm.operand_1, 1850 ecc->u.mm.operand_1_len, 1851 CCP_ECC_OPERAND_SIZE, false); 1852 src.address += CCP_ECC_OPERAND_SIZE; 1853 1854 if (ecc->function != CCP_ECC_FUNCTION_MINV_384BIT) { 1855 /* Copy the second operand */ 1856 ccp_reverse_set_dm_area(&src, ecc->u.mm.operand_2, 1857 ecc->u.mm.operand_2_len, 1858 CCP_ECC_OPERAND_SIZE, false); 1859 src.address += CCP_ECC_OPERAND_SIZE; 1860 } 1861 1862 /* Restore the workarea address */ 1863 src.address = save; 1864 1865 /* Prepare the output area for the operation */ 1866 ret = ccp_init_dm_workarea(&dst, cmd_q, CCP_ECC_DST_BUF_SIZE, 1867 DMA_FROM_DEVICE); 1868 if (ret) 1869 goto e_src; 1870 1871 op.soc = 1; 1872 op.src.u.dma.address = src.dma.address; 1873 op.src.u.dma.offset = 0; 1874 op.src.u.dma.length = src.length; 1875 op.dst.u.dma.address = dst.dma.address; 1876 op.dst.u.dma.offset = 0; 1877 op.dst.u.dma.length = dst.length; 1878 1879 op.u.ecc.function = cmd->u.ecc.function; 1880 1881 ret = ccp_perform_ecc(&op); 1882 if (ret) { 1883 cmd->engine_error = cmd_q->cmd_error; 1884 goto e_dst; 1885 } 1886 1887 ecc->ecc_result = le16_to_cpup( 1888 (const __le16 *)(dst.address + CCP_ECC_RESULT_OFFSET)); 1889 if (!(ecc->ecc_result & CCP_ECC_RESULT_SUCCESS)) { 1890 ret = -EIO; 1891 goto e_dst; 1892 } 1893 1894 /* Save the ECC result */ 1895 ccp_reverse_get_dm_area(&dst, ecc->u.mm.result, CCP_ECC_MODULUS_BYTES); 1896 1897 e_dst: 1898 ccp_dm_free(&dst); 1899 1900 e_src: 1901 ccp_dm_free(&src); 1902 1903 return ret; 1904 } 1905 1906 static int ccp_run_ecc_pm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) 1907 { 1908 struct ccp_ecc_engine *ecc = &cmd->u.ecc; 1909 struct ccp_dm_workarea src, dst; 1910 struct ccp_op op; 1911 int ret; 1912 u8 *save; 1913 1914 if (!ecc->u.pm.point_1.x || 1915 (ecc->u.pm.point_1.x_len > CCP_ECC_MODULUS_BYTES) || 1916 !ecc->u.pm.point_1.y || 1917 (ecc->u.pm.point_1.y_len > CCP_ECC_MODULUS_BYTES)) 1918 return -EINVAL; 1919 1920 if (ecc->function == CCP_ECC_FUNCTION_PADD_384BIT) { 1921 if (!ecc->u.pm.point_2.x || 1922 (ecc->u.pm.point_2.x_len > CCP_ECC_MODULUS_BYTES) || 1923 !ecc->u.pm.point_2.y || 1924 (ecc->u.pm.point_2.y_len > CCP_ECC_MODULUS_BYTES)) 1925 return -EINVAL; 1926 } else { 1927 if (!ecc->u.pm.domain_a || 1928 (ecc->u.pm.domain_a_len > CCP_ECC_MODULUS_BYTES)) 1929 return -EINVAL; 1930 1931 if (ecc->function == CCP_ECC_FUNCTION_PMUL_384BIT) 1932 if (!ecc->u.pm.scalar || 1933 (ecc->u.pm.scalar_len > CCP_ECC_MODULUS_BYTES)) 1934 return -EINVAL; 1935 } 1936 1937 if (!ecc->u.pm.result.x || 1938 (ecc->u.pm.result.x_len < CCP_ECC_MODULUS_BYTES) || 1939 !ecc->u.pm.result.y || 1940 (ecc->u.pm.result.y_len < CCP_ECC_MODULUS_BYTES)) 1941 return -EINVAL; 1942 1943 memset(&op, 0, sizeof(op)); 1944 op.cmd_q = cmd_q; 1945 op.jobid = ccp_gen_jobid(cmd_q->ccp); 1946 1947 /* Concatenate the modulus and the operands. Both the modulus and 1948 * the operands must be in little endian format. Since the input 1949 * is in big endian format it must be converted and placed in a 1950 * fixed length buffer. 1951 */ 1952 ret = ccp_init_dm_workarea(&src, cmd_q, CCP_ECC_SRC_BUF_SIZE, 1953 DMA_TO_DEVICE); 1954 if (ret) 1955 return ret; 1956 1957 /* Save the workarea address since it is updated in order to perform 1958 * the concatenation 1959 */ 1960 save = src.address; 1961 1962 /* Copy the ECC modulus */ 1963 ccp_reverse_set_dm_area(&src, ecc->mod, ecc->mod_len, 1964 CCP_ECC_OPERAND_SIZE, false); 1965 src.address += CCP_ECC_OPERAND_SIZE; 1966 1967 /* Copy the first point X and Y coordinate */ 1968 ccp_reverse_set_dm_area(&src, ecc->u.pm.point_1.x, 1969 ecc->u.pm.point_1.x_len, 1970 CCP_ECC_OPERAND_SIZE, false); 1971 src.address += CCP_ECC_OPERAND_SIZE; 1972 ccp_reverse_set_dm_area(&src, ecc->u.pm.point_1.y, 1973 ecc->u.pm.point_1.y_len, 1974 CCP_ECC_OPERAND_SIZE, false); 1975 src.address += CCP_ECC_OPERAND_SIZE; 1976 1977 /* Set the first point Z coordianate to 1 */ 1978 *src.address = 0x01; 1979 src.address += CCP_ECC_OPERAND_SIZE; 1980 1981 if (ecc->function == CCP_ECC_FUNCTION_PADD_384BIT) { 1982 /* Copy the second point X and Y coordinate */ 1983 ccp_reverse_set_dm_area(&src, ecc->u.pm.point_2.x, 1984 ecc->u.pm.point_2.x_len, 1985 CCP_ECC_OPERAND_SIZE, false); 1986 src.address += CCP_ECC_OPERAND_SIZE; 1987 ccp_reverse_set_dm_area(&src, ecc->u.pm.point_2.y, 1988 ecc->u.pm.point_2.y_len, 1989 CCP_ECC_OPERAND_SIZE, false); 1990 src.address += CCP_ECC_OPERAND_SIZE; 1991 1992 /* Set the second point Z coordianate to 1 */ 1993 *src.address = 0x01; 1994 src.address += CCP_ECC_OPERAND_SIZE; 1995 } else { 1996 /* Copy the Domain "a" parameter */ 1997 ccp_reverse_set_dm_area(&src, ecc->u.pm.domain_a, 1998 ecc->u.pm.domain_a_len, 1999 CCP_ECC_OPERAND_SIZE, false); 2000 src.address += CCP_ECC_OPERAND_SIZE; 2001 2002 if (ecc->function == CCP_ECC_FUNCTION_PMUL_384BIT) { 2003 /* Copy the scalar value */ 2004 ccp_reverse_set_dm_area(&src, ecc->u.pm.scalar, 2005 ecc->u.pm.scalar_len, 2006 CCP_ECC_OPERAND_SIZE, false); 2007 src.address += CCP_ECC_OPERAND_SIZE; 2008 } 2009 } 2010 2011 /* Restore the workarea address */ 2012 src.address = save; 2013 2014 /* Prepare the output area for the operation */ 2015 ret = ccp_init_dm_workarea(&dst, cmd_q, CCP_ECC_DST_BUF_SIZE, 2016 DMA_FROM_DEVICE); 2017 if (ret) 2018 goto e_src; 2019 2020 op.soc = 1; 2021 op.src.u.dma.address = src.dma.address; 2022 op.src.u.dma.offset = 0; 2023 op.src.u.dma.length = src.length; 2024 op.dst.u.dma.address = dst.dma.address; 2025 op.dst.u.dma.offset = 0; 2026 op.dst.u.dma.length = dst.length; 2027 2028 op.u.ecc.function = cmd->u.ecc.function; 2029 2030 ret = ccp_perform_ecc(&op); 2031 if (ret) { 2032 cmd->engine_error = cmd_q->cmd_error; 2033 goto e_dst; 2034 } 2035 2036 ecc->ecc_result = le16_to_cpup( 2037 (const __le16 *)(dst.address + CCP_ECC_RESULT_OFFSET)); 2038 if (!(ecc->ecc_result & CCP_ECC_RESULT_SUCCESS)) { 2039 ret = -EIO; 2040 goto e_dst; 2041 } 2042 2043 /* Save the workarea address since it is updated as we walk through 2044 * to copy the point math result 2045 */ 2046 save = dst.address; 2047 2048 /* Save the ECC result X and Y coordinates */ 2049 ccp_reverse_get_dm_area(&dst, ecc->u.pm.result.x, 2050 CCP_ECC_MODULUS_BYTES); 2051 dst.address += CCP_ECC_OUTPUT_SIZE; 2052 ccp_reverse_get_dm_area(&dst, ecc->u.pm.result.y, 2053 CCP_ECC_MODULUS_BYTES); 2054 dst.address += CCP_ECC_OUTPUT_SIZE; 2055 2056 /* Restore the workarea address */ 2057 dst.address = save; 2058 2059 e_dst: 2060 ccp_dm_free(&dst); 2061 2062 e_src: 2063 ccp_dm_free(&src); 2064 2065 return ret; 2066 } 2067 2068 static int ccp_run_ecc_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) 2069 { 2070 struct ccp_ecc_engine *ecc = &cmd->u.ecc; 2071 2072 ecc->ecc_result = 0; 2073 2074 if (!ecc->mod || 2075 (ecc->mod_len > CCP_ECC_MODULUS_BYTES)) 2076 return -EINVAL; 2077 2078 switch (ecc->function) { 2079 case CCP_ECC_FUNCTION_MMUL_384BIT: 2080 case CCP_ECC_FUNCTION_MADD_384BIT: 2081 case CCP_ECC_FUNCTION_MINV_384BIT: 2082 return ccp_run_ecc_mm_cmd(cmd_q, cmd); 2083 2084 case CCP_ECC_FUNCTION_PADD_384BIT: 2085 case CCP_ECC_FUNCTION_PMUL_384BIT: 2086 case CCP_ECC_FUNCTION_PDBL_384BIT: 2087 return ccp_run_ecc_pm_cmd(cmd_q, cmd); 2088 2089 default: 2090 return -EINVAL; 2091 } 2092 } 2093 2094 int ccp_run_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) 2095 { 2096 int ret; 2097 2098 cmd->engine_error = 0; 2099 cmd_q->cmd_error = 0; 2100 cmd_q->int_rcvd = 0; 2101 cmd_q->free_slots = CMD_Q_DEPTH(ioread32(cmd_q->reg_status)); 2102 2103 switch (cmd->engine) { 2104 case CCP_ENGINE_AES: 2105 ret = ccp_run_aes_cmd(cmd_q, cmd); 2106 break; 2107 case CCP_ENGINE_XTS_AES_128: 2108 ret = ccp_run_xts_aes_cmd(cmd_q, cmd); 2109 break; 2110 case CCP_ENGINE_SHA: 2111 ret = ccp_run_sha_cmd(cmd_q, cmd); 2112 break; 2113 case CCP_ENGINE_RSA: 2114 ret = ccp_run_rsa_cmd(cmd_q, cmd); 2115 break; 2116 case CCP_ENGINE_PASSTHRU: 2117 ret = ccp_run_passthru_cmd(cmd_q, cmd); 2118 break; 2119 case CCP_ENGINE_ECC: 2120 ret = ccp_run_ecc_cmd(cmd_q, cmd); 2121 break; 2122 default: 2123 ret = -EINVAL; 2124 } 2125 2126 return ret; 2127 } 2128