1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * AMD Cryptographic Coprocessor (CCP) driver 4 * 5 * Copyright (C) 2013,2017 Advanced Micro Devices, Inc. 6 * 7 * Author: Tom Lendacky <thomas.lendacky@amd.com> 8 * Author: Gary R Hook <gary.hook@amd.com> 9 */ 10 11 #include <linux/module.h> 12 #include <linux/kernel.h> 13 #include <linux/kthread.h> 14 #include <linux/interrupt.h> 15 #include <linux/ccp.h> 16 17 #include "ccp-dev.h" 18 19 static u32 ccp_alloc_ksb(struct ccp_cmd_queue *cmd_q, unsigned int count) 20 { 21 int start; 22 struct ccp_device *ccp = cmd_q->ccp; 23 24 for (;;) { 25 mutex_lock(&ccp->sb_mutex); 26 27 start = (u32)bitmap_find_next_zero_area(ccp->sb, 28 ccp->sb_count, 29 ccp->sb_start, 30 count, 0); 31 if (start <= ccp->sb_count) { 32 bitmap_set(ccp->sb, start, count); 33 34 mutex_unlock(&ccp->sb_mutex); 35 break; 36 } 37 38 ccp->sb_avail = 0; 39 40 mutex_unlock(&ccp->sb_mutex); 41 42 /* Wait for KSB entries to become available */ 43 if (wait_event_interruptible(ccp->sb_queue, ccp->sb_avail)) 44 return 0; 45 } 46 47 return KSB_START + start; 48 } 49 50 static void ccp_free_ksb(struct ccp_cmd_queue *cmd_q, unsigned int start, 51 unsigned int count) 52 { 53 struct ccp_device *ccp = cmd_q->ccp; 54 55 if (!start) 56 return; 57 58 mutex_lock(&ccp->sb_mutex); 59 60 bitmap_clear(ccp->sb, start - KSB_START, count); 61 62 ccp->sb_avail = 1; 63 64 mutex_unlock(&ccp->sb_mutex); 65 66 wake_up_interruptible_all(&ccp->sb_queue); 67 } 68 69 static unsigned int ccp_get_free_slots(struct ccp_cmd_queue *cmd_q) 70 { 71 return CMD_Q_DEPTH(ioread32(cmd_q->reg_status)); 72 } 73 74 static int ccp_do_cmd(struct ccp_op *op, u32 *cr, unsigned int cr_count) 75 { 76 struct ccp_cmd_queue *cmd_q = op->cmd_q; 77 struct ccp_device *ccp = cmd_q->ccp; 78 void __iomem *cr_addr; 79 u32 cr0, cmd; 80 unsigned int i; 81 int ret = 0; 82 83 /* We could read a status register to see how many free slots 84 * are actually available, but reading that register resets it 85 * and you could lose some error information. 86 */ 87 cmd_q->free_slots--; 88 89 cr0 = (cmd_q->id << REQ0_CMD_Q_SHIFT) 90 | (op->jobid << REQ0_JOBID_SHIFT) 91 | REQ0_WAIT_FOR_WRITE; 92 93 if (op->soc) 94 cr0 |= REQ0_STOP_ON_COMPLETE 95 | REQ0_INT_ON_COMPLETE; 96 97 if (op->ioc || !cmd_q->free_slots) 98 cr0 |= REQ0_INT_ON_COMPLETE; 99 100 /* Start at CMD_REQ1 */ 101 cr_addr = ccp->io_regs + CMD_REQ0 + CMD_REQ_INCR; 102 103 mutex_lock(&ccp->req_mutex); 104 105 /* Write CMD_REQ1 through CMD_REQx first */ 106 for (i = 0; i < cr_count; i++, cr_addr += CMD_REQ_INCR) 107 iowrite32(*(cr + i), cr_addr); 108 109 /* Tell the CCP to start */ 110 wmb(); 111 iowrite32(cr0, ccp->io_regs + CMD_REQ0); 112 113 mutex_unlock(&ccp->req_mutex); 114 115 if (cr0 & REQ0_INT_ON_COMPLETE) { 116 /* Wait for the job to complete */ 117 ret = wait_event_interruptible(cmd_q->int_queue, 118 cmd_q->int_rcvd); 119 if (ret || cmd_q->cmd_error) { 120 /* On error delete all related jobs from the queue */ 121 cmd = (cmd_q->id << DEL_Q_ID_SHIFT) 122 | op->jobid; 123 if (cmd_q->cmd_error) 124 ccp_log_error(cmd_q->ccp, 125 cmd_q->cmd_error); 126 127 iowrite32(cmd, ccp->io_regs + DEL_CMD_Q_JOB); 128 129 if (!ret) 130 ret = -EIO; 131 } else if (op->soc) { 132 /* Delete just head job from the queue on SoC */ 133 cmd = DEL_Q_ACTIVE 134 | (cmd_q->id << DEL_Q_ID_SHIFT) 135 | op->jobid; 136 137 iowrite32(cmd, ccp->io_regs + DEL_CMD_Q_JOB); 138 } 139 140 cmd_q->free_slots = CMD_Q_DEPTH(cmd_q->q_status); 141 142 cmd_q->int_rcvd = 0; 143 } 144 145 return ret; 146 } 147 148 static int ccp_perform_aes(struct ccp_op *op) 149 { 150 u32 cr[6]; 151 152 /* Fill out the register contents for REQ1 through REQ6 */ 153 cr[0] = (CCP_ENGINE_AES << REQ1_ENGINE_SHIFT) 154 | (op->u.aes.type << REQ1_AES_TYPE_SHIFT) 155 | (op->u.aes.mode << REQ1_AES_MODE_SHIFT) 156 | (op->u.aes.action << REQ1_AES_ACTION_SHIFT) 157 | (op->sb_key << REQ1_KEY_KSB_SHIFT); 158 cr[1] = op->src.u.dma.length - 1; 159 cr[2] = ccp_addr_lo(&op->src.u.dma); 160 cr[3] = (op->sb_ctx << REQ4_KSB_SHIFT) 161 | (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT) 162 | ccp_addr_hi(&op->src.u.dma); 163 cr[4] = ccp_addr_lo(&op->dst.u.dma); 164 cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT) 165 | ccp_addr_hi(&op->dst.u.dma); 166 167 if (op->u.aes.mode == CCP_AES_MODE_CFB) 168 cr[0] |= ((0x7f) << REQ1_AES_CFB_SIZE_SHIFT); 169 170 if (op->eom) 171 cr[0] |= REQ1_EOM; 172 173 if (op->init) 174 cr[0] |= REQ1_INIT; 175 176 return ccp_do_cmd(op, cr, ARRAY_SIZE(cr)); 177 } 178 179 static int ccp_perform_xts_aes(struct ccp_op *op) 180 { 181 u32 cr[6]; 182 183 /* Fill out the register contents for REQ1 through REQ6 */ 184 cr[0] = (CCP_ENGINE_XTS_AES_128 << REQ1_ENGINE_SHIFT) 185 | (op->u.xts.action << REQ1_AES_ACTION_SHIFT) 186 | (op->u.xts.unit_size << REQ1_XTS_AES_SIZE_SHIFT) 187 | (op->sb_key << REQ1_KEY_KSB_SHIFT); 188 cr[1] = op->src.u.dma.length - 1; 189 cr[2] = ccp_addr_lo(&op->src.u.dma); 190 cr[3] = (op->sb_ctx << REQ4_KSB_SHIFT) 191 | (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT) 192 | ccp_addr_hi(&op->src.u.dma); 193 cr[4] = ccp_addr_lo(&op->dst.u.dma); 194 cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT) 195 | ccp_addr_hi(&op->dst.u.dma); 196 197 if (op->eom) 198 cr[0] |= REQ1_EOM; 199 200 if (op->init) 201 cr[0] |= REQ1_INIT; 202 203 return ccp_do_cmd(op, cr, ARRAY_SIZE(cr)); 204 } 205 206 static int ccp_perform_sha(struct ccp_op *op) 207 { 208 u32 cr[6]; 209 210 /* Fill out the register contents for REQ1 through REQ6 */ 211 cr[0] = (CCP_ENGINE_SHA << REQ1_ENGINE_SHIFT) 212 | (op->u.sha.type << REQ1_SHA_TYPE_SHIFT) 213 | REQ1_INIT; 214 cr[1] = op->src.u.dma.length - 1; 215 cr[2] = ccp_addr_lo(&op->src.u.dma); 216 cr[3] = (op->sb_ctx << REQ4_KSB_SHIFT) 217 | (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT) 218 | ccp_addr_hi(&op->src.u.dma); 219 220 if (op->eom) { 221 cr[0] |= REQ1_EOM; 222 cr[4] = lower_32_bits(op->u.sha.msg_bits); 223 cr[5] = upper_32_bits(op->u.sha.msg_bits); 224 } else { 225 cr[4] = 0; 226 cr[5] = 0; 227 } 228 229 return ccp_do_cmd(op, cr, ARRAY_SIZE(cr)); 230 } 231 232 static int ccp_perform_rsa(struct ccp_op *op) 233 { 234 u32 cr[6]; 235 236 /* Fill out the register contents for REQ1 through REQ6 */ 237 cr[0] = (CCP_ENGINE_RSA << REQ1_ENGINE_SHIFT) 238 | (op->u.rsa.mod_size << REQ1_RSA_MOD_SIZE_SHIFT) 239 | (op->sb_key << REQ1_KEY_KSB_SHIFT) 240 | REQ1_EOM; 241 cr[1] = op->u.rsa.input_len - 1; 242 cr[2] = ccp_addr_lo(&op->src.u.dma); 243 cr[3] = (op->sb_ctx << REQ4_KSB_SHIFT) 244 | (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT) 245 | ccp_addr_hi(&op->src.u.dma); 246 cr[4] = ccp_addr_lo(&op->dst.u.dma); 247 cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT) 248 | ccp_addr_hi(&op->dst.u.dma); 249 250 return ccp_do_cmd(op, cr, ARRAY_SIZE(cr)); 251 } 252 253 static int ccp_perform_passthru(struct ccp_op *op) 254 { 255 u32 cr[6]; 256 257 /* Fill out the register contents for REQ1 through REQ6 */ 258 cr[0] = (CCP_ENGINE_PASSTHRU << REQ1_ENGINE_SHIFT) 259 | (op->u.passthru.bit_mod << REQ1_PT_BW_SHIFT) 260 | (op->u.passthru.byte_swap << REQ1_PT_BS_SHIFT); 261 262 if (op->src.type == CCP_MEMTYPE_SYSTEM) 263 cr[1] = op->src.u.dma.length - 1; 264 else 265 cr[1] = op->dst.u.dma.length - 1; 266 267 if (op->src.type == CCP_MEMTYPE_SYSTEM) { 268 cr[2] = ccp_addr_lo(&op->src.u.dma); 269 cr[3] = (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT) 270 | ccp_addr_hi(&op->src.u.dma); 271 272 if (op->u.passthru.bit_mod != CCP_PASSTHRU_BITWISE_NOOP) 273 cr[3] |= (op->sb_key << REQ4_KSB_SHIFT); 274 } else { 275 cr[2] = op->src.u.sb * CCP_SB_BYTES; 276 cr[3] = (CCP_MEMTYPE_SB << REQ4_MEMTYPE_SHIFT); 277 } 278 279 if (op->dst.type == CCP_MEMTYPE_SYSTEM) { 280 cr[4] = ccp_addr_lo(&op->dst.u.dma); 281 cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT) 282 | ccp_addr_hi(&op->dst.u.dma); 283 } else { 284 cr[4] = op->dst.u.sb * CCP_SB_BYTES; 285 cr[5] = (CCP_MEMTYPE_SB << REQ6_MEMTYPE_SHIFT); 286 } 287 288 if (op->eom) 289 cr[0] |= REQ1_EOM; 290 291 return ccp_do_cmd(op, cr, ARRAY_SIZE(cr)); 292 } 293 294 static int ccp_perform_ecc(struct ccp_op *op) 295 { 296 u32 cr[6]; 297 298 /* Fill out the register contents for REQ1 through REQ6 */ 299 cr[0] = REQ1_ECC_AFFINE_CONVERT 300 | (CCP_ENGINE_ECC << REQ1_ENGINE_SHIFT) 301 | (op->u.ecc.function << REQ1_ECC_FUNCTION_SHIFT) 302 | REQ1_EOM; 303 cr[1] = op->src.u.dma.length - 1; 304 cr[2] = ccp_addr_lo(&op->src.u.dma); 305 cr[3] = (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT) 306 | ccp_addr_hi(&op->src.u.dma); 307 cr[4] = ccp_addr_lo(&op->dst.u.dma); 308 cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT) 309 | ccp_addr_hi(&op->dst.u.dma); 310 311 return ccp_do_cmd(op, cr, ARRAY_SIZE(cr)); 312 } 313 314 static void ccp_disable_queue_interrupts(struct ccp_device *ccp) 315 { 316 iowrite32(0x00, ccp->io_regs + IRQ_MASK_REG); 317 } 318 319 static void ccp_enable_queue_interrupts(struct ccp_device *ccp) 320 { 321 iowrite32(ccp->qim, ccp->io_regs + IRQ_MASK_REG); 322 } 323 324 static void ccp_irq_bh(unsigned long data) 325 { 326 struct ccp_device *ccp = (struct ccp_device *)data; 327 struct ccp_cmd_queue *cmd_q; 328 u32 q_int, status; 329 unsigned int i; 330 331 status = ioread32(ccp->io_regs + IRQ_STATUS_REG); 332 333 for (i = 0; i < ccp->cmd_q_count; i++) { 334 cmd_q = &ccp->cmd_q[i]; 335 336 q_int = status & (cmd_q->int_ok | cmd_q->int_err); 337 if (q_int) { 338 cmd_q->int_status = status; 339 cmd_q->q_status = ioread32(cmd_q->reg_status); 340 cmd_q->q_int_status = ioread32(cmd_q->reg_int_status); 341 342 /* On error, only save the first error value */ 343 if ((q_int & cmd_q->int_err) && !cmd_q->cmd_error) 344 cmd_q->cmd_error = CMD_Q_ERROR(cmd_q->q_status); 345 346 cmd_q->int_rcvd = 1; 347 348 /* Acknowledge the interrupt and wake the kthread */ 349 iowrite32(q_int, ccp->io_regs + IRQ_STATUS_REG); 350 wake_up_interruptible(&cmd_q->int_queue); 351 } 352 } 353 ccp_enable_queue_interrupts(ccp); 354 } 355 356 static irqreturn_t ccp_irq_handler(int irq, void *data) 357 { 358 struct ccp_device *ccp = (struct ccp_device *)data; 359 360 ccp_disable_queue_interrupts(ccp); 361 if (ccp->use_tasklet) 362 tasklet_schedule(&ccp->irq_tasklet); 363 else 364 ccp_irq_bh((unsigned long)ccp); 365 366 return IRQ_HANDLED; 367 } 368 369 static int ccp_init(struct ccp_device *ccp) 370 { 371 struct device *dev = ccp->dev; 372 struct ccp_cmd_queue *cmd_q; 373 struct dma_pool *dma_pool; 374 char dma_pool_name[MAX_DMAPOOL_NAME_LEN]; 375 unsigned int qmr, i; 376 int ret; 377 378 /* Find available queues */ 379 ccp->qim = 0; 380 qmr = ioread32(ccp->io_regs + Q_MASK_REG); 381 for (i = 0; (i < MAX_HW_QUEUES) && (ccp->cmd_q_count < ccp->max_q_count); i++) { 382 if (!(qmr & (1 << i))) 383 continue; 384 385 /* Allocate a dma pool for this queue */ 386 snprintf(dma_pool_name, sizeof(dma_pool_name), "%s_q%d", 387 ccp->name, i); 388 dma_pool = dma_pool_create(dma_pool_name, dev, 389 CCP_DMAPOOL_MAX_SIZE, 390 CCP_DMAPOOL_ALIGN, 0); 391 if (!dma_pool) { 392 dev_err(dev, "unable to allocate dma pool\n"); 393 ret = -ENOMEM; 394 goto e_pool; 395 } 396 397 cmd_q = &ccp->cmd_q[ccp->cmd_q_count]; 398 ccp->cmd_q_count++; 399 400 cmd_q->ccp = ccp; 401 cmd_q->id = i; 402 cmd_q->dma_pool = dma_pool; 403 404 /* Reserve 2 KSB regions for the queue */ 405 cmd_q->sb_key = KSB_START + ccp->sb_start++; 406 cmd_q->sb_ctx = KSB_START + ccp->sb_start++; 407 ccp->sb_count -= 2; 408 409 /* Preset some register values and masks that are queue 410 * number dependent 411 */ 412 cmd_q->reg_status = ccp->io_regs + CMD_Q_STATUS_BASE + 413 (CMD_Q_STATUS_INCR * i); 414 cmd_q->reg_int_status = ccp->io_regs + CMD_Q_INT_STATUS_BASE + 415 (CMD_Q_STATUS_INCR * i); 416 cmd_q->int_ok = 1 << (i * 2); 417 cmd_q->int_err = 1 << ((i * 2) + 1); 418 419 cmd_q->free_slots = ccp_get_free_slots(cmd_q); 420 421 init_waitqueue_head(&cmd_q->int_queue); 422 423 /* Build queue interrupt mask (two interrupts per queue) */ 424 ccp->qim |= cmd_q->int_ok | cmd_q->int_err; 425 426 #ifdef CONFIG_ARM64 427 /* For arm64 set the recommended queue cache settings */ 428 iowrite32(ccp->axcache, ccp->io_regs + CMD_Q_CACHE_BASE + 429 (CMD_Q_CACHE_INC * i)); 430 #endif 431 432 dev_dbg(dev, "queue #%u available\n", i); 433 } 434 if (ccp->cmd_q_count == 0) { 435 dev_notice(dev, "no command queues available\n"); 436 ret = -EIO; 437 goto e_pool; 438 } 439 dev_notice(dev, "%u command queues available\n", ccp->cmd_q_count); 440 441 /* Disable and clear interrupts until ready */ 442 ccp_disable_queue_interrupts(ccp); 443 for (i = 0; i < ccp->cmd_q_count; i++) { 444 cmd_q = &ccp->cmd_q[i]; 445 446 ioread32(cmd_q->reg_int_status); 447 ioread32(cmd_q->reg_status); 448 } 449 iowrite32(ccp->qim, ccp->io_regs + IRQ_STATUS_REG); 450 451 /* Request an irq */ 452 ret = sp_request_ccp_irq(ccp->sp, ccp_irq_handler, ccp->name, ccp); 453 if (ret) { 454 dev_err(dev, "unable to allocate an IRQ\n"); 455 goto e_pool; 456 } 457 458 /* Initialize the ISR tasklet? */ 459 if (ccp->use_tasklet) 460 tasklet_init(&ccp->irq_tasklet, ccp_irq_bh, 461 (unsigned long)ccp); 462 463 dev_dbg(dev, "Starting threads...\n"); 464 /* Create a kthread for each queue */ 465 for (i = 0; i < ccp->cmd_q_count; i++) { 466 struct task_struct *kthread; 467 468 cmd_q = &ccp->cmd_q[i]; 469 470 kthread = kthread_run(ccp_cmd_queue_thread, cmd_q, 471 "%s-q%u", ccp->name, cmd_q->id); 472 if (IS_ERR(kthread)) { 473 dev_err(dev, "error creating queue thread (%ld)\n", 474 PTR_ERR(kthread)); 475 ret = PTR_ERR(kthread); 476 goto e_kthread; 477 } 478 479 cmd_q->kthread = kthread; 480 } 481 482 dev_dbg(dev, "Enabling interrupts...\n"); 483 /* Enable interrupts */ 484 ccp_enable_queue_interrupts(ccp); 485 486 dev_dbg(dev, "Registering device...\n"); 487 ccp_add_device(ccp); 488 489 ret = ccp_register_rng(ccp); 490 if (ret) 491 goto e_kthread; 492 493 /* Register the DMA engine support */ 494 ret = ccp_dmaengine_register(ccp); 495 if (ret) 496 goto e_hwrng; 497 498 return 0; 499 500 e_hwrng: 501 ccp_unregister_rng(ccp); 502 503 e_kthread: 504 for (i = 0; i < ccp->cmd_q_count; i++) 505 if (ccp->cmd_q[i].kthread) 506 kthread_stop(ccp->cmd_q[i].kthread); 507 508 sp_free_ccp_irq(ccp->sp, ccp); 509 510 e_pool: 511 for (i = 0; i < ccp->cmd_q_count; i++) 512 dma_pool_destroy(ccp->cmd_q[i].dma_pool); 513 514 return ret; 515 } 516 517 static void ccp_destroy(struct ccp_device *ccp) 518 { 519 struct ccp_cmd_queue *cmd_q; 520 struct ccp_cmd *cmd; 521 unsigned int i; 522 523 /* Unregister the DMA engine */ 524 ccp_dmaengine_unregister(ccp); 525 526 /* Unregister the RNG */ 527 ccp_unregister_rng(ccp); 528 529 /* Remove this device from the list of available units */ 530 ccp_del_device(ccp); 531 532 /* Disable and clear interrupts */ 533 ccp_disable_queue_interrupts(ccp); 534 for (i = 0; i < ccp->cmd_q_count; i++) { 535 cmd_q = &ccp->cmd_q[i]; 536 537 ioread32(cmd_q->reg_int_status); 538 ioread32(cmd_q->reg_status); 539 } 540 iowrite32(ccp->qim, ccp->io_regs + IRQ_STATUS_REG); 541 542 /* Stop the queue kthreads */ 543 for (i = 0; i < ccp->cmd_q_count; i++) 544 if (ccp->cmd_q[i].kthread) 545 kthread_stop(ccp->cmd_q[i].kthread); 546 547 sp_free_ccp_irq(ccp->sp, ccp); 548 549 for (i = 0; i < ccp->cmd_q_count; i++) 550 dma_pool_destroy(ccp->cmd_q[i].dma_pool); 551 552 /* Flush the cmd and backlog queue */ 553 while (!list_empty(&ccp->cmd)) { 554 /* Invoke the callback directly with an error code */ 555 cmd = list_first_entry(&ccp->cmd, struct ccp_cmd, entry); 556 list_del(&cmd->entry); 557 cmd->callback(cmd->data, -ENODEV); 558 } 559 while (!list_empty(&ccp->backlog)) { 560 /* Invoke the callback directly with an error code */ 561 cmd = list_first_entry(&ccp->backlog, struct ccp_cmd, entry); 562 list_del(&cmd->entry); 563 cmd->callback(cmd->data, -ENODEV); 564 } 565 } 566 567 static const struct ccp_actions ccp3_actions = { 568 .aes = ccp_perform_aes, 569 .xts_aes = ccp_perform_xts_aes, 570 .des3 = NULL, 571 .sha = ccp_perform_sha, 572 .rsa = ccp_perform_rsa, 573 .passthru = ccp_perform_passthru, 574 .ecc = ccp_perform_ecc, 575 .sballoc = ccp_alloc_ksb, 576 .sbfree = ccp_free_ksb, 577 .init = ccp_init, 578 .destroy = ccp_destroy, 579 .get_free_slots = ccp_get_free_slots, 580 .irqhandler = ccp_irq_handler, 581 }; 582 583 const struct ccp_vdata ccpv3_platform = { 584 .version = CCP_VERSION(3, 0), 585 .setup = NULL, 586 .perform = &ccp3_actions, 587 .offset = 0, 588 .rsamax = CCP_RSA_MAX_WIDTH, 589 }; 590 591 const struct ccp_vdata ccpv3 = { 592 .version = CCP_VERSION(3, 0), 593 .setup = NULL, 594 .perform = &ccp3_actions, 595 .offset = 0x20000, 596 .rsamax = CCP_RSA_MAX_WIDTH, 597 }; 598