1 /* 2 * AMD Cryptographic Coprocessor (CCP) driver 3 * 4 * Copyright (C) 2013,2016 Advanced Micro Devices, Inc. 5 * 6 * Author: Tom Lendacky <thomas.lendacky@amd.com> 7 * Author: Gary R Hook <gary.hook@amd.com> 8 * 9 * This program is free software; you can redistribute it and/or modify 10 * it under the terms of the GNU General Public License version 2 as 11 * published by the Free Software Foundation. 12 */ 13 14 #include <linux/module.h> 15 #include <linux/kernel.h> 16 #include <linux/pci.h> 17 #include <linux/kthread.h> 18 #include <linux/interrupt.h> 19 #include <linux/ccp.h> 20 21 #include "ccp-dev.h" 22 23 static u32 ccp_alloc_ksb(struct ccp_cmd_queue *cmd_q, unsigned int count) 24 { 25 int start; 26 struct ccp_device *ccp = cmd_q->ccp; 27 28 for (;;) { 29 mutex_lock(&ccp->sb_mutex); 30 31 start = (u32)bitmap_find_next_zero_area(ccp->sb, 32 ccp->sb_count, 33 ccp->sb_start, 34 count, 0); 35 if (start <= ccp->sb_count) { 36 bitmap_set(ccp->sb, start, count); 37 38 mutex_unlock(&ccp->sb_mutex); 39 break; 40 } 41 42 ccp->sb_avail = 0; 43 44 mutex_unlock(&ccp->sb_mutex); 45 46 /* Wait for KSB entries to become available */ 47 if (wait_event_interruptible(ccp->sb_queue, ccp->sb_avail)) 48 return 0; 49 } 50 51 return KSB_START + start; 52 } 53 54 static void ccp_free_ksb(struct ccp_cmd_queue *cmd_q, unsigned int start, 55 unsigned int count) 56 { 57 struct ccp_device *ccp = cmd_q->ccp; 58 59 if (!start) 60 return; 61 62 mutex_lock(&ccp->sb_mutex); 63 64 bitmap_clear(ccp->sb, start - KSB_START, count); 65 66 ccp->sb_avail = 1; 67 68 mutex_unlock(&ccp->sb_mutex); 69 70 wake_up_interruptible_all(&ccp->sb_queue); 71 } 72 73 static unsigned int ccp_get_free_slots(struct ccp_cmd_queue *cmd_q) 74 { 75 return CMD_Q_DEPTH(ioread32(cmd_q->reg_status)); 76 } 77 78 static int ccp_do_cmd(struct ccp_op *op, u32 *cr, unsigned int cr_count) 79 { 80 struct ccp_cmd_queue *cmd_q = op->cmd_q; 81 struct ccp_device *ccp = cmd_q->ccp; 82 void __iomem *cr_addr; 83 u32 cr0, cmd; 84 unsigned int i; 85 int ret = 0; 86 87 /* We could read a status register to see how many free slots 88 * are actually available, but reading that register resets it 89 * and you could lose some error information. 90 */ 91 cmd_q->free_slots--; 92 93 cr0 = (cmd_q->id << REQ0_CMD_Q_SHIFT) 94 | (op->jobid << REQ0_JOBID_SHIFT) 95 | REQ0_WAIT_FOR_WRITE; 96 97 if (op->soc) 98 cr0 |= REQ0_STOP_ON_COMPLETE 99 | REQ0_INT_ON_COMPLETE; 100 101 if (op->ioc || !cmd_q->free_slots) 102 cr0 |= REQ0_INT_ON_COMPLETE; 103 104 /* Start at CMD_REQ1 */ 105 cr_addr = ccp->io_regs + CMD_REQ0 + CMD_REQ_INCR; 106 107 mutex_lock(&ccp->req_mutex); 108 109 /* Write CMD_REQ1 through CMD_REQx first */ 110 for (i = 0; i < cr_count; i++, cr_addr += CMD_REQ_INCR) 111 iowrite32(*(cr + i), cr_addr); 112 113 /* Tell the CCP to start */ 114 wmb(); 115 iowrite32(cr0, ccp->io_regs + CMD_REQ0); 116 117 mutex_unlock(&ccp->req_mutex); 118 119 if (cr0 & REQ0_INT_ON_COMPLETE) { 120 /* Wait for the job to complete */ 121 ret = wait_event_interruptible(cmd_q->int_queue, 122 cmd_q->int_rcvd); 123 if (ret || cmd_q->cmd_error) { 124 /* On error delete all related jobs from the queue */ 125 cmd = (cmd_q->id << DEL_Q_ID_SHIFT) 126 | op->jobid; 127 if (cmd_q->cmd_error) 128 ccp_log_error(cmd_q->ccp, 129 cmd_q->cmd_error); 130 131 iowrite32(cmd, ccp->io_regs + DEL_CMD_Q_JOB); 132 133 if (!ret) 134 ret = -EIO; 135 } else if (op->soc) { 136 /* Delete just head job from the queue on SoC */ 137 cmd = DEL_Q_ACTIVE 138 | (cmd_q->id << DEL_Q_ID_SHIFT) 139 | op->jobid; 140 141 iowrite32(cmd, ccp->io_regs + DEL_CMD_Q_JOB); 142 } 143 144 cmd_q->free_slots = CMD_Q_DEPTH(cmd_q->q_status); 145 146 cmd_q->int_rcvd = 0; 147 } 148 149 return ret; 150 } 151 152 static int ccp_perform_aes(struct ccp_op *op) 153 { 154 u32 cr[6]; 155 156 /* Fill out the register contents for REQ1 through REQ6 */ 157 cr[0] = (CCP_ENGINE_AES << REQ1_ENGINE_SHIFT) 158 | (op->u.aes.type << REQ1_AES_TYPE_SHIFT) 159 | (op->u.aes.mode << REQ1_AES_MODE_SHIFT) 160 | (op->u.aes.action << REQ1_AES_ACTION_SHIFT) 161 | (op->sb_key << REQ1_KEY_KSB_SHIFT); 162 cr[1] = op->src.u.dma.length - 1; 163 cr[2] = ccp_addr_lo(&op->src.u.dma); 164 cr[3] = (op->sb_ctx << REQ4_KSB_SHIFT) 165 | (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT) 166 | ccp_addr_hi(&op->src.u.dma); 167 cr[4] = ccp_addr_lo(&op->dst.u.dma); 168 cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT) 169 | ccp_addr_hi(&op->dst.u.dma); 170 171 if (op->u.aes.mode == CCP_AES_MODE_CFB) 172 cr[0] |= ((0x7f) << REQ1_AES_CFB_SIZE_SHIFT); 173 174 if (op->eom) 175 cr[0] |= REQ1_EOM; 176 177 if (op->init) 178 cr[0] |= REQ1_INIT; 179 180 return ccp_do_cmd(op, cr, ARRAY_SIZE(cr)); 181 } 182 183 static int ccp_perform_xts_aes(struct ccp_op *op) 184 { 185 u32 cr[6]; 186 187 /* Fill out the register contents for REQ1 through REQ6 */ 188 cr[0] = (CCP_ENGINE_XTS_AES_128 << REQ1_ENGINE_SHIFT) 189 | (op->u.xts.action << REQ1_AES_ACTION_SHIFT) 190 | (op->u.xts.unit_size << REQ1_XTS_AES_SIZE_SHIFT) 191 | (op->sb_key << REQ1_KEY_KSB_SHIFT); 192 cr[1] = op->src.u.dma.length - 1; 193 cr[2] = ccp_addr_lo(&op->src.u.dma); 194 cr[3] = (op->sb_ctx << REQ4_KSB_SHIFT) 195 | (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT) 196 | ccp_addr_hi(&op->src.u.dma); 197 cr[4] = ccp_addr_lo(&op->dst.u.dma); 198 cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT) 199 | ccp_addr_hi(&op->dst.u.dma); 200 201 if (op->eom) 202 cr[0] |= REQ1_EOM; 203 204 if (op->init) 205 cr[0] |= REQ1_INIT; 206 207 return ccp_do_cmd(op, cr, ARRAY_SIZE(cr)); 208 } 209 210 static int ccp_perform_sha(struct ccp_op *op) 211 { 212 u32 cr[6]; 213 214 /* Fill out the register contents for REQ1 through REQ6 */ 215 cr[0] = (CCP_ENGINE_SHA << REQ1_ENGINE_SHIFT) 216 | (op->u.sha.type << REQ1_SHA_TYPE_SHIFT) 217 | REQ1_INIT; 218 cr[1] = op->src.u.dma.length - 1; 219 cr[2] = ccp_addr_lo(&op->src.u.dma); 220 cr[3] = (op->sb_ctx << REQ4_KSB_SHIFT) 221 | (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT) 222 | ccp_addr_hi(&op->src.u.dma); 223 224 if (op->eom) { 225 cr[0] |= REQ1_EOM; 226 cr[4] = lower_32_bits(op->u.sha.msg_bits); 227 cr[5] = upper_32_bits(op->u.sha.msg_bits); 228 } else { 229 cr[4] = 0; 230 cr[5] = 0; 231 } 232 233 return ccp_do_cmd(op, cr, ARRAY_SIZE(cr)); 234 } 235 236 static int ccp_perform_rsa(struct ccp_op *op) 237 { 238 u32 cr[6]; 239 240 /* Fill out the register contents for REQ1 through REQ6 */ 241 cr[0] = (CCP_ENGINE_RSA << REQ1_ENGINE_SHIFT) 242 | (op->u.rsa.mod_size << REQ1_RSA_MOD_SIZE_SHIFT) 243 | (op->sb_key << REQ1_KEY_KSB_SHIFT) 244 | REQ1_EOM; 245 cr[1] = op->u.rsa.input_len - 1; 246 cr[2] = ccp_addr_lo(&op->src.u.dma); 247 cr[3] = (op->sb_ctx << REQ4_KSB_SHIFT) 248 | (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT) 249 | ccp_addr_hi(&op->src.u.dma); 250 cr[4] = ccp_addr_lo(&op->dst.u.dma); 251 cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT) 252 | ccp_addr_hi(&op->dst.u.dma); 253 254 return ccp_do_cmd(op, cr, ARRAY_SIZE(cr)); 255 } 256 257 static int ccp_perform_passthru(struct ccp_op *op) 258 { 259 u32 cr[6]; 260 261 /* Fill out the register contents for REQ1 through REQ6 */ 262 cr[0] = (CCP_ENGINE_PASSTHRU << REQ1_ENGINE_SHIFT) 263 | (op->u.passthru.bit_mod << REQ1_PT_BW_SHIFT) 264 | (op->u.passthru.byte_swap << REQ1_PT_BS_SHIFT); 265 266 if (op->src.type == CCP_MEMTYPE_SYSTEM) 267 cr[1] = op->src.u.dma.length - 1; 268 else 269 cr[1] = op->dst.u.dma.length - 1; 270 271 if (op->src.type == CCP_MEMTYPE_SYSTEM) { 272 cr[2] = ccp_addr_lo(&op->src.u.dma); 273 cr[3] = (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT) 274 | ccp_addr_hi(&op->src.u.dma); 275 276 if (op->u.passthru.bit_mod != CCP_PASSTHRU_BITWISE_NOOP) 277 cr[3] |= (op->sb_key << REQ4_KSB_SHIFT); 278 } else { 279 cr[2] = op->src.u.sb * CCP_SB_BYTES; 280 cr[3] = (CCP_MEMTYPE_SB << REQ4_MEMTYPE_SHIFT); 281 } 282 283 if (op->dst.type == CCP_MEMTYPE_SYSTEM) { 284 cr[4] = ccp_addr_lo(&op->dst.u.dma); 285 cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT) 286 | ccp_addr_hi(&op->dst.u.dma); 287 } else { 288 cr[4] = op->dst.u.sb * CCP_SB_BYTES; 289 cr[5] = (CCP_MEMTYPE_SB << REQ6_MEMTYPE_SHIFT); 290 } 291 292 if (op->eom) 293 cr[0] |= REQ1_EOM; 294 295 return ccp_do_cmd(op, cr, ARRAY_SIZE(cr)); 296 } 297 298 static int ccp_perform_ecc(struct ccp_op *op) 299 { 300 u32 cr[6]; 301 302 /* Fill out the register contents for REQ1 through REQ6 */ 303 cr[0] = REQ1_ECC_AFFINE_CONVERT 304 | (CCP_ENGINE_ECC << REQ1_ENGINE_SHIFT) 305 | (op->u.ecc.function << REQ1_ECC_FUNCTION_SHIFT) 306 | REQ1_EOM; 307 cr[1] = op->src.u.dma.length - 1; 308 cr[2] = ccp_addr_lo(&op->src.u.dma); 309 cr[3] = (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT) 310 | ccp_addr_hi(&op->src.u.dma); 311 cr[4] = ccp_addr_lo(&op->dst.u.dma); 312 cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT) 313 | ccp_addr_hi(&op->dst.u.dma); 314 315 return ccp_do_cmd(op, cr, ARRAY_SIZE(cr)); 316 } 317 318 static int ccp_init(struct ccp_device *ccp) 319 { 320 struct device *dev = ccp->dev; 321 struct ccp_cmd_queue *cmd_q; 322 struct dma_pool *dma_pool; 323 char dma_pool_name[MAX_DMAPOOL_NAME_LEN]; 324 unsigned int qmr, qim, i; 325 int ret; 326 327 /* Find available queues */ 328 qim = 0; 329 qmr = ioread32(ccp->io_regs + Q_MASK_REG); 330 for (i = 0; i < MAX_HW_QUEUES; i++) { 331 if (!(qmr & (1 << i))) 332 continue; 333 334 /* Allocate a dma pool for this queue */ 335 snprintf(dma_pool_name, sizeof(dma_pool_name), "%s_q%d", 336 ccp->name, i); 337 dma_pool = dma_pool_create(dma_pool_name, dev, 338 CCP_DMAPOOL_MAX_SIZE, 339 CCP_DMAPOOL_ALIGN, 0); 340 if (!dma_pool) { 341 dev_err(dev, "unable to allocate dma pool\n"); 342 ret = -ENOMEM; 343 goto e_pool; 344 } 345 346 cmd_q = &ccp->cmd_q[ccp->cmd_q_count]; 347 ccp->cmd_q_count++; 348 349 cmd_q->ccp = ccp; 350 cmd_q->id = i; 351 cmd_q->dma_pool = dma_pool; 352 353 /* Reserve 2 KSB regions for the queue */ 354 cmd_q->sb_key = KSB_START + ccp->sb_start++; 355 cmd_q->sb_ctx = KSB_START + ccp->sb_start++; 356 ccp->sb_count -= 2; 357 358 /* Preset some register values and masks that are queue 359 * number dependent 360 */ 361 cmd_q->reg_status = ccp->io_regs + CMD_Q_STATUS_BASE + 362 (CMD_Q_STATUS_INCR * i); 363 cmd_q->reg_int_status = ccp->io_regs + CMD_Q_INT_STATUS_BASE + 364 (CMD_Q_STATUS_INCR * i); 365 cmd_q->int_ok = 1 << (i * 2); 366 cmd_q->int_err = 1 << ((i * 2) + 1); 367 368 cmd_q->free_slots = ccp_get_free_slots(cmd_q); 369 370 init_waitqueue_head(&cmd_q->int_queue); 371 372 /* Build queue interrupt mask (two interrupts per queue) */ 373 qim |= cmd_q->int_ok | cmd_q->int_err; 374 375 #ifdef CONFIG_ARM64 376 /* For arm64 set the recommended queue cache settings */ 377 iowrite32(ccp->axcache, ccp->io_regs + CMD_Q_CACHE_BASE + 378 (CMD_Q_CACHE_INC * i)); 379 #endif 380 381 dev_dbg(dev, "queue #%u available\n", i); 382 } 383 if (ccp->cmd_q_count == 0) { 384 dev_notice(dev, "no command queues available\n"); 385 ret = -EIO; 386 goto e_pool; 387 } 388 dev_notice(dev, "%u command queues available\n", ccp->cmd_q_count); 389 390 /* Disable and clear interrupts until ready */ 391 iowrite32(0x00, ccp->io_regs + IRQ_MASK_REG); 392 for (i = 0; i < ccp->cmd_q_count; i++) { 393 cmd_q = &ccp->cmd_q[i]; 394 395 ioread32(cmd_q->reg_int_status); 396 ioread32(cmd_q->reg_status); 397 } 398 iowrite32(qim, ccp->io_regs + IRQ_STATUS_REG); 399 400 /* Request an irq */ 401 ret = ccp->get_irq(ccp); 402 if (ret) { 403 dev_err(dev, "unable to allocate an IRQ\n"); 404 goto e_pool; 405 } 406 407 dev_dbg(dev, "Starting threads...\n"); 408 /* Create a kthread for each queue */ 409 for (i = 0; i < ccp->cmd_q_count; i++) { 410 struct task_struct *kthread; 411 412 cmd_q = &ccp->cmd_q[i]; 413 414 kthread = kthread_create(ccp_cmd_queue_thread, cmd_q, 415 "%s-q%u", ccp->name, cmd_q->id); 416 if (IS_ERR(kthread)) { 417 dev_err(dev, "error creating queue thread (%ld)\n", 418 PTR_ERR(kthread)); 419 ret = PTR_ERR(kthread); 420 goto e_kthread; 421 } 422 423 cmd_q->kthread = kthread; 424 wake_up_process(kthread); 425 } 426 427 dev_dbg(dev, "Enabling interrupts...\n"); 428 /* Enable interrupts */ 429 iowrite32(qim, ccp->io_regs + IRQ_MASK_REG); 430 431 dev_dbg(dev, "Registering device...\n"); 432 ccp_add_device(ccp); 433 434 ret = ccp_register_rng(ccp); 435 if (ret) 436 goto e_kthread; 437 438 /* Register the DMA engine support */ 439 ret = ccp_dmaengine_register(ccp); 440 if (ret) 441 goto e_hwrng; 442 443 return 0; 444 445 e_hwrng: 446 ccp_unregister_rng(ccp); 447 448 e_kthread: 449 for (i = 0; i < ccp->cmd_q_count; i++) 450 if (ccp->cmd_q[i].kthread) 451 kthread_stop(ccp->cmd_q[i].kthread); 452 453 ccp->free_irq(ccp); 454 455 e_pool: 456 for (i = 0; i < ccp->cmd_q_count; i++) 457 dma_pool_destroy(ccp->cmd_q[i].dma_pool); 458 459 return ret; 460 } 461 462 static void ccp_destroy(struct ccp_device *ccp) 463 { 464 struct ccp_cmd_queue *cmd_q; 465 struct ccp_cmd *cmd; 466 unsigned int qim, i; 467 468 /* Unregister the DMA engine */ 469 ccp_dmaengine_unregister(ccp); 470 471 /* Unregister the RNG */ 472 ccp_unregister_rng(ccp); 473 474 /* Remove this device from the list of available units */ 475 ccp_del_device(ccp); 476 477 /* Build queue interrupt mask (two interrupt masks per queue) */ 478 qim = 0; 479 for (i = 0; i < ccp->cmd_q_count; i++) { 480 cmd_q = &ccp->cmd_q[i]; 481 qim |= cmd_q->int_ok | cmd_q->int_err; 482 } 483 484 /* Disable and clear interrupts */ 485 iowrite32(0x00, ccp->io_regs + IRQ_MASK_REG); 486 for (i = 0; i < ccp->cmd_q_count; i++) { 487 cmd_q = &ccp->cmd_q[i]; 488 489 ioread32(cmd_q->reg_int_status); 490 ioread32(cmd_q->reg_status); 491 } 492 iowrite32(qim, ccp->io_regs + IRQ_STATUS_REG); 493 494 /* Stop the queue kthreads */ 495 for (i = 0; i < ccp->cmd_q_count; i++) 496 if (ccp->cmd_q[i].kthread) 497 kthread_stop(ccp->cmd_q[i].kthread); 498 499 ccp->free_irq(ccp); 500 501 for (i = 0; i < ccp->cmd_q_count; i++) 502 dma_pool_destroy(ccp->cmd_q[i].dma_pool); 503 504 /* Flush the cmd and backlog queue */ 505 while (!list_empty(&ccp->cmd)) { 506 /* Invoke the callback directly with an error code */ 507 cmd = list_first_entry(&ccp->cmd, struct ccp_cmd, entry); 508 list_del(&cmd->entry); 509 cmd->callback(cmd->data, -ENODEV); 510 } 511 while (!list_empty(&ccp->backlog)) { 512 /* Invoke the callback directly with an error code */ 513 cmd = list_first_entry(&ccp->backlog, struct ccp_cmd, entry); 514 list_del(&cmd->entry); 515 cmd->callback(cmd->data, -ENODEV); 516 } 517 } 518 519 static irqreturn_t ccp_irq_handler(int irq, void *data) 520 { 521 struct device *dev = data; 522 struct ccp_device *ccp = dev_get_drvdata(dev); 523 struct ccp_cmd_queue *cmd_q; 524 u32 q_int, status; 525 unsigned int i; 526 527 status = ioread32(ccp->io_regs + IRQ_STATUS_REG); 528 529 for (i = 0; i < ccp->cmd_q_count; i++) { 530 cmd_q = &ccp->cmd_q[i]; 531 532 q_int = status & (cmd_q->int_ok | cmd_q->int_err); 533 if (q_int) { 534 cmd_q->int_status = status; 535 cmd_q->q_status = ioread32(cmd_q->reg_status); 536 cmd_q->q_int_status = ioread32(cmd_q->reg_int_status); 537 538 /* On error, only save the first error value */ 539 if ((q_int & cmd_q->int_err) && !cmd_q->cmd_error) 540 cmd_q->cmd_error = CMD_Q_ERROR(cmd_q->q_status); 541 542 cmd_q->int_rcvd = 1; 543 544 /* Acknowledge the interrupt and wake the kthread */ 545 iowrite32(q_int, ccp->io_regs + IRQ_STATUS_REG); 546 wake_up_interruptible(&cmd_q->int_queue); 547 } 548 } 549 550 return IRQ_HANDLED; 551 } 552 553 static const struct ccp_actions ccp3_actions = { 554 .aes = ccp_perform_aes, 555 .xts_aes = ccp_perform_xts_aes, 556 .sha = ccp_perform_sha, 557 .rsa = ccp_perform_rsa, 558 .passthru = ccp_perform_passthru, 559 .ecc = ccp_perform_ecc, 560 .sballoc = ccp_alloc_ksb, 561 .sbfree = ccp_free_ksb, 562 .init = ccp_init, 563 .destroy = ccp_destroy, 564 .get_free_slots = ccp_get_free_slots, 565 .irqhandler = ccp_irq_handler, 566 }; 567 568 const struct ccp_vdata ccpv3 = { 569 .version = CCP_VERSION(3, 0), 570 .setup = NULL, 571 .perform = &ccp3_actions, 572 .bar = 2, 573 .offset = 0x20000, 574 }; 575