1 /* 2 * AMD Cryptographic Coprocessor (CCP) driver 3 * 4 * Copyright (C) 2013,2016 Advanced Micro Devices, Inc. 5 * 6 * Author: Tom Lendacky <thomas.lendacky@amd.com> 7 * Author: Gary R Hook <gary.hook@amd.com> 8 * 9 * This program is free software; you can redistribute it and/or modify 10 * it under the terms of the GNU General Public License version 2 as 11 * published by the Free Software Foundation. 12 */ 13 14 #include <linux/module.h> 15 #include <linux/kernel.h> 16 #include <linux/kthread.h> 17 #include <linux/sched.h> 18 #include <linux/interrupt.h> 19 #include <linux/spinlock.h> 20 #include <linux/spinlock_types.h> 21 #include <linux/types.h> 22 #include <linux/mutex.h> 23 #include <linux/delay.h> 24 #include <linux/hw_random.h> 25 #include <linux/cpu.h> 26 #ifdef CONFIG_X86 27 #include <asm/cpu_device_id.h> 28 #endif 29 #include <linux/ccp.h> 30 31 #include "ccp-dev.h" 32 33 MODULE_AUTHOR("Tom Lendacky <thomas.lendacky@amd.com>"); 34 MODULE_LICENSE("GPL"); 35 MODULE_VERSION("1.0.0"); 36 MODULE_DESCRIPTION("AMD Cryptographic Coprocessor driver"); 37 38 struct ccp_tasklet_data { 39 struct completion completion; 40 struct ccp_cmd *cmd; 41 }; 42 43 /* Human-readable error strings */ 44 static char *ccp_error_codes[] = { 45 "", 46 "ERR 01: ILLEGAL_ENGINE", 47 "ERR 02: ILLEGAL_KEY_ID", 48 "ERR 03: ILLEGAL_FUNCTION_TYPE", 49 "ERR 04: ILLEGAL_FUNCTION_MODE", 50 "ERR 05: ILLEGAL_FUNCTION_ENCRYPT", 51 "ERR 06: ILLEGAL_FUNCTION_SIZE", 52 "ERR 07: Zlib_MISSING_INIT_EOM", 53 "ERR 08: ILLEGAL_FUNCTION_RSVD", 54 "ERR 09: ILLEGAL_BUFFER_LENGTH", 55 "ERR 10: VLSB_FAULT", 56 "ERR 11: ILLEGAL_MEM_ADDR", 57 "ERR 12: ILLEGAL_MEM_SEL", 58 "ERR 13: ILLEGAL_CONTEXT_ID", 59 "ERR 14: ILLEGAL_KEY_ADDR", 60 "ERR 15: 0xF Reserved", 61 "ERR 16: Zlib_ILLEGAL_MULTI_QUEUE", 62 "ERR 17: Zlib_ILLEGAL_JOBID_CHANGE", 63 "ERR 18: CMD_TIMEOUT", 64 "ERR 19: IDMA0_AXI_SLVERR", 65 "ERR 20: IDMA0_AXI_DECERR", 66 "ERR 21: 0x15 Reserved", 67 "ERR 22: IDMA1_AXI_SLAVE_FAULT", 68 "ERR 23: IDMA1_AIXI_DECERR", 69 "ERR 24: 0x18 Reserved", 70 "ERR 25: ZLIBVHB_AXI_SLVERR", 71 "ERR 26: ZLIBVHB_AXI_DECERR", 72 "ERR 27: 0x1B Reserved", 73 "ERR 27: ZLIB_UNEXPECTED_EOM", 74 "ERR 27: ZLIB_EXTRA_DATA", 75 "ERR 30: ZLIB_BTYPE", 76 "ERR 31: ZLIB_UNDEFINED_SYMBOL", 77 "ERR 32: ZLIB_UNDEFINED_DISTANCE_S", 78 "ERR 33: ZLIB_CODE_LENGTH_SYMBOL", 79 "ERR 34: ZLIB _VHB_ILLEGAL_FETCH", 80 "ERR 35: ZLIB_UNCOMPRESSED_LEN", 81 "ERR 36: ZLIB_LIMIT_REACHED", 82 "ERR 37: ZLIB_CHECKSUM_MISMATCH0", 83 "ERR 38: ODMA0_AXI_SLVERR", 84 "ERR 39: ODMA0_AXI_DECERR", 85 "ERR 40: 0x28 Reserved", 86 "ERR 41: ODMA1_AXI_SLVERR", 87 "ERR 42: ODMA1_AXI_DECERR", 88 "ERR 43: LSB_PARITY_ERR", 89 }; 90 91 void ccp_log_error(struct ccp_device *d, int e) 92 { 93 dev_err(d->dev, "CCP error: %s (0x%x)\n", ccp_error_codes[e], e); 94 } 95 96 /* List of CCPs, CCP count, read-write access lock, and access functions 97 * 98 * Lock structure: get ccp_unit_lock for reading whenever we need to 99 * examine the CCP list. While holding it for reading we can acquire 100 * the RR lock to update the round-robin next-CCP pointer. The unit lock 101 * must be acquired before the RR lock. 102 * 103 * If the unit-lock is acquired for writing, we have total control over 104 * the list, so there's no value in getting the RR lock. 105 */ 106 static DEFINE_RWLOCK(ccp_unit_lock); 107 static LIST_HEAD(ccp_units); 108 109 /* Round-robin counter */ 110 static DEFINE_SPINLOCK(ccp_rr_lock); 111 static struct ccp_device *ccp_rr; 112 113 /* Ever-increasing value to produce unique unit numbers */ 114 static atomic_t ccp_unit_ordinal; 115 static unsigned int ccp_increment_unit_ordinal(void) 116 { 117 return atomic_inc_return(&ccp_unit_ordinal); 118 } 119 120 /** 121 * ccp_add_device - add a CCP device to the list 122 * 123 * @ccp: ccp_device struct pointer 124 * 125 * Put this CCP on the unit list, which makes it available 126 * for use. 127 * 128 * Returns zero if a CCP device is present, -ENODEV otherwise. 129 */ 130 void ccp_add_device(struct ccp_device *ccp) 131 { 132 unsigned long flags; 133 134 write_lock_irqsave(&ccp_unit_lock, flags); 135 list_add_tail(&ccp->entry, &ccp_units); 136 if (!ccp_rr) 137 /* We already have the list lock (we're first) so this 138 * pointer can't change on us. Set its initial value. 139 */ 140 ccp_rr = ccp; 141 write_unlock_irqrestore(&ccp_unit_lock, flags); 142 } 143 144 /** 145 * ccp_del_device - remove a CCP device from the list 146 * 147 * @ccp: ccp_device struct pointer 148 * 149 * Remove this unit from the list of devices. If the next device 150 * up for use is this one, adjust the pointer. If this is the last 151 * device, NULL the pointer. 152 */ 153 void ccp_del_device(struct ccp_device *ccp) 154 { 155 unsigned long flags; 156 157 write_lock_irqsave(&ccp_unit_lock, flags); 158 if (ccp_rr == ccp) { 159 /* ccp_unit_lock is read/write; any read access 160 * will be suspended while we make changes to the 161 * list and RR pointer. 162 */ 163 if (list_is_last(&ccp_rr->entry, &ccp_units)) 164 ccp_rr = list_first_entry(&ccp_units, struct ccp_device, 165 entry); 166 else 167 ccp_rr = list_next_entry(ccp_rr, entry); 168 } 169 list_del(&ccp->entry); 170 if (list_empty(&ccp_units)) 171 ccp_rr = NULL; 172 write_unlock_irqrestore(&ccp_unit_lock, flags); 173 } 174 175 176 177 int ccp_register_rng(struct ccp_device *ccp) 178 { 179 int ret = 0; 180 181 dev_dbg(ccp->dev, "Registering RNG...\n"); 182 /* Register an RNG */ 183 ccp->hwrng.name = ccp->rngname; 184 ccp->hwrng.read = ccp_trng_read; 185 ret = hwrng_register(&ccp->hwrng); 186 if (ret) 187 dev_err(ccp->dev, "error registering hwrng (%d)\n", ret); 188 189 return ret; 190 } 191 192 void ccp_unregister_rng(struct ccp_device *ccp) 193 { 194 if (ccp->hwrng.name) 195 hwrng_unregister(&ccp->hwrng); 196 } 197 198 static struct ccp_device *ccp_get_device(void) 199 { 200 unsigned long flags; 201 struct ccp_device *dp = NULL; 202 203 /* We round-robin through the unit list. 204 * The (ccp_rr) pointer refers to the next unit to use. 205 */ 206 read_lock_irqsave(&ccp_unit_lock, flags); 207 if (!list_empty(&ccp_units)) { 208 spin_lock(&ccp_rr_lock); 209 dp = ccp_rr; 210 if (list_is_last(&ccp_rr->entry, &ccp_units)) 211 ccp_rr = list_first_entry(&ccp_units, struct ccp_device, 212 entry); 213 else 214 ccp_rr = list_next_entry(ccp_rr, entry); 215 spin_unlock(&ccp_rr_lock); 216 } 217 read_unlock_irqrestore(&ccp_unit_lock, flags); 218 219 return dp; 220 } 221 222 /** 223 * ccp_present - check if a CCP device is present 224 * 225 * Returns zero if a CCP device is present, -ENODEV otherwise. 226 */ 227 int ccp_present(void) 228 { 229 unsigned long flags; 230 int ret; 231 232 read_lock_irqsave(&ccp_unit_lock, flags); 233 ret = list_empty(&ccp_units); 234 read_unlock_irqrestore(&ccp_unit_lock, flags); 235 236 return ret ? -ENODEV : 0; 237 } 238 EXPORT_SYMBOL_GPL(ccp_present); 239 240 /** 241 * ccp_version - get the version of the CCP device 242 * 243 * Returns the version from the first unit on the list; 244 * otherwise a zero if no CCP device is present 245 */ 246 unsigned int ccp_version(void) 247 { 248 struct ccp_device *dp; 249 unsigned long flags; 250 int ret = 0; 251 252 read_lock_irqsave(&ccp_unit_lock, flags); 253 if (!list_empty(&ccp_units)) { 254 dp = list_first_entry(&ccp_units, struct ccp_device, entry); 255 ret = dp->vdata->version; 256 } 257 read_unlock_irqrestore(&ccp_unit_lock, flags); 258 259 return ret; 260 } 261 EXPORT_SYMBOL_GPL(ccp_version); 262 263 /** 264 * ccp_enqueue_cmd - queue an operation for processing by the CCP 265 * 266 * @cmd: ccp_cmd struct to be processed 267 * 268 * Queue a cmd to be processed by the CCP. If queueing the cmd 269 * would exceed the defined length of the cmd queue the cmd will 270 * only be queued if the CCP_CMD_MAY_BACKLOG flag is set and will 271 * result in a return code of -EBUSY. 272 * 273 * The callback routine specified in the ccp_cmd struct will be 274 * called to notify the caller of completion (if the cmd was not 275 * backlogged) or advancement out of the backlog. If the cmd has 276 * advanced out of the backlog the "err" value of the callback 277 * will be -EINPROGRESS. Any other "err" value during callback is 278 * the result of the operation. 279 * 280 * The cmd has been successfully queued if: 281 * the return code is -EINPROGRESS or 282 * the return code is -EBUSY and CCP_CMD_MAY_BACKLOG flag is set 283 */ 284 int ccp_enqueue_cmd(struct ccp_cmd *cmd) 285 { 286 struct ccp_device *ccp; 287 unsigned long flags; 288 unsigned int i; 289 int ret; 290 291 /* Some commands might need to be sent to a specific device */ 292 ccp = cmd->ccp ? cmd->ccp : ccp_get_device(); 293 294 if (!ccp) 295 return -ENODEV; 296 297 /* Caller must supply a callback routine */ 298 if (!cmd->callback) 299 return -EINVAL; 300 301 cmd->ccp = ccp; 302 303 spin_lock_irqsave(&ccp->cmd_lock, flags); 304 305 i = ccp->cmd_q_count; 306 307 if (ccp->cmd_count >= MAX_CMD_QLEN) { 308 ret = -EBUSY; 309 if (cmd->flags & CCP_CMD_MAY_BACKLOG) 310 list_add_tail(&cmd->entry, &ccp->backlog); 311 } else { 312 ret = -EINPROGRESS; 313 ccp->cmd_count++; 314 list_add_tail(&cmd->entry, &ccp->cmd); 315 316 /* Find an idle queue */ 317 if (!ccp->suspending) { 318 for (i = 0; i < ccp->cmd_q_count; i++) { 319 if (ccp->cmd_q[i].active) 320 continue; 321 322 break; 323 } 324 } 325 } 326 327 spin_unlock_irqrestore(&ccp->cmd_lock, flags); 328 329 /* If we found an idle queue, wake it up */ 330 if (i < ccp->cmd_q_count) 331 wake_up_process(ccp->cmd_q[i].kthread); 332 333 return ret; 334 } 335 EXPORT_SYMBOL_GPL(ccp_enqueue_cmd); 336 337 static void ccp_do_cmd_backlog(struct work_struct *work) 338 { 339 struct ccp_cmd *cmd = container_of(work, struct ccp_cmd, work); 340 struct ccp_device *ccp = cmd->ccp; 341 unsigned long flags; 342 unsigned int i; 343 344 cmd->callback(cmd->data, -EINPROGRESS); 345 346 spin_lock_irqsave(&ccp->cmd_lock, flags); 347 348 ccp->cmd_count++; 349 list_add_tail(&cmd->entry, &ccp->cmd); 350 351 /* Find an idle queue */ 352 for (i = 0; i < ccp->cmd_q_count; i++) { 353 if (ccp->cmd_q[i].active) 354 continue; 355 356 break; 357 } 358 359 spin_unlock_irqrestore(&ccp->cmd_lock, flags); 360 361 /* If we found an idle queue, wake it up */ 362 if (i < ccp->cmd_q_count) 363 wake_up_process(ccp->cmd_q[i].kthread); 364 } 365 366 static struct ccp_cmd *ccp_dequeue_cmd(struct ccp_cmd_queue *cmd_q) 367 { 368 struct ccp_device *ccp = cmd_q->ccp; 369 struct ccp_cmd *cmd = NULL; 370 struct ccp_cmd *backlog = NULL; 371 unsigned long flags; 372 373 spin_lock_irqsave(&ccp->cmd_lock, flags); 374 375 cmd_q->active = 0; 376 377 if (ccp->suspending) { 378 cmd_q->suspended = 1; 379 380 spin_unlock_irqrestore(&ccp->cmd_lock, flags); 381 wake_up_interruptible(&ccp->suspend_queue); 382 383 return NULL; 384 } 385 386 if (ccp->cmd_count) { 387 cmd_q->active = 1; 388 389 cmd = list_first_entry(&ccp->cmd, struct ccp_cmd, entry); 390 list_del(&cmd->entry); 391 392 ccp->cmd_count--; 393 } 394 395 if (!list_empty(&ccp->backlog)) { 396 backlog = list_first_entry(&ccp->backlog, struct ccp_cmd, 397 entry); 398 list_del(&backlog->entry); 399 } 400 401 spin_unlock_irqrestore(&ccp->cmd_lock, flags); 402 403 if (backlog) { 404 INIT_WORK(&backlog->work, ccp_do_cmd_backlog); 405 schedule_work(&backlog->work); 406 } 407 408 return cmd; 409 } 410 411 static void ccp_do_cmd_complete(unsigned long data) 412 { 413 struct ccp_tasklet_data *tdata = (struct ccp_tasklet_data *)data; 414 struct ccp_cmd *cmd = tdata->cmd; 415 416 cmd->callback(cmd->data, cmd->ret); 417 complete(&tdata->completion); 418 } 419 420 /** 421 * ccp_cmd_queue_thread - create a kernel thread to manage a CCP queue 422 * 423 * @data: thread-specific data 424 */ 425 int ccp_cmd_queue_thread(void *data) 426 { 427 struct ccp_cmd_queue *cmd_q = (struct ccp_cmd_queue *)data; 428 struct ccp_cmd *cmd; 429 struct ccp_tasklet_data tdata; 430 struct tasklet_struct tasklet; 431 432 tasklet_init(&tasklet, ccp_do_cmd_complete, (unsigned long)&tdata); 433 434 set_current_state(TASK_INTERRUPTIBLE); 435 while (!kthread_should_stop()) { 436 schedule(); 437 438 set_current_state(TASK_INTERRUPTIBLE); 439 440 cmd = ccp_dequeue_cmd(cmd_q); 441 if (!cmd) 442 continue; 443 444 __set_current_state(TASK_RUNNING); 445 446 /* Execute the command */ 447 cmd->ret = ccp_run_cmd(cmd_q, cmd); 448 449 /* Schedule the completion callback */ 450 tdata.cmd = cmd; 451 init_completion(&tdata.completion); 452 tasklet_schedule(&tasklet); 453 wait_for_completion(&tdata.completion); 454 } 455 456 __set_current_state(TASK_RUNNING); 457 458 return 0; 459 } 460 461 /** 462 * ccp_alloc_struct - allocate and initialize the ccp_device struct 463 * 464 * @dev: device struct of the CCP 465 */ 466 struct ccp_device *ccp_alloc_struct(struct device *dev) 467 { 468 struct ccp_device *ccp; 469 470 ccp = devm_kzalloc(dev, sizeof(*ccp), GFP_KERNEL); 471 if (!ccp) 472 return NULL; 473 ccp->dev = dev; 474 475 INIT_LIST_HEAD(&ccp->cmd); 476 INIT_LIST_HEAD(&ccp->backlog); 477 478 spin_lock_init(&ccp->cmd_lock); 479 mutex_init(&ccp->req_mutex); 480 mutex_init(&ccp->sb_mutex); 481 ccp->sb_count = KSB_COUNT; 482 ccp->sb_start = 0; 483 484 /* Initialize the wait queues */ 485 init_waitqueue_head(&ccp->sb_queue); 486 init_waitqueue_head(&ccp->suspend_queue); 487 488 ccp->ord = ccp_increment_unit_ordinal(); 489 snprintf(ccp->name, MAX_CCP_NAME_LEN, "ccp-%u", ccp->ord); 490 snprintf(ccp->rngname, MAX_CCP_NAME_LEN, "ccp-%u-rng", ccp->ord); 491 492 return ccp; 493 } 494 495 int ccp_trng_read(struct hwrng *rng, void *data, size_t max, bool wait) 496 { 497 struct ccp_device *ccp = container_of(rng, struct ccp_device, hwrng); 498 u32 trng_value; 499 int len = min_t(int, sizeof(trng_value), max); 500 501 /* Locking is provided by the caller so we can update device 502 * hwrng-related fields safely 503 */ 504 trng_value = ioread32(ccp->io_regs + TRNG_OUT_REG); 505 if (!trng_value) { 506 /* Zero is returned if not data is available or if a 507 * bad-entropy error is present. Assume an error if 508 * we exceed TRNG_RETRIES reads of zero. 509 */ 510 if (ccp->hwrng_retries++ > TRNG_RETRIES) 511 return -EIO; 512 513 return 0; 514 } 515 516 /* Reset the counter and save the rng value */ 517 ccp->hwrng_retries = 0; 518 memcpy(data, &trng_value, len); 519 520 return len; 521 } 522 523 #ifdef CONFIG_PM 524 bool ccp_queues_suspended(struct ccp_device *ccp) 525 { 526 unsigned int suspended = 0; 527 unsigned long flags; 528 unsigned int i; 529 530 spin_lock_irqsave(&ccp->cmd_lock, flags); 531 532 for (i = 0; i < ccp->cmd_q_count; i++) 533 if (ccp->cmd_q[i].suspended) 534 suspended++; 535 536 spin_unlock_irqrestore(&ccp->cmd_lock, flags); 537 538 return ccp->cmd_q_count == suspended; 539 } 540 #endif 541 542 static int __init ccp_mod_init(void) 543 { 544 #ifdef CONFIG_X86 545 int ret; 546 547 ret = ccp_pci_init(); 548 if (ret) 549 return ret; 550 551 /* Don't leave the driver loaded if init failed */ 552 if (ccp_present() != 0) { 553 ccp_pci_exit(); 554 return -ENODEV; 555 } 556 557 return 0; 558 #endif 559 560 #ifdef CONFIG_ARM64 561 int ret; 562 563 ret = ccp_platform_init(); 564 if (ret) 565 return ret; 566 567 /* Don't leave the driver loaded if init failed */ 568 if (ccp_present() != 0) { 569 ccp_platform_exit(); 570 return -ENODEV; 571 } 572 573 return 0; 574 #endif 575 576 return -ENODEV; 577 } 578 579 static void __exit ccp_mod_exit(void) 580 { 581 #ifdef CONFIG_X86 582 ccp_pci_exit(); 583 #endif 584 585 #ifdef CONFIG_ARM64 586 ccp_platform_exit(); 587 #endif 588 } 589 590 module_init(ccp_mod_init); 591 module_exit(ccp_mod_exit); 592