1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * AMD Secure Encrypted Virtualization (SEV) interface 4 * 5 * Copyright (C) 2016,2019 Advanced Micro Devices, Inc. 6 * 7 * Author: Brijesh Singh <brijesh.singh@amd.com> 8 */ 9 10 #include <linux/module.h> 11 #include <linux/kernel.h> 12 #include <linux/kthread.h> 13 #include <linux/sched.h> 14 #include <linux/interrupt.h> 15 #include <linux/spinlock.h> 16 #include <linux/spinlock_types.h> 17 #include <linux/types.h> 18 #include <linux/mutex.h> 19 #include <linux/delay.h> 20 #include <linux/hw_random.h> 21 #include <linux/ccp.h> 22 #include <linux/firmware.h> 23 #include <linux/gfp.h> 24 #include <linux/cpufeature.h> 25 #include <linux/fs.h> 26 #include <linux/fs_struct.h> 27 28 #include <asm/smp.h> 29 #include <asm/cacheflush.h> 30 31 #include "psp-dev.h" 32 #include "sev-dev.h" 33 34 #define DEVICE_NAME "sev" 35 #define SEV_FW_FILE "amd/sev.fw" 36 #define SEV_FW_NAME_SIZE 64 37 38 static DEFINE_MUTEX(sev_cmd_mutex); 39 static struct sev_misc_dev *misc_dev; 40 41 static int psp_cmd_timeout = 100; 42 module_param(psp_cmd_timeout, int, 0644); 43 MODULE_PARM_DESC(psp_cmd_timeout, " default timeout value, in seconds, for PSP commands"); 44 45 static int psp_probe_timeout = 5; 46 module_param(psp_probe_timeout, int, 0644); 47 MODULE_PARM_DESC(psp_probe_timeout, " default timeout value, in seconds, during PSP device probe"); 48 49 static char *init_ex_path; 50 module_param(init_ex_path, charp, 0444); 51 MODULE_PARM_DESC(init_ex_path, " Path for INIT_EX data; if set try INIT_EX"); 52 53 static bool psp_init_on_probe = true; 54 module_param(psp_init_on_probe, bool, 0444); 55 MODULE_PARM_DESC(psp_init_on_probe, " if true, the PSP will be initialized on module init. Else the PSP will be initialized on the first command requiring it"); 56 57 MODULE_FIRMWARE("amd/amd_sev_fam17h_model0xh.sbin"); /* 1st gen EPYC */ 58 MODULE_FIRMWARE("amd/amd_sev_fam17h_model3xh.sbin"); /* 2nd gen EPYC */ 59 MODULE_FIRMWARE("amd/amd_sev_fam19h_model0xh.sbin"); /* 3rd gen EPYC */ 60 MODULE_FIRMWARE("amd/amd_sev_fam19h_model1xh.sbin"); /* 4th gen EPYC */ 61 62 static bool psp_dead; 63 static int psp_timeout; 64 65 /* Trusted Memory Region (TMR): 66 * The TMR is a 1MB area that must be 1MB aligned. Use the page allocator 67 * to allocate the memory, which will return aligned memory for the specified 68 * allocation order. 69 */ 70 #define SEV_ES_TMR_SIZE (1024 * 1024) 71 static void *sev_es_tmr; 72 73 /* INIT_EX NV Storage: 74 * The NV Storage is a 32Kb area and must be 4Kb page aligned. Use the page 75 * allocator to allocate the memory, which will return aligned memory for the 76 * specified allocation order. 77 */ 78 #define NV_LENGTH (32 * 1024) 79 static void *sev_init_ex_buffer; 80 81 static inline bool sev_version_greater_or_equal(u8 maj, u8 min) 82 { 83 struct sev_device *sev = psp_master->sev_data; 84 85 if (sev->api_major > maj) 86 return true; 87 88 if (sev->api_major == maj && sev->api_minor >= min) 89 return true; 90 91 return false; 92 } 93 94 static void sev_irq_handler(int irq, void *data, unsigned int status) 95 { 96 struct sev_device *sev = data; 97 int reg; 98 99 /* Check if it is command completion: */ 100 if (!(status & SEV_CMD_COMPLETE)) 101 return; 102 103 /* Check if it is SEV command completion: */ 104 reg = ioread32(sev->io_regs + sev->vdata->cmdresp_reg); 105 if (reg & PSP_CMDRESP_RESP) { 106 sev->int_rcvd = 1; 107 wake_up(&sev->int_queue); 108 } 109 } 110 111 static int sev_wait_cmd_ioc(struct sev_device *sev, 112 unsigned int *reg, unsigned int timeout) 113 { 114 int ret; 115 116 ret = wait_event_timeout(sev->int_queue, 117 sev->int_rcvd, timeout * HZ); 118 if (!ret) 119 return -ETIMEDOUT; 120 121 *reg = ioread32(sev->io_regs + sev->vdata->cmdresp_reg); 122 123 return 0; 124 } 125 126 static int sev_cmd_buffer_len(int cmd) 127 { 128 switch (cmd) { 129 case SEV_CMD_INIT: return sizeof(struct sev_data_init); 130 case SEV_CMD_INIT_EX: return sizeof(struct sev_data_init_ex); 131 case SEV_CMD_PLATFORM_STATUS: return sizeof(struct sev_user_data_status); 132 case SEV_CMD_PEK_CSR: return sizeof(struct sev_data_pek_csr); 133 case SEV_CMD_PEK_CERT_IMPORT: return sizeof(struct sev_data_pek_cert_import); 134 case SEV_CMD_PDH_CERT_EXPORT: return sizeof(struct sev_data_pdh_cert_export); 135 case SEV_CMD_LAUNCH_START: return sizeof(struct sev_data_launch_start); 136 case SEV_CMD_LAUNCH_UPDATE_DATA: return sizeof(struct sev_data_launch_update_data); 137 case SEV_CMD_LAUNCH_UPDATE_VMSA: return sizeof(struct sev_data_launch_update_vmsa); 138 case SEV_CMD_LAUNCH_FINISH: return sizeof(struct sev_data_launch_finish); 139 case SEV_CMD_LAUNCH_MEASURE: return sizeof(struct sev_data_launch_measure); 140 case SEV_CMD_ACTIVATE: return sizeof(struct sev_data_activate); 141 case SEV_CMD_DEACTIVATE: return sizeof(struct sev_data_deactivate); 142 case SEV_CMD_DECOMMISSION: return sizeof(struct sev_data_decommission); 143 case SEV_CMD_GUEST_STATUS: return sizeof(struct sev_data_guest_status); 144 case SEV_CMD_DBG_DECRYPT: return sizeof(struct sev_data_dbg); 145 case SEV_CMD_DBG_ENCRYPT: return sizeof(struct sev_data_dbg); 146 case SEV_CMD_SEND_START: return sizeof(struct sev_data_send_start); 147 case SEV_CMD_SEND_UPDATE_DATA: return sizeof(struct sev_data_send_update_data); 148 case SEV_CMD_SEND_UPDATE_VMSA: return sizeof(struct sev_data_send_update_vmsa); 149 case SEV_CMD_SEND_FINISH: return sizeof(struct sev_data_send_finish); 150 case SEV_CMD_RECEIVE_START: return sizeof(struct sev_data_receive_start); 151 case SEV_CMD_RECEIVE_FINISH: return sizeof(struct sev_data_receive_finish); 152 case SEV_CMD_RECEIVE_UPDATE_DATA: return sizeof(struct sev_data_receive_update_data); 153 case SEV_CMD_RECEIVE_UPDATE_VMSA: return sizeof(struct sev_data_receive_update_vmsa); 154 case SEV_CMD_LAUNCH_UPDATE_SECRET: return sizeof(struct sev_data_launch_secret); 155 case SEV_CMD_DOWNLOAD_FIRMWARE: return sizeof(struct sev_data_download_firmware); 156 case SEV_CMD_GET_ID: return sizeof(struct sev_data_get_id); 157 case SEV_CMD_ATTESTATION_REPORT: return sizeof(struct sev_data_attestation_report); 158 case SEV_CMD_SEND_CANCEL: return sizeof(struct sev_data_send_cancel); 159 default: return 0; 160 } 161 162 return 0; 163 } 164 165 static void *sev_fw_alloc(unsigned long len) 166 { 167 struct page *page; 168 169 page = alloc_pages(GFP_KERNEL, get_order(len)); 170 if (!page) 171 return NULL; 172 173 return page_address(page); 174 } 175 176 static struct file *open_file_as_root(const char *filename, int flags, umode_t mode) 177 { 178 struct file *fp; 179 struct path root; 180 struct cred *cred; 181 const struct cred *old_cred; 182 183 task_lock(&init_task); 184 get_fs_root(init_task.fs, &root); 185 task_unlock(&init_task); 186 187 cred = prepare_creds(); 188 if (!cred) 189 return ERR_PTR(-ENOMEM); 190 cred->fsuid = GLOBAL_ROOT_UID; 191 old_cred = override_creds(cred); 192 193 fp = file_open_root(&root, filename, flags, mode); 194 path_put(&root); 195 196 revert_creds(old_cred); 197 198 return fp; 199 } 200 201 static int sev_read_init_ex_file(void) 202 { 203 struct sev_device *sev = psp_master->sev_data; 204 struct file *fp; 205 ssize_t nread; 206 207 lockdep_assert_held(&sev_cmd_mutex); 208 209 if (!sev_init_ex_buffer) 210 return -EOPNOTSUPP; 211 212 fp = open_file_as_root(init_ex_path, O_RDONLY, 0); 213 if (IS_ERR(fp)) { 214 int ret = PTR_ERR(fp); 215 216 if (ret == -ENOENT) { 217 dev_info(sev->dev, 218 "SEV: %s does not exist and will be created later.\n", 219 init_ex_path); 220 ret = 0; 221 } else { 222 dev_err(sev->dev, 223 "SEV: could not open %s for read, error %d\n", 224 init_ex_path, ret); 225 } 226 return ret; 227 } 228 229 nread = kernel_read(fp, sev_init_ex_buffer, NV_LENGTH, NULL); 230 if (nread != NV_LENGTH) { 231 dev_info(sev->dev, 232 "SEV: could not read %u bytes to non volatile memory area, ret %ld\n", 233 NV_LENGTH, nread); 234 } 235 236 dev_dbg(sev->dev, "SEV: read %ld bytes from NV file\n", nread); 237 filp_close(fp, NULL); 238 239 return 0; 240 } 241 242 static int sev_write_init_ex_file(void) 243 { 244 struct sev_device *sev = psp_master->sev_data; 245 struct file *fp; 246 loff_t offset = 0; 247 ssize_t nwrite; 248 249 lockdep_assert_held(&sev_cmd_mutex); 250 251 if (!sev_init_ex_buffer) 252 return 0; 253 254 fp = open_file_as_root(init_ex_path, O_CREAT | O_WRONLY, 0600); 255 if (IS_ERR(fp)) { 256 int ret = PTR_ERR(fp); 257 258 dev_err(sev->dev, 259 "SEV: could not open file for write, error %d\n", 260 ret); 261 return ret; 262 } 263 264 nwrite = kernel_write(fp, sev_init_ex_buffer, NV_LENGTH, &offset); 265 vfs_fsync(fp, 0); 266 filp_close(fp, NULL); 267 268 if (nwrite != NV_LENGTH) { 269 dev_err(sev->dev, 270 "SEV: failed to write %u bytes to non volatile memory area, ret %ld\n", 271 NV_LENGTH, nwrite); 272 return -EIO; 273 } 274 275 dev_dbg(sev->dev, "SEV: write successful to NV file\n"); 276 277 return 0; 278 } 279 280 static int sev_write_init_ex_file_if_required(int cmd_id) 281 { 282 lockdep_assert_held(&sev_cmd_mutex); 283 284 if (!sev_init_ex_buffer) 285 return 0; 286 287 /* 288 * Only a few platform commands modify the SPI/NV area, but none of the 289 * non-platform commands do. Only INIT(_EX), PLATFORM_RESET, PEK_GEN, 290 * PEK_CERT_IMPORT, and PDH_GEN do. 291 */ 292 switch (cmd_id) { 293 case SEV_CMD_FACTORY_RESET: 294 case SEV_CMD_INIT_EX: 295 case SEV_CMD_PDH_GEN: 296 case SEV_CMD_PEK_CERT_IMPORT: 297 case SEV_CMD_PEK_GEN: 298 break; 299 default: 300 return 0; 301 } 302 303 return sev_write_init_ex_file(); 304 } 305 306 static int __sev_do_cmd_locked(int cmd, void *data, int *psp_ret) 307 { 308 struct psp_device *psp = psp_master; 309 struct sev_device *sev; 310 unsigned int phys_lsb, phys_msb; 311 unsigned int reg, ret = 0; 312 int buf_len; 313 314 if (!psp || !psp->sev_data) 315 return -ENODEV; 316 317 if (psp_dead) 318 return -EBUSY; 319 320 sev = psp->sev_data; 321 322 buf_len = sev_cmd_buffer_len(cmd); 323 if (WARN_ON_ONCE(!data != !buf_len)) 324 return -EINVAL; 325 326 /* 327 * Copy the incoming data to driver's scratch buffer as __pa() will not 328 * work for some memory, e.g. vmalloc'd addresses, and @data may not be 329 * physically contiguous. 330 */ 331 if (data) 332 memcpy(sev->cmd_buf, data, buf_len); 333 334 /* Get the physical address of the command buffer */ 335 phys_lsb = data ? lower_32_bits(__psp_pa(sev->cmd_buf)) : 0; 336 phys_msb = data ? upper_32_bits(__psp_pa(sev->cmd_buf)) : 0; 337 338 dev_dbg(sev->dev, "sev command id %#x buffer 0x%08x%08x timeout %us\n", 339 cmd, phys_msb, phys_lsb, psp_timeout); 340 341 print_hex_dump_debug("(in): ", DUMP_PREFIX_OFFSET, 16, 2, data, 342 buf_len, false); 343 344 iowrite32(phys_lsb, sev->io_regs + sev->vdata->cmdbuff_addr_lo_reg); 345 iowrite32(phys_msb, sev->io_regs + sev->vdata->cmdbuff_addr_hi_reg); 346 347 sev->int_rcvd = 0; 348 349 reg = cmd; 350 reg <<= SEV_CMDRESP_CMD_SHIFT; 351 reg |= SEV_CMDRESP_IOC; 352 iowrite32(reg, sev->io_regs + sev->vdata->cmdresp_reg); 353 354 /* wait for command completion */ 355 ret = sev_wait_cmd_ioc(sev, ®, psp_timeout); 356 if (ret) { 357 if (psp_ret) 358 *psp_ret = 0; 359 360 dev_err(sev->dev, "sev command %#x timed out, disabling PSP\n", cmd); 361 psp_dead = true; 362 363 return ret; 364 } 365 366 psp_timeout = psp_cmd_timeout; 367 368 if (psp_ret) 369 *psp_ret = reg & PSP_CMDRESP_ERR_MASK; 370 371 if (reg & PSP_CMDRESP_ERR_MASK) { 372 dev_dbg(sev->dev, "sev command %#x failed (%#010x)\n", 373 cmd, reg & PSP_CMDRESP_ERR_MASK); 374 ret = -EIO; 375 } else { 376 ret = sev_write_init_ex_file_if_required(cmd); 377 } 378 379 print_hex_dump_debug("(out): ", DUMP_PREFIX_OFFSET, 16, 2, data, 380 buf_len, false); 381 382 /* 383 * Copy potential output from the PSP back to data. Do this even on 384 * failure in case the caller wants to glean something from the error. 385 */ 386 if (data) 387 memcpy(data, sev->cmd_buf, buf_len); 388 389 return ret; 390 } 391 392 static int sev_do_cmd(int cmd, void *data, int *psp_ret) 393 { 394 int rc; 395 396 mutex_lock(&sev_cmd_mutex); 397 rc = __sev_do_cmd_locked(cmd, data, psp_ret); 398 mutex_unlock(&sev_cmd_mutex); 399 400 return rc; 401 } 402 403 static int __sev_init_locked(int *error) 404 { 405 struct sev_data_init data; 406 407 memset(&data, 0, sizeof(data)); 408 if (sev_es_tmr) { 409 /* 410 * Do not include the encryption mask on the physical 411 * address of the TMR (firmware should clear it anyway). 412 */ 413 data.tmr_address = __pa(sev_es_tmr); 414 415 data.flags |= SEV_INIT_FLAGS_SEV_ES; 416 data.tmr_len = SEV_ES_TMR_SIZE; 417 } 418 419 return __sev_do_cmd_locked(SEV_CMD_INIT, &data, error); 420 } 421 422 static int __sev_init_ex_locked(int *error) 423 { 424 struct sev_data_init_ex data; 425 426 memset(&data, 0, sizeof(data)); 427 data.length = sizeof(data); 428 data.nv_address = __psp_pa(sev_init_ex_buffer); 429 data.nv_len = NV_LENGTH; 430 431 if (sev_es_tmr) { 432 /* 433 * Do not include the encryption mask on the physical 434 * address of the TMR (firmware should clear it anyway). 435 */ 436 data.tmr_address = __pa(sev_es_tmr); 437 438 data.flags |= SEV_INIT_FLAGS_SEV_ES; 439 data.tmr_len = SEV_ES_TMR_SIZE; 440 } 441 442 return __sev_do_cmd_locked(SEV_CMD_INIT_EX, &data, error); 443 } 444 445 static int __sev_platform_init_locked(int *error) 446 { 447 struct psp_device *psp = psp_master; 448 struct sev_device *sev; 449 int rc = 0, psp_ret = -1; 450 int (*init_function)(int *error); 451 452 if (!psp || !psp->sev_data) 453 return -ENODEV; 454 455 sev = psp->sev_data; 456 457 if (sev->state == SEV_STATE_INIT) 458 return 0; 459 460 if (sev_init_ex_buffer) { 461 init_function = __sev_init_ex_locked; 462 rc = sev_read_init_ex_file(); 463 if (rc) 464 return rc; 465 } else { 466 init_function = __sev_init_locked; 467 } 468 469 rc = init_function(&psp_ret); 470 if (rc && psp_ret == SEV_RET_SECURE_DATA_INVALID) { 471 /* 472 * Initialization command returned an integrity check failure 473 * status code, meaning that firmware load and validation of SEV 474 * related persistent data has failed. Retrying the 475 * initialization function should succeed by replacing the state 476 * with a reset state. 477 */ 478 dev_err(sev->dev, "SEV: retrying INIT command because of SECURE_DATA_INVALID error. Retrying once to reset PSP SEV state."); 479 rc = init_function(&psp_ret); 480 } 481 if (error) 482 *error = psp_ret; 483 484 if (rc) 485 return rc; 486 487 sev->state = SEV_STATE_INIT; 488 489 /* Prepare for first SEV guest launch after INIT */ 490 wbinvd_on_all_cpus(); 491 rc = __sev_do_cmd_locked(SEV_CMD_DF_FLUSH, NULL, error); 492 if (rc) 493 return rc; 494 495 dev_dbg(sev->dev, "SEV firmware initialized\n"); 496 497 dev_info(sev->dev, "SEV API:%d.%d build:%d\n", sev->api_major, 498 sev->api_minor, sev->build); 499 500 return 0; 501 } 502 503 int sev_platform_init(int *error) 504 { 505 int rc; 506 507 mutex_lock(&sev_cmd_mutex); 508 rc = __sev_platform_init_locked(error); 509 mutex_unlock(&sev_cmd_mutex); 510 511 return rc; 512 } 513 EXPORT_SYMBOL_GPL(sev_platform_init); 514 515 static int __sev_platform_shutdown_locked(int *error) 516 { 517 struct sev_device *sev = psp_master->sev_data; 518 int ret; 519 520 if (!sev || sev->state == SEV_STATE_UNINIT) 521 return 0; 522 523 ret = __sev_do_cmd_locked(SEV_CMD_SHUTDOWN, NULL, error); 524 if (ret) 525 return ret; 526 527 sev->state = SEV_STATE_UNINIT; 528 dev_dbg(sev->dev, "SEV firmware shutdown\n"); 529 530 return ret; 531 } 532 533 static int sev_platform_shutdown(int *error) 534 { 535 int rc; 536 537 mutex_lock(&sev_cmd_mutex); 538 rc = __sev_platform_shutdown_locked(NULL); 539 mutex_unlock(&sev_cmd_mutex); 540 541 return rc; 542 } 543 544 static int sev_get_platform_state(int *state, int *error) 545 { 546 struct sev_user_data_status data; 547 int rc; 548 549 rc = __sev_do_cmd_locked(SEV_CMD_PLATFORM_STATUS, &data, error); 550 if (rc) 551 return rc; 552 553 *state = data.state; 554 return rc; 555 } 556 557 static int sev_ioctl_do_reset(struct sev_issue_cmd *argp, bool writable) 558 { 559 int state, rc; 560 561 if (!writable) 562 return -EPERM; 563 564 /* 565 * The SEV spec requires that FACTORY_RESET must be issued in 566 * UNINIT state. Before we go further lets check if any guest is 567 * active. 568 * 569 * If FW is in WORKING state then deny the request otherwise issue 570 * SHUTDOWN command do INIT -> UNINIT before issuing the FACTORY_RESET. 571 * 572 */ 573 rc = sev_get_platform_state(&state, &argp->error); 574 if (rc) 575 return rc; 576 577 if (state == SEV_STATE_WORKING) 578 return -EBUSY; 579 580 if (state == SEV_STATE_INIT) { 581 rc = __sev_platform_shutdown_locked(&argp->error); 582 if (rc) 583 return rc; 584 } 585 586 return __sev_do_cmd_locked(SEV_CMD_FACTORY_RESET, NULL, &argp->error); 587 } 588 589 static int sev_ioctl_do_platform_status(struct sev_issue_cmd *argp) 590 { 591 struct sev_user_data_status data; 592 int ret; 593 594 memset(&data, 0, sizeof(data)); 595 596 ret = __sev_do_cmd_locked(SEV_CMD_PLATFORM_STATUS, &data, &argp->error); 597 if (ret) 598 return ret; 599 600 if (copy_to_user((void __user *)argp->data, &data, sizeof(data))) 601 ret = -EFAULT; 602 603 return ret; 604 } 605 606 static int sev_ioctl_do_pek_pdh_gen(int cmd, struct sev_issue_cmd *argp, bool writable) 607 { 608 struct sev_device *sev = psp_master->sev_data; 609 int rc; 610 611 if (!writable) 612 return -EPERM; 613 614 if (sev->state == SEV_STATE_UNINIT) { 615 rc = __sev_platform_init_locked(&argp->error); 616 if (rc) 617 return rc; 618 } 619 620 return __sev_do_cmd_locked(cmd, NULL, &argp->error); 621 } 622 623 static int sev_ioctl_do_pek_csr(struct sev_issue_cmd *argp, bool writable) 624 { 625 struct sev_device *sev = psp_master->sev_data; 626 struct sev_user_data_pek_csr input; 627 struct sev_data_pek_csr data; 628 void __user *input_address; 629 void *blob = NULL; 630 int ret; 631 632 if (!writable) 633 return -EPERM; 634 635 if (copy_from_user(&input, (void __user *)argp->data, sizeof(input))) 636 return -EFAULT; 637 638 memset(&data, 0, sizeof(data)); 639 640 /* userspace wants to query CSR length */ 641 if (!input.address || !input.length) 642 goto cmd; 643 644 /* allocate a physically contiguous buffer to store the CSR blob */ 645 input_address = (void __user *)input.address; 646 if (input.length > SEV_FW_BLOB_MAX_SIZE) 647 return -EFAULT; 648 649 blob = kzalloc(input.length, GFP_KERNEL); 650 if (!blob) 651 return -ENOMEM; 652 653 data.address = __psp_pa(blob); 654 data.len = input.length; 655 656 cmd: 657 if (sev->state == SEV_STATE_UNINIT) { 658 ret = __sev_platform_init_locked(&argp->error); 659 if (ret) 660 goto e_free_blob; 661 } 662 663 ret = __sev_do_cmd_locked(SEV_CMD_PEK_CSR, &data, &argp->error); 664 665 /* If we query the CSR length, FW responded with expected data. */ 666 input.length = data.len; 667 668 if (copy_to_user((void __user *)argp->data, &input, sizeof(input))) { 669 ret = -EFAULT; 670 goto e_free_blob; 671 } 672 673 if (blob) { 674 if (copy_to_user(input_address, blob, input.length)) 675 ret = -EFAULT; 676 } 677 678 e_free_blob: 679 kfree(blob); 680 return ret; 681 } 682 683 void *psp_copy_user_blob(u64 uaddr, u32 len) 684 { 685 if (!uaddr || !len) 686 return ERR_PTR(-EINVAL); 687 688 /* verify that blob length does not exceed our limit */ 689 if (len > SEV_FW_BLOB_MAX_SIZE) 690 return ERR_PTR(-EINVAL); 691 692 return memdup_user((void __user *)uaddr, len); 693 } 694 EXPORT_SYMBOL_GPL(psp_copy_user_blob); 695 696 static int sev_get_api_version(void) 697 { 698 struct sev_device *sev = psp_master->sev_data; 699 struct sev_user_data_status status; 700 int error = 0, ret; 701 702 ret = sev_platform_status(&status, &error); 703 if (ret) { 704 dev_err(sev->dev, 705 "SEV: failed to get status. Error: %#x\n", error); 706 return 1; 707 } 708 709 sev->api_major = status.api_major; 710 sev->api_minor = status.api_minor; 711 sev->build = status.build; 712 sev->state = status.state; 713 714 return 0; 715 } 716 717 static int sev_get_firmware(struct device *dev, 718 const struct firmware **firmware) 719 { 720 char fw_name_specific[SEV_FW_NAME_SIZE]; 721 char fw_name_subset[SEV_FW_NAME_SIZE]; 722 723 snprintf(fw_name_specific, sizeof(fw_name_specific), 724 "amd/amd_sev_fam%.2xh_model%.2xh.sbin", 725 boot_cpu_data.x86, boot_cpu_data.x86_model); 726 727 snprintf(fw_name_subset, sizeof(fw_name_subset), 728 "amd/amd_sev_fam%.2xh_model%.1xxh.sbin", 729 boot_cpu_data.x86, (boot_cpu_data.x86_model & 0xf0) >> 4); 730 731 /* Check for SEV FW for a particular model. 732 * Ex. amd_sev_fam17h_model00h.sbin for Family 17h Model 00h 733 * 734 * or 735 * 736 * Check for SEV FW common to a subset of models. 737 * Ex. amd_sev_fam17h_model0xh.sbin for 738 * Family 17h Model 00h -- Family 17h Model 0Fh 739 * 740 * or 741 * 742 * Fall-back to using generic name: sev.fw 743 */ 744 if ((firmware_request_nowarn(firmware, fw_name_specific, dev) >= 0) || 745 (firmware_request_nowarn(firmware, fw_name_subset, dev) >= 0) || 746 (firmware_request_nowarn(firmware, SEV_FW_FILE, dev) >= 0)) 747 return 0; 748 749 return -ENOENT; 750 } 751 752 /* Don't fail if SEV FW couldn't be updated. Continue with existing SEV FW */ 753 static int sev_update_firmware(struct device *dev) 754 { 755 struct sev_data_download_firmware *data; 756 const struct firmware *firmware; 757 int ret, error, order; 758 struct page *p; 759 u64 data_size; 760 761 if (!sev_version_greater_or_equal(0, 15)) { 762 dev_dbg(dev, "DOWNLOAD_FIRMWARE not supported\n"); 763 return -1; 764 } 765 766 if (sev_get_firmware(dev, &firmware) == -ENOENT) { 767 dev_dbg(dev, "No SEV firmware file present\n"); 768 return -1; 769 } 770 771 /* 772 * SEV FW expects the physical address given to it to be 32 773 * byte aligned. Memory allocated has structure placed at the 774 * beginning followed by the firmware being passed to the SEV 775 * FW. Allocate enough memory for data structure + alignment 776 * padding + SEV FW. 777 */ 778 data_size = ALIGN(sizeof(struct sev_data_download_firmware), 32); 779 780 order = get_order(firmware->size + data_size); 781 p = alloc_pages(GFP_KERNEL, order); 782 if (!p) { 783 ret = -1; 784 goto fw_err; 785 } 786 787 /* 788 * Copy firmware data to a kernel allocated contiguous 789 * memory region. 790 */ 791 data = page_address(p); 792 memcpy(page_address(p) + data_size, firmware->data, firmware->size); 793 794 data->address = __psp_pa(page_address(p) + data_size); 795 data->len = firmware->size; 796 797 ret = sev_do_cmd(SEV_CMD_DOWNLOAD_FIRMWARE, data, &error); 798 799 /* 800 * A quirk for fixing the committed TCB version, when upgrading from 801 * earlier firmware version than 1.50. 802 */ 803 if (!ret && !sev_version_greater_or_equal(1, 50)) 804 ret = sev_do_cmd(SEV_CMD_DOWNLOAD_FIRMWARE, data, &error); 805 806 if (ret) 807 dev_dbg(dev, "Failed to update SEV firmware: %#x\n", error); 808 else 809 dev_info(dev, "SEV firmware update successful\n"); 810 811 __free_pages(p, order); 812 813 fw_err: 814 release_firmware(firmware); 815 816 return ret; 817 } 818 819 static int sev_ioctl_do_pek_import(struct sev_issue_cmd *argp, bool writable) 820 { 821 struct sev_device *sev = psp_master->sev_data; 822 struct sev_user_data_pek_cert_import input; 823 struct sev_data_pek_cert_import data; 824 void *pek_blob, *oca_blob; 825 int ret; 826 827 if (!writable) 828 return -EPERM; 829 830 if (copy_from_user(&input, (void __user *)argp->data, sizeof(input))) 831 return -EFAULT; 832 833 /* copy PEK certificate blobs from userspace */ 834 pek_blob = psp_copy_user_blob(input.pek_cert_address, input.pek_cert_len); 835 if (IS_ERR(pek_blob)) 836 return PTR_ERR(pek_blob); 837 838 data.reserved = 0; 839 data.pek_cert_address = __psp_pa(pek_blob); 840 data.pek_cert_len = input.pek_cert_len; 841 842 /* copy PEK certificate blobs from userspace */ 843 oca_blob = psp_copy_user_blob(input.oca_cert_address, input.oca_cert_len); 844 if (IS_ERR(oca_blob)) { 845 ret = PTR_ERR(oca_blob); 846 goto e_free_pek; 847 } 848 849 data.oca_cert_address = __psp_pa(oca_blob); 850 data.oca_cert_len = input.oca_cert_len; 851 852 /* If platform is not in INIT state then transition it to INIT */ 853 if (sev->state != SEV_STATE_INIT) { 854 ret = __sev_platform_init_locked(&argp->error); 855 if (ret) 856 goto e_free_oca; 857 } 858 859 ret = __sev_do_cmd_locked(SEV_CMD_PEK_CERT_IMPORT, &data, &argp->error); 860 861 e_free_oca: 862 kfree(oca_blob); 863 e_free_pek: 864 kfree(pek_blob); 865 return ret; 866 } 867 868 static int sev_ioctl_do_get_id2(struct sev_issue_cmd *argp) 869 { 870 struct sev_user_data_get_id2 input; 871 struct sev_data_get_id data; 872 void __user *input_address; 873 void *id_blob = NULL; 874 int ret; 875 876 /* SEV GET_ID is available from SEV API v0.16 and up */ 877 if (!sev_version_greater_or_equal(0, 16)) 878 return -ENOTSUPP; 879 880 if (copy_from_user(&input, (void __user *)argp->data, sizeof(input))) 881 return -EFAULT; 882 883 input_address = (void __user *)input.address; 884 885 if (input.address && input.length) { 886 /* 887 * The length of the ID shouldn't be assumed by software since 888 * it may change in the future. The allocation size is limited 889 * to 1 << (PAGE_SHIFT + MAX_ORDER - 1) by the page allocator. 890 * If the allocation fails, simply return ENOMEM rather than 891 * warning in the kernel log. 892 */ 893 id_blob = kzalloc(input.length, GFP_KERNEL | __GFP_NOWARN); 894 if (!id_blob) 895 return -ENOMEM; 896 897 data.address = __psp_pa(id_blob); 898 data.len = input.length; 899 } else { 900 data.address = 0; 901 data.len = 0; 902 } 903 904 ret = __sev_do_cmd_locked(SEV_CMD_GET_ID, &data, &argp->error); 905 906 /* 907 * Firmware will return the length of the ID value (either the minimum 908 * required length or the actual length written), return it to the user. 909 */ 910 input.length = data.len; 911 912 if (copy_to_user((void __user *)argp->data, &input, sizeof(input))) { 913 ret = -EFAULT; 914 goto e_free; 915 } 916 917 if (id_blob) { 918 if (copy_to_user(input_address, id_blob, data.len)) { 919 ret = -EFAULT; 920 goto e_free; 921 } 922 } 923 924 e_free: 925 kfree(id_blob); 926 927 return ret; 928 } 929 930 static int sev_ioctl_do_get_id(struct sev_issue_cmd *argp) 931 { 932 struct sev_data_get_id *data; 933 u64 data_size, user_size; 934 void *id_blob, *mem; 935 int ret; 936 937 /* SEV GET_ID available from SEV API v0.16 and up */ 938 if (!sev_version_greater_or_equal(0, 16)) 939 return -ENOTSUPP; 940 941 /* SEV FW expects the buffer it fills with the ID to be 942 * 8-byte aligned. Memory allocated should be enough to 943 * hold data structure + alignment padding + memory 944 * where SEV FW writes the ID. 945 */ 946 data_size = ALIGN(sizeof(struct sev_data_get_id), 8); 947 user_size = sizeof(struct sev_user_data_get_id); 948 949 mem = kzalloc(data_size + user_size, GFP_KERNEL); 950 if (!mem) 951 return -ENOMEM; 952 953 data = mem; 954 id_blob = mem + data_size; 955 956 data->address = __psp_pa(id_blob); 957 data->len = user_size; 958 959 ret = __sev_do_cmd_locked(SEV_CMD_GET_ID, data, &argp->error); 960 if (!ret) { 961 if (copy_to_user((void __user *)argp->data, id_blob, data->len)) 962 ret = -EFAULT; 963 } 964 965 kfree(mem); 966 967 return ret; 968 } 969 970 static int sev_ioctl_do_pdh_export(struct sev_issue_cmd *argp, bool writable) 971 { 972 struct sev_device *sev = psp_master->sev_data; 973 struct sev_user_data_pdh_cert_export input; 974 void *pdh_blob = NULL, *cert_blob = NULL; 975 struct sev_data_pdh_cert_export data; 976 void __user *input_cert_chain_address; 977 void __user *input_pdh_cert_address; 978 int ret; 979 980 /* If platform is not in INIT state then transition it to INIT. */ 981 if (sev->state != SEV_STATE_INIT) { 982 if (!writable) 983 return -EPERM; 984 985 ret = __sev_platform_init_locked(&argp->error); 986 if (ret) 987 return ret; 988 } 989 990 if (copy_from_user(&input, (void __user *)argp->data, sizeof(input))) 991 return -EFAULT; 992 993 memset(&data, 0, sizeof(data)); 994 995 /* Userspace wants to query the certificate length. */ 996 if (!input.pdh_cert_address || 997 !input.pdh_cert_len || 998 !input.cert_chain_address) 999 goto cmd; 1000 1001 input_pdh_cert_address = (void __user *)input.pdh_cert_address; 1002 input_cert_chain_address = (void __user *)input.cert_chain_address; 1003 1004 /* Allocate a physically contiguous buffer to store the PDH blob. */ 1005 if (input.pdh_cert_len > SEV_FW_BLOB_MAX_SIZE) 1006 return -EFAULT; 1007 1008 /* Allocate a physically contiguous buffer to store the cert chain blob. */ 1009 if (input.cert_chain_len > SEV_FW_BLOB_MAX_SIZE) 1010 return -EFAULT; 1011 1012 pdh_blob = kzalloc(input.pdh_cert_len, GFP_KERNEL); 1013 if (!pdh_blob) 1014 return -ENOMEM; 1015 1016 data.pdh_cert_address = __psp_pa(pdh_blob); 1017 data.pdh_cert_len = input.pdh_cert_len; 1018 1019 cert_blob = kzalloc(input.cert_chain_len, GFP_KERNEL); 1020 if (!cert_blob) { 1021 ret = -ENOMEM; 1022 goto e_free_pdh; 1023 } 1024 1025 data.cert_chain_address = __psp_pa(cert_blob); 1026 data.cert_chain_len = input.cert_chain_len; 1027 1028 cmd: 1029 ret = __sev_do_cmd_locked(SEV_CMD_PDH_CERT_EXPORT, &data, &argp->error); 1030 1031 /* If we query the length, FW responded with expected data. */ 1032 input.cert_chain_len = data.cert_chain_len; 1033 input.pdh_cert_len = data.pdh_cert_len; 1034 1035 if (copy_to_user((void __user *)argp->data, &input, sizeof(input))) { 1036 ret = -EFAULT; 1037 goto e_free_cert; 1038 } 1039 1040 if (pdh_blob) { 1041 if (copy_to_user(input_pdh_cert_address, 1042 pdh_blob, input.pdh_cert_len)) { 1043 ret = -EFAULT; 1044 goto e_free_cert; 1045 } 1046 } 1047 1048 if (cert_blob) { 1049 if (copy_to_user(input_cert_chain_address, 1050 cert_blob, input.cert_chain_len)) 1051 ret = -EFAULT; 1052 } 1053 1054 e_free_cert: 1055 kfree(cert_blob); 1056 e_free_pdh: 1057 kfree(pdh_blob); 1058 return ret; 1059 } 1060 1061 static long sev_ioctl(struct file *file, unsigned int ioctl, unsigned long arg) 1062 { 1063 void __user *argp = (void __user *)arg; 1064 struct sev_issue_cmd input; 1065 int ret = -EFAULT; 1066 bool writable = file->f_mode & FMODE_WRITE; 1067 1068 if (!psp_master || !psp_master->sev_data) 1069 return -ENODEV; 1070 1071 if (ioctl != SEV_ISSUE_CMD) 1072 return -EINVAL; 1073 1074 if (copy_from_user(&input, argp, sizeof(struct sev_issue_cmd))) 1075 return -EFAULT; 1076 1077 if (input.cmd > SEV_MAX) 1078 return -EINVAL; 1079 1080 mutex_lock(&sev_cmd_mutex); 1081 1082 switch (input.cmd) { 1083 1084 case SEV_FACTORY_RESET: 1085 ret = sev_ioctl_do_reset(&input, writable); 1086 break; 1087 case SEV_PLATFORM_STATUS: 1088 ret = sev_ioctl_do_platform_status(&input); 1089 break; 1090 case SEV_PEK_GEN: 1091 ret = sev_ioctl_do_pek_pdh_gen(SEV_CMD_PEK_GEN, &input, writable); 1092 break; 1093 case SEV_PDH_GEN: 1094 ret = sev_ioctl_do_pek_pdh_gen(SEV_CMD_PDH_GEN, &input, writable); 1095 break; 1096 case SEV_PEK_CSR: 1097 ret = sev_ioctl_do_pek_csr(&input, writable); 1098 break; 1099 case SEV_PEK_CERT_IMPORT: 1100 ret = sev_ioctl_do_pek_import(&input, writable); 1101 break; 1102 case SEV_PDH_CERT_EXPORT: 1103 ret = sev_ioctl_do_pdh_export(&input, writable); 1104 break; 1105 case SEV_GET_ID: 1106 pr_warn_once("SEV_GET_ID command is deprecated, use SEV_GET_ID2\n"); 1107 ret = sev_ioctl_do_get_id(&input); 1108 break; 1109 case SEV_GET_ID2: 1110 ret = sev_ioctl_do_get_id2(&input); 1111 break; 1112 default: 1113 ret = -EINVAL; 1114 goto out; 1115 } 1116 1117 if (copy_to_user(argp, &input, sizeof(struct sev_issue_cmd))) 1118 ret = -EFAULT; 1119 out: 1120 mutex_unlock(&sev_cmd_mutex); 1121 1122 return ret; 1123 } 1124 1125 static const struct file_operations sev_fops = { 1126 .owner = THIS_MODULE, 1127 .unlocked_ioctl = sev_ioctl, 1128 }; 1129 1130 int sev_platform_status(struct sev_user_data_status *data, int *error) 1131 { 1132 return sev_do_cmd(SEV_CMD_PLATFORM_STATUS, data, error); 1133 } 1134 EXPORT_SYMBOL_GPL(sev_platform_status); 1135 1136 int sev_guest_deactivate(struct sev_data_deactivate *data, int *error) 1137 { 1138 return sev_do_cmd(SEV_CMD_DEACTIVATE, data, error); 1139 } 1140 EXPORT_SYMBOL_GPL(sev_guest_deactivate); 1141 1142 int sev_guest_activate(struct sev_data_activate *data, int *error) 1143 { 1144 return sev_do_cmd(SEV_CMD_ACTIVATE, data, error); 1145 } 1146 EXPORT_SYMBOL_GPL(sev_guest_activate); 1147 1148 int sev_guest_decommission(struct sev_data_decommission *data, int *error) 1149 { 1150 return sev_do_cmd(SEV_CMD_DECOMMISSION, data, error); 1151 } 1152 EXPORT_SYMBOL_GPL(sev_guest_decommission); 1153 1154 int sev_guest_df_flush(int *error) 1155 { 1156 return sev_do_cmd(SEV_CMD_DF_FLUSH, NULL, error); 1157 } 1158 EXPORT_SYMBOL_GPL(sev_guest_df_flush); 1159 1160 static void sev_exit(struct kref *ref) 1161 { 1162 misc_deregister(&misc_dev->misc); 1163 kfree(misc_dev); 1164 misc_dev = NULL; 1165 } 1166 1167 static int sev_misc_init(struct sev_device *sev) 1168 { 1169 struct device *dev = sev->dev; 1170 int ret; 1171 1172 /* 1173 * SEV feature support can be detected on multiple devices but the SEV 1174 * FW commands must be issued on the master. During probe, we do not 1175 * know the master hence we create /dev/sev on the first device probe. 1176 * sev_do_cmd() finds the right master device to which to issue the 1177 * command to the firmware. 1178 */ 1179 if (!misc_dev) { 1180 struct miscdevice *misc; 1181 1182 misc_dev = kzalloc(sizeof(*misc_dev), GFP_KERNEL); 1183 if (!misc_dev) 1184 return -ENOMEM; 1185 1186 misc = &misc_dev->misc; 1187 misc->minor = MISC_DYNAMIC_MINOR; 1188 misc->name = DEVICE_NAME; 1189 misc->fops = &sev_fops; 1190 1191 ret = misc_register(misc); 1192 if (ret) 1193 return ret; 1194 1195 kref_init(&misc_dev->refcount); 1196 } else { 1197 kref_get(&misc_dev->refcount); 1198 } 1199 1200 init_waitqueue_head(&sev->int_queue); 1201 sev->misc = misc_dev; 1202 dev_dbg(dev, "registered SEV device\n"); 1203 1204 return 0; 1205 } 1206 1207 int sev_dev_init(struct psp_device *psp) 1208 { 1209 struct device *dev = psp->dev; 1210 struct sev_device *sev; 1211 int ret = -ENOMEM; 1212 1213 if (!boot_cpu_has(X86_FEATURE_SEV)) { 1214 dev_info_once(dev, "SEV: memory encryption not enabled by BIOS\n"); 1215 return 0; 1216 } 1217 1218 sev = devm_kzalloc(dev, sizeof(*sev), GFP_KERNEL); 1219 if (!sev) 1220 goto e_err; 1221 1222 sev->cmd_buf = (void *)devm_get_free_pages(dev, GFP_KERNEL, 0); 1223 if (!sev->cmd_buf) 1224 goto e_sev; 1225 1226 psp->sev_data = sev; 1227 1228 sev->dev = dev; 1229 sev->psp = psp; 1230 1231 sev->io_regs = psp->io_regs; 1232 1233 sev->vdata = (struct sev_vdata *)psp->vdata->sev; 1234 if (!sev->vdata) { 1235 ret = -ENODEV; 1236 dev_err(dev, "sev: missing driver data\n"); 1237 goto e_buf; 1238 } 1239 1240 psp_set_sev_irq_handler(psp, sev_irq_handler, sev); 1241 1242 ret = sev_misc_init(sev); 1243 if (ret) 1244 goto e_irq; 1245 1246 dev_notice(dev, "sev enabled\n"); 1247 1248 return 0; 1249 1250 e_irq: 1251 psp_clear_sev_irq_handler(psp); 1252 e_buf: 1253 devm_free_pages(dev, (unsigned long)sev->cmd_buf); 1254 e_sev: 1255 devm_kfree(dev, sev); 1256 e_err: 1257 psp->sev_data = NULL; 1258 1259 dev_notice(dev, "sev initialization failed\n"); 1260 1261 return ret; 1262 } 1263 1264 static void sev_firmware_shutdown(struct sev_device *sev) 1265 { 1266 sev_platform_shutdown(NULL); 1267 1268 if (sev_es_tmr) { 1269 /* The TMR area was encrypted, flush it from the cache */ 1270 wbinvd_on_all_cpus(); 1271 1272 free_pages((unsigned long)sev_es_tmr, 1273 get_order(SEV_ES_TMR_SIZE)); 1274 sev_es_tmr = NULL; 1275 } 1276 1277 if (sev_init_ex_buffer) { 1278 free_pages((unsigned long)sev_init_ex_buffer, 1279 get_order(NV_LENGTH)); 1280 sev_init_ex_buffer = NULL; 1281 } 1282 } 1283 1284 void sev_dev_destroy(struct psp_device *psp) 1285 { 1286 struct sev_device *sev = psp->sev_data; 1287 1288 if (!sev) 1289 return; 1290 1291 sev_firmware_shutdown(sev); 1292 1293 if (sev->misc) 1294 kref_put(&misc_dev->refcount, sev_exit); 1295 1296 psp_clear_sev_irq_handler(psp); 1297 } 1298 1299 int sev_issue_cmd_external_user(struct file *filep, unsigned int cmd, 1300 void *data, int *error) 1301 { 1302 if (!filep || filep->f_op != &sev_fops) 1303 return -EBADF; 1304 1305 return sev_do_cmd(cmd, data, error); 1306 } 1307 EXPORT_SYMBOL_GPL(sev_issue_cmd_external_user); 1308 1309 void sev_pci_init(void) 1310 { 1311 struct sev_device *sev = psp_master->sev_data; 1312 int error, rc; 1313 1314 if (!sev) 1315 return; 1316 1317 psp_timeout = psp_probe_timeout; 1318 1319 if (sev_get_api_version()) 1320 goto err; 1321 1322 if (sev_update_firmware(sev->dev) == 0) 1323 sev_get_api_version(); 1324 1325 /* If an init_ex_path is provided rely on INIT_EX for PSP initialization 1326 * instead of INIT. 1327 */ 1328 if (init_ex_path) { 1329 sev_init_ex_buffer = sev_fw_alloc(NV_LENGTH); 1330 if (!sev_init_ex_buffer) { 1331 dev_err(sev->dev, 1332 "SEV: INIT_EX NV memory allocation failed\n"); 1333 goto err; 1334 } 1335 } 1336 1337 /* Obtain the TMR memory area for SEV-ES use */ 1338 sev_es_tmr = sev_fw_alloc(SEV_ES_TMR_SIZE); 1339 if (sev_es_tmr) 1340 /* Must flush the cache before giving it to the firmware */ 1341 clflush_cache_range(sev_es_tmr, SEV_ES_TMR_SIZE); 1342 else 1343 dev_warn(sev->dev, 1344 "SEV: TMR allocation failed, SEV-ES support unavailable\n"); 1345 1346 if (!psp_init_on_probe) 1347 return; 1348 1349 /* Initialize the platform */ 1350 rc = sev_platform_init(&error); 1351 if (rc) 1352 dev_err(sev->dev, "SEV: failed to INIT error %#x, rc %d\n", 1353 error, rc); 1354 1355 return; 1356 1357 err: 1358 psp_master->sev_data = NULL; 1359 } 1360 1361 void sev_pci_exit(void) 1362 { 1363 struct sev_device *sev = psp_master->sev_data; 1364 1365 if (!sev) 1366 return; 1367 1368 sev_firmware_shutdown(sev); 1369 } 1370