1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright(c) 2016-20 Intel Corporation. */ 3 4 #include <linux/file.h> 5 #include <linux/freezer.h> 6 #include <linux/highmem.h> 7 #include <linux/kthread.h> 8 #include <linux/miscdevice.h> 9 #include <linux/node.h> 10 #include <linux/pagemap.h> 11 #include <linux/ratelimit.h> 12 #include <linux/sched/mm.h> 13 #include <linux/sched/signal.h> 14 #include <linux/slab.h> 15 #include <linux/sysfs.h> 16 #include <asm/sgx.h> 17 #include "driver.h" 18 #include "encl.h" 19 #include "encls.h" 20 21 struct sgx_epc_section sgx_epc_sections[SGX_MAX_EPC_SECTIONS]; 22 static int sgx_nr_epc_sections; 23 static struct task_struct *ksgxd_tsk; 24 static DECLARE_WAIT_QUEUE_HEAD(ksgxd_waitq); 25 static DEFINE_XARRAY(sgx_epc_address_space); 26 27 /* 28 * These variables are part of the state of the reclaimer, and must be accessed 29 * with sgx_reclaimer_lock acquired. 30 */ 31 static LIST_HEAD(sgx_active_page_list); 32 static DEFINE_SPINLOCK(sgx_reclaimer_lock); 33 34 static atomic_long_t sgx_nr_free_pages = ATOMIC_LONG_INIT(0); 35 36 /* Nodes with one or more EPC sections. */ 37 static nodemask_t sgx_numa_mask; 38 39 /* 40 * Array with one list_head for each possible NUMA node. Each 41 * list contains all the sgx_epc_section's which are on that 42 * node. 43 */ 44 static struct sgx_numa_node *sgx_numa_nodes; 45 46 static LIST_HEAD(sgx_dirty_page_list); 47 48 /* 49 * Reset post-kexec EPC pages to the uninitialized state. The pages are removed 50 * from the input list, and made available for the page allocator. SECS pages 51 * prepending their children in the input list are left intact. 52 */ 53 static void __sgx_sanitize_pages(struct list_head *dirty_page_list) 54 { 55 struct sgx_epc_page *page; 56 LIST_HEAD(dirty); 57 int ret; 58 59 /* dirty_page_list is thread-local, no need for a lock: */ 60 while (!list_empty(dirty_page_list)) { 61 if (kthread_should_stop()) 62 return; 63 64 page = list_first_entry(dirty_page_list, struct sgx_epc_page, list); 65 66 /* 67 * Checking page->poison without holding the node->lock 68 * is racy, but losing the race (i.e. poison is set just 69 * after the check) just means __eremove() will be uselessly 70 * called for a page that sgx_free_epc_page() will put onto 71 * the node->sgx_poison_page_list later. 72 */ 73 if (page->poison) { 74 struct sgx_epc_section *section = &sgx_epc_sections[page->section]; 75 struct sgx_numa_node *node = section->node; 76 77 spin_lock(&node->lock); 78 list_move(&page->list, &node->sgx_poison_page_list); 79 spin_unlock(&node->lock); 80 81 continue; 82 } 83 84 ret = __eremove(sgx_get_epc_virt_addr(page)); 85 if (!ret) { 86 /* 87 * page is now sanitized. Make it available via the SGX 88 * page allocator: 89 */ 90 list_del(&page->list); 91 sgx_free_epc_page(page); 92 } else { 93 /* The page is not yet clean - move to the dirty list. */ 94 list_move_tail(&page->list, &dirty); 95 } 96 97 cond_resched(); 98 } 99 100 list_splice(&dirty, dirty_page_list); 101 } 102 103 static bool sgx_reclaimer_age(struct sgx_epc_page *epc_page) 104 { 105 struct sgx_encl_page *page = epc_page->owner; 106 struct sgx_encl *encl = page->encl; 107 struct sgx_encl_mm *encl_mm; 108 bool ret = true; 109 int idx; 110 111 idx = srcu_read_lock(&encl->srcu); 112 113 list_for_each_entry_rcu(encl_mm, &encl->mm_list, list) { 114 if (!mmget_not_zero(encl_mm->mm)) 115 continue; 116 117 mmap_read_lock(encl_mm->mm); 118 ret = !sgx_encl_test_and_clear_young(encl_mm->mm, page); 119 mmap_read_unlock(encl_mm->mm); 120 121 mmput_async(encl_mm->mm); 122 123 if (!ret) 124 break; 125 } 126 127 srcu_read_unlock(&encl->srcu, idx); 128 129 if (!ret) 130 return false; 131 132 return true; 133 } 134 135 static void sgx_reclaimer_block(struct sgx_epc_page *epc_page) 136 { 137 struct sgx_encl_page *page = epc_page->owner; 138 unsigned long addr = page->desc & PAGE_MASK; 139 struct sgx_encl *encl = page->encl; 140 int ret; 141 142 sgx_zap_enclave_ptes(encl, addr); 143 144 mutex_lock(&encl->lock); 145 146 ret = __eblock(sgx_get_epc_virt_addr(epc_page)); 147 if (encls_failed(ret)) 148 ENCLS_WARN(ret, "EBLOCK"); 149 150 mutex_unlock(&encl->lock); 151 } 152 153 static int __sgx_encl_ewb(struct sgx_epc_page *epc_page, void *va_slot, 154 struct sgx_backing *backing) 155 { 156 struct sgx_pageinfo pginfo; 157 int ret; 158 159 pginfo.addr = 0; 160 pginfo.secs = 0; 161 162 pginfo.contents = (unsigned long)kmap_atomic(backing->contents); 163 pginfo.metadata = (unsigned long)kmap_atomic(backing->pcmd) + 164 backing->pcmd_offset; 165 166 ret = __ewb(&pginfo, sgx_get_epc_virt_addr(epc_page), va_slot); 167 set_page_dirty(backing->pcmd); 168 set_page_dirty(backing->contents); 169 170 kunmap_atomic((void *)(unsigned long)(pginfo.metadata - 171 backing->pcmd_offset)); 172 kunmap_atomic((void *)(unsigned long)pginfo.contents); 173 174 return ret; 175 } 176 177 static void sgx_ipi_cb(void *info) 178 { 179 } 180 181 /* 182 * Swap page to the regular memory transformed to the blocked state by using 183 * EBLOCK, which means that it can no longer be referenced (no new TLB entries). 184 * 185 * The first trial just tries to write the page assuming that some other thread 186 * has reset the count for threads inside the enclave by using ETRACK, and 187 * previous thread count has been zeroed out. The second trial calls ETRACK 188 * before EWB. If that fails we kick all the HW threads out, and then do EWB, 189 * which should be guaranteed the succeed. 190 */ 191 static void sgx_encl_ewb(struct sgx_epc_page *epc_page, 192 struct sgx_backing *backing) 193 { 194 struct sgx_encl_page *encl_page = epc_page->owner; 195 struct sgx_encl *encl = encl_page->encl; 196 struct sgx_va_page *va_page; 197 unsigned int va_offset; 198 void *va_slot; 199 int ret; 200 201 encl_page->desc &= ~SGX_ENCL_PAGE_BEING_RECLAIMED; 202 203 va_page = list_first_entry(&encl->va_pages, struct sgx_va_page, 204 list); 205 va_offset = sgx_alloc_va_slot(va_page); 206 va_slot = sgx_get_epc_virt_addr(va_page->epc_page) + va_offset; 207 if (sgx_va_page_full(va_page)) 208 list_move_tail(&va_page->list, &encl->va_pages); 209 210 ret = __sgx_encl_ewb(epc_page, va_slot, backing); 211 if (ret == SGX_NOT_TRACKED) { 212 ret = __etrack(sgx_get_epc_virt_addr(encl->secs.epc_page)); 213 if (ret) { 214 if (encls_failed(ret)) 215 ENCLS_WARN(ret, "ETRACK"); 216 } 217 218 ret = __sgx_encl_ewb(epc_page, va_slot, backing); 219 if (ret == SGX_NOT_TRACKED) { 220 /* 221 * Slow path, send IPIs to kick cpus out of the 222 * enclave. Note, it's imperative that the cpu 223 * mask is generated *after* ETRACK, else we'll 224 * miss cpus that entered the enclave between 225 * generating the mask and incrementing epoch. 226 */ 227 on_each_cpu_mask(sgx_encl_cpumask(encl), 228 sgx_ipi_cb, NULL, 1); 229 ret = __sgx_encl_ewb(epc_page, va_slot, backing); 230 } 231 } 232 233 if (ret) { 234 if (encls_failed(ret)) 235 ENCLS_WARN(ret, "EWB"); 236 237 sgx_free_va_slot(va_page, va_offset); 238 } else { 239 encl_page->desc |= va_offset; 240 encl_page->va_page = va_page; 241 } 242 } 243 244 static void sgx_reclaimer_write(struct sgx_epc_page *epc_page, 245 struct sgx_backing *backing) 246 { 247 struct sgx_encl_page *encl_page = epc_page->owner; 248 struct sgx_encl *encl = encl_page->encl; 249 struct sgx_backing secs_backing; 250 int ret; 251 252 mutex_lock(&encl->lock); 253 254 sgx_encl_ewb(epc_page, backing); 255 encl_page->epc_page = NULL; 256 encl->secs_child_cnt--; 257 sgx_encl_put_backing(backing); 258 259 if (!encl->secs_child_cnt && test_bit(SGX_ENCL_INITIALIZED, &encl->flags)) { 260 ret = sgx_encl_get_backing(encl, PFN_DOWN(encl->size), 261 &secs_backing); 262 if (ret) 263 goto out; 264 265 sgx_encl_ewb(encl->secs.epc_page, &secs_backing); 266 267 sgx_encl_free_epc_page(encl->secs.epc_page); 268 encl->secs.epc_page = NULL; 269 270 sgx_encl_put_backing(&secs_backing); 271 } 272 273 out: 274 mutex_unlock(&encl->lock); 275 } 276 277 /* 278 * Take a fixed number of pages from the head of the active page pool and 279 * reclaim them to the enclave's private shmem files. Skip the pages, which have 280 * been accessed since the last scan. Move those pages to the tail of active 281 * page pool so that the pages get scanned in LRU like fashion. 282 * 283 * Batch process a chunk of pages (at the moment 16) in order to degrade amount 284 * of IPI's and ETRACK's potentially required. sgx_encl_ewb() does degrade a bit 285 * among the HW threads with three stage EWB pipeline (EWB, ETRACK + EWB and IPI 286 * + EWB) but not sufficiently. Reclaiming one page at a time would also be 287 * problematic as it would increase the lock contention too much, which would 288 * halt forward progress. 289 */ 290 static void sgx_reclaim_pages(void) 291 { 292 struct sgx_epc_page *chunk[SGX_NR_TO_SCAN]; 293 struct sgx_backing backing[SGX_NR_TO_SCAN]; 294 struct sgx_encl_page *encl_page; 295 struct sgx_epc_page *epc_page; 296 pgoff_t page_index; 297 int cnt = 0; 298 int ret; 299 int i; 300 301 spin_lock(&sgx_reclaimer_lock); 302 for (i = 0; i < SGX_NR_TO_SCAN; i++) { 303 if (list_empty(&sgx_active_page_list)) 304 break; 305 306 epc_page = list_first_entry(&sgx_active_page_list, 307 struct sgx_epc_page, list); 308 list_del_init(&epc_page->list); 309 encl_page = epc_page->owner; 310 311 if (kref_get_unless_zero(&encl_page->encl->refcount) != 0) 312 chunk[cnt++] = epc_page; 313 else 314 /* The owner is freeing the page. No need to add the 315 * page back to the list of reclaimable pages. 316 */ 317 epc_page->flags &= ~SGX_EPC_PAGE_RECLAIMER_TRACKED; 318 } 319 spin_unlock(&sgx_reclaimer_lock); 320 321 for (i = 0; i < cnt; i++) { 322 epc_page = chunk[i]; 323 encl_page = epc_page->owner; 324 325 if (!sgx_reclaimer_age(epc_page)) 326 goto skip; 327 328 page_index = PFN_DOWN(encl_page->desc - encl_page->encl->base); 329 330 mutex_lock(&encl_page->encl->lock); 331 ret = sgx_encl_get_backing(encl_page->encl, page_index, &backing[i]); 332 if (ret) { 333 mutex_unlock(&encl_page->encl->lock); 334 goto skip; 335 } 336 337 encl_page->desc |= SGX_ENCL_PAGE_BEING_RECLAIMED; 338 mutex_unlock(&encl_page->encl->lock); 339 continue; 340 341 skip: 342 spin_lock(&sgx_reclaimer_lock); 343 list_add_tail(&epc_page->list, &sgx_active_page_list); 344 spin_unlock(&sgx_reclaimer_lock); 345 346 kref_put(&encl_page->encl->refcount, sgx_encl_release); 347 348 chunk[i] = NULL; 349 } 350 351 for (i = 0; i < cnt; i++) { 352 epc_page = chunk[i]; 353 if (epc_page) 354 sgx_reclaimer_block(epc_page); 355 } 356 357 for (i = 0; i < cnt; i++) { 358 epc_page = chunk[i]; 359 if (!epc_page) 360 continue; 361 362 encl_page = epc_page->owner; 363 sgx_reclaimer_write(epc_page, &backing[i]); 364 365 kref_put(&encl_page->encl->refcount, sgx_encl_release); 366 epc_page->flags &= ~SGX_EPC_PAGE_RECLAIMER_TRACKED; 367 368 sgx_free_epc_page(epc_page); 369 } 370 } 371 372 static bool sgx_should_reclaim(unsigned long watermark) 373 { 374 return atomic_long_read(&sgx_nr_free_pages) < watermark && 375 !list_empty(&sgx_active_page_list); 376 } 377 378 static int ksgxd(void *p) 379 { 380 set_freezable(); 381 382 /* 383 * Sanitize pages in order to recover from kexec(). The 2nd pass is 384 * required for SECS pages, whose child pages blocked EREMOVE. 385 */ 386 __sgx_sanitize_pages(&sgx_dirty_page_list); 387 __sgx_sanitize_pages(&sgx_dirty_page_list); 388 389 /* sanity check: */ 390 WARN_ON(!list_empty(&sgx_dirty_page_list)); 391 392 while (!kthread_should_stop()) { 393 if (try_to_freeze()) 394 continue; 395 396 wait_event_freezable(ksgxd_waitq, 397 kthread_should_stop() || 398 sgx_should_reclaim(SGX_NR_HIGH_PAGES)); 399 400 if (sgx_should_reclaim(SGX_NR_HIGH_PAGES)) 401 sgx_reclaim_pages(); 402 403 cond_resched(); 404 } 405 406 return 0; 407 } 408 409 static bool __init sgx_page_reclaimer_init(void) 410 { 411 struct task_struct *tsk; 412 413 tsk = kthread_run(ksgxd, NULL, "ksgxd"); 414 if (IS_ERR(tsk)) 415 return false; 416 417 ksgxd_tsk = tsk; 418 419 return true; 420 } 421 422 static struct sgx_epc_page *__sgx_alloc_epc_page_from_node(int nid) 423 { 424 struct sgx_numa_node *node = &sgx_numa_nodes[nid]; 425 struct sgx_epc_page *page = NULL; 426 427 spin_lock(&node->lock); 428 429 if (list_empty(&node->free_page_list)) { 430 spin_unlock(&node->lock); 431 return NULL; 432 } 433 434 page = list_first_entry(&node->free_page_list, struct sgx_epc_page, list); 435 list_del_init(&page->list); 436 page->flags = 0; 437 438 spin_unlock(&node->lock); 439 atomic_long_dec(&sgx_nr_free_pages); 440 441 return page; 442 } 443 444 /** 445 * __sgx_alloc_epc_page() - Allocate an EPC page 446 * 447 * Iterate through NUMA nodes and reserve ia free EPC page to the caller. Start 448 * from the NUMA node, where the caller is executing. 449 * 450 * Return: 451 * - an EPC page: A borrowed EPC pages were available. 452 * - NULL: Out of EPC pages. 453 */ 454 struct sgx_epc_page *__sgx_alloc_epc_page(void) 455 { 456 struct sgx_epc_page *page; 457 int nid_of_current = numa_node_id(); 458 int nid = nid_of_current; 459 460 if (node_isset(nid_of_current, sgx_numa_mask)) { 461 page = __sgx_alloc_epc_page_from_node(nid_of_current); 462 if (page) 463 return page; 464 } 465 466 /* Fall back to the non-local NUMA nodes: */ 467 while (true) { 468 nid = next_node_in(nid, sgx_numa_mask); 469 if (nid == nid_of_current) 470 break; 471 472 page = __sgx_alloc_epc_page_from_node(nid); 473 if (page) 474 return page; 475 } 476 477 return ERR_PTR(-ENOMEM); 478 } 479 480 /** 481 * sgx_mark_page_reclaimable() - Mark a page as reclaimable 482 * @page: EPC page 483 * 484 * Mark a page as reclaimable and add it to the active page list. Pages 485 * are automatically removed from the active list when freed. 486 */ 487 void sgx_mark_page_reclaimable(struct sgx_epc_page *page) 488 { 489 spin_lock(&sgx_reclaimer_lock); 490 page->flags |= SGX_EPC_PAGE_RECLAIMER_TRACKED; 491 list_add_tail(&page->list, &sgx_active_page_list); 492 spin_unlock(&sgx_reclaimer_lock); 493 } 494 495 /** 496 * sgx_unmark_page_reclaimable() - Remove a page from the reclaim list 497 * @page: EPC page 498 * 499 * Clear the reclaimable flag and remove the page from the active page list. 500 * 501 * Return: 502 * 0 on success, 503 * -EBUSY if the page is in the process of being reclaimed 504 */ 505 int sgx_unmark_page_reclaimable(struct sgx_epc_page *page) 506 { 507 spin_lock(&sgx_reclaimer_lock); 508 if (page->flags & SGX_EPC_PAGE_RECLAIMER_TRACKED) { 509 /* The page is being reclaimed. */ 510 if (list_empty(&page->list)) { 511 spin_unlock(&sgx_reclaimer_lock); 512 return -EBUSY; 513 } 514 515 list_del(&page->list); 516 page->flags &= ~SGX_EPC_PAGE_RECLAIMER_TRACKED; 517 } 518 spin_unlock(&sgx_reclaimer_lock); 519 520 return 0; 521 } 522 523 /** 524 * sgx_alloc_epc_page() - Allocate an EPC page 525 * @owner: the owner of the EPC page 526 * @reclaim: reclaim pages if necessary 527 * 528 * Iterate through EPC sections and borrow a free EPC page to the caller. When a 529 * page is no longer needed it must be released with sgx_free_epc_page(). If 530 * @reclaim is set to true, directly reclaim pages when we are out of pages. No 531 * mm's can be locked when @reclaim is set to true. 532 * 533 * Finally, wake up ksgxd when the number of pages goes below the watermark 534 * before returning back to the caller. 535 * 536 * Return: 537 * an EPC page, 538 * -errno on error 539 */ 540 struct sgx_epc_page *sgx_alloc_epc_page(void *owner, bool reclaim) 541 { 542 struct sgx_epc_page *page; 543 544 for ( ; ; ) { 545 page = __sgx_alloc_epc_page(); 546 if (!IS_ERR(page)) { 547 page->owner = owner; 548 break; 549 } 550 551 if (list_empty(&sgx_active_page_list)) 552 return ERR_PTR(-ENOMEM); 553 554 if (!reclaim) { 555 page = ERR_PTR(-EBUSY); 556 break; 557 } 558 559 if (signal_pending(current)) { 560 page = ERR_PTR(-ERESTARTSYS); 561 break; 562 } 563 564 sgx_reclaim_pages(); 565 cond_resched(); 566 } 567 568 if (sgx_should_reclaim(SGX_NR_LOW_PAGES)) 569 wake_up(&ksgxd_waitq); 570 571 return page; 572 } 573 574 /** 575 * sgx_free_epc_page() - Free an EPC page 576 * @page: an EPC page 577 * 578 * Put the EPC page back to the list of free pages. It's the caller's 579 * responsibility to make sure that the page is in uninitialized state. In other 580 * words, do EREMOVE, EWB or whatever operation is necessary before calling 581 * this function. 582 */ 583 void sgx_free_epc_page(struct sgx_epc_page *page) 584 { 585 struct sgx_epc_section *section = &sgx_epc_sections[page->section]; 586 struct sgx_numa_node *node = section->node; 587 588 spin_lock(&node->lock); 589 590 page->owner = NULL; 591 if (page->poison) 592 list_add(&page->list, &node->sgx_poison_page_list); 593 else 594 list_add_tail(&page->list, &node->free_page_list); 595 page->flags = SGX_EPC_PAGE_IS_FREE; 596 597 spin_unlock(&node->lock); 598 atomic_long_inc(&sgx_nr_free_pages); 599 } 600 601 static bool __init sgx_setup_epc_section(u64 phys_addr, u64 size, 602 unsigned long index, 603 struct sgx_epc_section *section) 604 { 605 unsigned long nr_pages = size >> PAGE_SHIFT; 606 unsigned long i; 607 608 section->virt_addr = memremap(phys_addr, size, MEMREMAP_WB); 609 if (!section->virt_addr) 610 return false; 611 612 section->pages = vmalloc(nr_pages * sizeof(struct sgx_epc_page)); 613 if (!section->pages) { 614 memunmap(section->virt_addr); 615 return false; 616 } 617 618 section->phys_addr = phys_addr; 619 xa_store_range(&sgx_epc_address_space, section->phys_addr, 620 phys_addr + size - 1, section, GFP_KERNEL); 621 622 for (i = 0; i < nr_pages; i++) { 623 section->pages[i].section = index; 624 section->pages[i].flags = 0; 625 section->pages[i].owner = NULL; 626 section->pages[i].poison = 0; 627 list_add_tail(§ion->pages[i].list, &sgx_dirty_page_list); 628 } 629 630 return true; 631 } 632 633 bool arch_is_platform_page(u64 paddr) 634 { 635 return !!xa_load(&sgx_epc_address_space, paddr); 636 } 637 EXPORT_SYMBOL_GPL(arch_is_platform_page); 638 639 static struct sgx_epc_page *sgx_paddr_to_page(u64 paddr) 640 { 641 struct sgx_epc_section *section; 642 643 section = xa_load(&sgx_epc_address_space, paddr); 644 if (!section) 645 return NULL; 646 647 return §ion->pages[PFN_DOWN(paddr - section->phys_addr)]; 648 } 649 650 /* 651 * Called in process context to handle a hardware reported 652 * error in an SGX EPC page. 653 * If the MF_ACTION_REQUIRED bit is set in flags, then the 654 * context is the task that consumed the poison data. Otherwise 655 * this is called from a kernel thread unrelated to the page. 656 */ 657 int arch_memory_failure(unsigned long pfn, int flags) 658 { 659 struct sgx_epc_page *page = sgx_paddr_to_page(pfn << PAGE_SHIFT); 660 struct sgx_epc_section *section; 661 struct sgx_numa_node *node; 662 663 /* 664 * mm/memory-failure.c calls this routine for all errors 665 * where there isn't a "struct page" for the address. But that 666 * includes other address ranges besides SGX. 667 */ 668 if (!page) 669 return -ENXIO; 670 671 /* 672 * If poison was consumed synchronously. Send a SIGBUS to 673 * the task. Hardware has already exited the SGX enclave and 674 * will not allow re-entry to an enclave that has a memory 675 * error. The signal may help the task understand why the 676 * enclave is broken. 677 */ 678 if (flags & MF_ACTION_REQUIRED) 679 force_sig(SIGBUS); 680 681 section = &sgx_epc_sections[page->section]; 682 node = section->node; 683 684 spin_lock(&node->lock); 685 686 /* Already poisoned? Nothing more to do */ 687 if (page->poison) 688 goto out; 689 690 page->poison = 1; 691 692 /* 693 * If the page is on a free list, move it to the per-node 694 * poison page list. 695 */ 696 if (page->flags & SGX_EPC_PAGE_IS_FREE) { 697 list_move(&page->list, &node->sgx_poison_page_list); 698 goto out; 699 } 700 701 /* 702 * TBD: Add additional plumbing to enable pre-emptive 703 * action for asynchronous poison notification. Until 704 * then just hope that the poison: 705 * a) is not accessed - sgx_free_epc_page() will deal with it 706 * when the user gives it back 707 * b) results in a recoverable machine check rather than 708 * a fatal one 709 */ 710 out: 711 spin_unlock(&node->lock); 712 return 0; 713 } 714 715 /** 716 * A section metric is concatenated in a way that @low bits 12-31 define the 717 * bits 12-31 of the metric and @high bits 0-19 define the bits 32-51 of the 718 * metric. 719 */ 720 static inline u64 __init sgx_calc_section_metric(u64 low, u64 high) 721 { 722 return (low & GENMASK_ULL(31, 12)) + 723 ((high & GENMASK_ULL(19, 0)) << 32); 724 } 725 726 #ifdef CONFIG_NUMA 727 static ssize_t sgx_total_bytes_show(struct device *dev, struct device_attribute *attr, char *buf) 728 { 729 return sysfs_emit(buf, "%lu\n", sgx_numa_nodes[dev->id].size); 730 } 731 static DEVICE_ATTR_RO(sgx_total_bytes); 732 733 static umode_t arch_node_attr_is_visible(struct kobject *kobj, 734 struct attribute *attr, int idx) 735 { 736 /* Make all x86/ attributes invisible when SGX is not initialized: */ 737 if (nodes_empty(sgx_numa_mask)) 738 return 0; 739 740 return attr->mode; 741 } 742 743 static struct attribute *arch_node_dev_attrs[] = { 744 &dev_attr_sgx_total_bytes.attr, 745 NULL, 746 }; 747 748 const struct attribute_group arch_node_dev_group = { 749 .name = "x86", 750 .attrs = arch_node_dev_attrs, 751 .is_visible = arch_node_attr_is_visible, 752 }; 753 754 static void __init arch_update_sysfs_visibility(int nid) 755 { 756 struct node *node = node_devices[nid]; 757 int ret; 758 759 ret = sysfs_update_group(&node->dev.kobj, &arch_node_dev_group); 760 761 if (ret) 762 pr_err("sysfs update failed (%d), files may be invisible", ret); 763 } 764 #else /* !CONFIG_NUMA */ 765 static void __init arch_update_sysfs_visibility(int nid) {} 766 #endif 767 768 static bool __init sgx_page_cache_init(void) 769 { 770 u32 eax, ebx, ecx, edx, type; 771 u64 pa, size; 772 int nid; 773 int i; 774 775 sgx_numa_nodes = kmalloc_array(num_possible_nodes(), sizeof(*sgx_numa_nodes), GFP_KERNEL); 776 if (!sgx_numa_nodes) 777 return false; 778 779 for (i = 0; i < ARRAY_SIZE(sgx_epc_sections); i++) { 780 cpuid_count(SGX_CPUID, i + SGX_CPUID_EPC, &eax, &ebx, &ecx, &edx); 781 782 type = eax & SGX_CPUID_EPC_MASK; 783 if (type == SGX_CPUID_EPC_INVALID) 784 break; 785 786 if (type != SGX_CPUID_EPC_SECTION) { 787 pr_err_once("Unknown EPC section type: %u\n", type); 788 break; 789 } 790 791 pa = sgx_calc_section_metric(eax, ebx); 792 size = sgx_calc_section_metric(ecx, edx); 793 794 pr_info("EPC section 0x%llx-0x%llx\n", pa, pa + size - 1); 795 796 if (!sgx_setup_epc_section(pa, size, i, &sgx_epc_sections[i])) { 797 pr_err("No free memory for an EPC section\n"); 798 break; 799 } 800 801 nid = numa_map_to_online_node(phys_to_target_node(pa)); 802 if (nid == NUMA_NO_NODE) { 803 /* The physical address is already printed above. */ 804 pr_warn(FW_BUG "Unable to map EPC section to online node. Fallback to the NUMA node 0.\n"); 805 nid = 0; 806 } 807 808 if (!node_isset(nid, sgx_numa_mask)) { 809 spin_lock_init(&sgx_numa_nodes[nid].lock); 810 INIT_LIST_HEAD(&sgx_numa_nodes[nid].free_page_list); 811 INIT_LIST_HEAD(&sgx_numa_nodes[nid].sgx_poison_page_list); 812 node_set(nid, sgx_numa_mask); 813 sgx_numa_nodes[nid].size = 0; 814 815 /* Make SGX-specific node sysfs files visible: */ 816 arch_update_sysfs_visibility(nid); 817 } 818 819 sgx_epc_sections[i].node = &sgx_numa_nodes[nid]; 820 sgx_numa_nodes[nid].size += size; 821 822 sgx_nr_epc_sections++; 823 } 824 825 if (!sgx_nr_epc_sections) { 826 pr_err("There are zero EPC sections.\n"); 827 return false; 828 } 829 830 return true; 831 } 832 833 /* 834 * Update the SGX_LEPUBKEYHASH MSRs to the values specified by caller. 835 * Bare-metal driver requires to update them to hash of enclave's signer 836 * before EINIT. KVM needs to update them to guest's virtual MSR values 837 * before doing EINIT from guest. 838 */ 839 void sgx_update_lepubkeyhash(u64 *lepubkeyhash) 840 { 841 int i; 842 843 WARN_ON_ONCE(preemptible()); 844 845 for (i = 0; i < 4; i++) 846 wrmsrl(MSR_IA32_SGXLEPUBKEYHASH0 + i, lepubkeyhash[i]); 847 } 848 849 const struct file_operations sgx_provision_fops = { 850 .owner = THIS_MODULE, 851 }; 852 853 static struct miscdevice sgx_dev_provision = { 854 .minor = MISC_DYNAMIC_MINOR, 855 .name = "sgx_provision", 856 .nodename = "sgx_provision", 857 .fops = &sgx_provision_fops, 858 }; 859 860 /** 861 * sgx_set_attribute() - Update allowed attributes given file descriptor 862 * @allowed_attributes: Pointer to allowed enclave attributes 863 * @attribute_fd: File descriptor for specific attribute 864 * 865 * Append enclave attribute indicated by file descriptor to allowed 866 * attributes. Currently only SGX_ATTR_PROVISIONKEY indicated by 867 * /dev/sgx_provision is supported. 868 * 869 * Return: 870 * -0: SGX_ATTR_PROVISIONKEY is appended to allowed_attributes 871 * -EINVAL: Invalid, or not supported file descriptor 872 */ 873 int sgx_set_attribute(unsigned long *allowed_attributes, 874 unsigned int attribute_fd) 875 { 876 struct file *file; 877 878 file = fget(attribute_fd); 879 if (!file) 880 return -EINVAL; 881 882 if (file->f_op != &sgx_provision_fops) { 883 fput(file); 884 return -EINVAL; 885 } 886 887 *allowed_attributes |= SGX_ATTR_PROVISIONKEY; 888 889 fput(file); 890 return 0; 891 } 892 EXPORT_SYMBOL_GPL(sgx_set_attribute); 893 894 static int __init sgx_init(void) 895 { 896 int ret; 897 int i; 898 899 if (!cpu_feature_enabled(X86_FEATURE_SGX)) 900 return -ENODEV; 901 902 if (!sgx_page_cache_init()) 903 return -ENOMEM; 904 905 if (!sgx_page_reclaimer_init()) { 906 ret = -ENOMEM; 907 goto err_page_cache; 908 } 909 910 ret = misc_register(&sgx_dev_provision); 911 if (ret) 912 goto err_kthread; 913 914 /* 915 * Always try to initialize the native *and* KVM drivers. 916 * The KVM driver is less picky than the native one and 917 * can function if the native one is not supported on the 918 * current system or fails to initialize. 919 * 920 * Error out only if both fail to initialize. 921 */ 922 ret = sgx_drv_init(); 923 924 if (sgx_vepc_init() && ret) 925 goto err_provision; 926 927 return 0; 928 929 err_provision: 930 misc_deregister(&sgx_dev_provision); 931 932 err_kthread: 933 kthread_stop(ksgxd_tsk); 934 935 err_page_cache: 936 for (i = 0; i < sgx_nr_epc_sections; i++) { 937 vfree(sgx_epc_sections[i].pages); 938 memunmap(sgx_epc_sections[i].virt_addr); 939 } 940 941 return ret; 942 } 943 944 device_initcall(sgx_init); 945