xref: /openbmc/linux/arch/x86/kernel/cpu/sgx/main.c (revision f89c2f9bf5a64f619de06ded4349dff5a35da860)
1 // SPDX-License-Identifier: GPL-2.0
2 /*  Copyright(c) 2016-20 Intel Corporation. */
3 
4 #include <linux/file.h>
5 #include <linux/freezer.h>
6 #include <linux/highmem.h>
7 #include <linux/kthread.h>
8 #include <linux/miscdevice.h>
9 #include <linux/node.h>
10 #include <linux/pagemap.h>
11 #include <linux/ratelimit.h>
12 #include <linux/sched/mm.h>
13 #include <linux/sched/signal.h>
14 #include <linux/slab.h>
15 #include <linux/sysfs.h>
16 #include <asm/sgx.h>
17 #include "driver.h"
18 #include "encl.h"
19 #include "encls.h"
20 
21 struct sgx_epc_section sgx_epc_sections[SGX_MAX_EPC_SECTIONS];
22 static int sgx_nr_epc_sections;
23 static struct task_struct *ksgxd_tsk;
24 static DECLARE_WAIT_QUEUE_HEAD(ksgxd_waitq);
25 static DEFINE_XARRAY(sgx_epc_address_space);
26 
27 /*
28  * These variables are part of the state of the reclaimer, and must be accessed
29  * with sgx_reclaimer_lock acquired.
30  */
31 static LIST_HEAD(sgx_active_page_list);
32 static DEFINE_SPINLOCK(sgx_reclaimer_lock);
33 
34 static atomic_long_t sgx_nr_free_pages = ATOMIC_LONG_INIT(0);
35 
36 /* Nodes with one or more EPC sections. */
37 static nodemask_t sgx_numa_mask;
38 
39 /*
40  * Array with one list_head for each possible NUMA node.  Each
41  * list contains all the sgx_epc_section's which are on that
42  * node.
43  */
44 static struct sgx_numa_node *sgx_numa_nodes;
45 
46 static LIST_HEAD(sgx_dirty_page_list);
47 
48 /*
49  * Reset post-kexec EPC pages to the uninitialized state. The pages are removed
50  * from the input list, and made available for the page allocator. SECS pages
51  * prepending their children in the input list are left intact.
52  */
53 static void __sgx_sanitize_pages(struct list_head *dirty_page_list)
54 {
55 	struct sgx_epc_page *page;
56 	LIST_HEAD(dirty);
57 	int ret;
58 
59 	/* dirty_page_list is thread-local, no need for a lock: */
60 	while (!list_empty(dirty_page_list)) {
61 		if (kthread_should_stop())
62 			return;
63 
64 		page = list_first_entry(dirty_page_list, struct sgx_epc_page, list);
65 
66 		/*
67 		 * Checking page->poison without holding the node->lock
68 		 * is racy, but losing the race (i.e. poison is set just
69 		 * after the check) just means __eremove() will be uselessly
70 		 * called for a page that sgx_free_epc_page() will put onto
71 		 * the node->sgx_poison_page_list later.
72 		 */
73 		if (page->poison) {
74 			struct sgx_epc_section *section = &sgx_epc_sections[page->section];
75 			struct sgx_numa_node *node = section->node;
76 
77 			spin_lock(&node->lock);
78 			list_move(&page->list, &node->sgx_poison_page_list);
79 			spin_unlock(&node->lock);
80 
81 			continue;
82 		}
83 
84 		ret = __eremove(sgx_get_epc_virt_addr(page));
85 		if (!ret) {
86 			/*
87 			 * page is now sanitized.  Make it available via the SGX
88 			 * page allocator:
89 			 */
90 			list_del(&page->list);
91 			sgx_free_epc_page(page);
92 		} else {
93 			/* The page is not yet clean - move to the dirty list. */
94 			list_move_tail(&page->list, &dirty);
95 		}
96 
97 		cond_resched();
98 	}
99 
100 	list_splice(&dirty, dirty_page_list);
101 }
102 
103 static bool sgx_reclaimer_age(struct sgx_epc_page *epc_page)
104 {
105 	struct sgx_encl_page *page = epc_page->owner;
106 	struct sgx_encl *encl = page->encl;
107 	struct sgx_encl_mm *encl_mm;
108 	bool ret = true;
109 	int idx;
110 
111 	idx = srcu_read_lock(&encl->srcu);
112 
113 	list_for_each_entry_rcu(encl_mm, &encl->mm_list, list) {
114 		if (!mmget_not_zero(encl_mm->mm))
115 			continue;
116 
117 		mmap_read_lock(encl_mm->mm);
118 		ret = !sgx_encl_test_and_clear_young(encl_mm->mm, page);
119 		mmap_read_unlock(encl_mm->mm);
120 
121 		mmput_async(encl_mm->mm);
122 
123 		if (!ret)
124 			break;
125 	}
126 
127 	srcu_read_unlock(&encl->srcu, idx);
128 
129 	if (!ret)
130 		return false;
131 
132 	return true;
133 }
134 
135 static void sgx_reclaimer_block(struct sgx_epc_page *epc_page)
136 {
137 	struct sgx_encl_page *page = epc_page->owner;
138 	unsigned long addr = page->desc & PAGE_MASK;
139 	struct sgx_encl *encl = page->encl;
140 	int ret;
141 
142 	sgx_zap_enclave_ptes(encl, addr);
143 
144 	mutex_lock(&encl->lock);
145 
146 	ret = __eblock(sgx_get_epc_virt_addr(epc_page));
147 	if (encls_failed(ret))
148 		ENCLS_WARN(ret, "EBLOCK");
149 
150 	mutex_unlock(&encl->lock);
151 }
152 
153 static int __sgx_encl_ewb(struct sgx_epc_page *epc_page, void *va_slot,
154 			  struct sgx_backing *backing)
155 {
156 	struct sgx_pageinfo pginfo;
157 	int ret;
158 
159 	pginfo.addr = 0;
160 	pginfo.secs = 0;
161 
162 	pginfo.contents = (unsigned long)kmap_atomic(backing->contents);
163 	pginfo.metadata = (unsigned long)kmap_atomic(backing->pcmd) +
164 			  backing->pcmd_offset;
165 
166 	ret = __ewb(&pginfo, sgx_get_epc_virt_addr(epc_page), va_slot);
167 	set_page_dirty(backing->pcmd);
168 	set_page_dirty(backing->contents);
169 
170 	kunmap_atomic((void *)(unsigned long)(pginfo.metadata -
171 					      backing->pcmd_offset));
172 	kunmap_atomic((void *)(unsigned long)pginfo.contents);
173 
174 	return ret;
175 }
176 
177 static void sgx_ipi_cb(void *info)
178 {
179 }
180 
181 /*
182  * Swap page to the regular memory transformed to the blocked state by using
183  * EBLOCK, which means that it can no longer be referenced (no new TLB entries).
184  *
185  * The first trial just tries to write the page assuming that some other thread
186  * has reset the count for threads inside the enclave by using ETRACK, and
187  * previous thread count has been zeroed out. The second trial calls ETRACK
188  * before EWB. If that fails we kick all the HW threads out, and then do EWB,
189  * which should be guaranteed the succeed.
190  */
191 static void sgx_encl_ewb(struct sgx_epc_page *epc_page,
192 			 struct sgx_backing *backing)
193 {
194 	struct sgx_encl_page *encl_page = epc_page->owner;
195 	struct sgx_encl *encl = encl_page->encl;
196 	struct sgx_va_page *va_page;
197 	unsigned int va_offset;
198 	void *va_slot;
199 	int ret;
200 
201 	encl_page->desc &= ~SGX_ENCL_PAGE_BEING_RECLAIMED;
202 
203 	va_page = list_first_entry(&encl->va_pages, struct sgx_va_page,
204 				   list);
205 	va_offset = sgx_alloc_va_slot(va_page);
206 	va_slot = sgx_get_epc_virt_addr(va_page->epc_page) + va_offset;
207 	if (sgx_va_page_full(va_page))
208 		list_move_tail(&va_page->list, &encl->va_pages);
209 
210 	ret = __sgx_encl_ewb(epc_page, va_slot, backing);
211 	if (ret == SGX_NOT_TRACKED) {
212 		ret = __etrack(sgx_get_epc_virt_addr(encl->secs.epc_page));
213 		if (ret) {
214 			if (encls_failed(ret))
215 				ENCLS_WARN(ret, "ETRACK");
216 		}
217 
218 		ret = __sgx_encl_ewb(epc_page, va_slot, backing);
219 		if (ret == SGX_NOT_TRACKED) {
220 			/*
221 			 * Slow path, send IPIs to kick cpus out of the
222 			 * enclave.  Note, it's imperative that the cpu
223 			 * mask is generated *after* ETRACK, else we'll
224 			 * miss cpus that entered the enclave between
225 			 * generating the mask and incrementing epoch.
226 			 */
227 			on_each_cpu_mask(sgx_encl_cpumask(encl),
228 					 sgx_ipi_cb, NULL, 1);
229 			ret = __sgx_encl_ewb(epc_page, va_slot, backing);
230 		}
231 	}
232 
233 	if (ret) {
234 		if (encls_failed(ret))
235 			ENCLS_WARN(ret, "EWB");
236 
237 		sgx_free_va_slot(va_page, va_offset);
238 	} else {
239 		encl_page->desc |= va_offset;
240 		encl_page->va_page = va_page;
241 	}
242 }
243 
244 static void sgx_reclaimer_write(struct sgx_epc_page *epc_page,
245 				struct sgx_backing *backing)
246 {
247 	struct sgx_encl_page *encl_page = epc_page->owner;
248 	struct sgx_encl *encl = encl_page->encl;
249 	struct sgx_backing secs_backing;
250 	int ret;
251 
252 	mutex_lock(&encl->lock);
253 
254 	sgx_encl_ewb(epc_page, backing);
255 	encl_page->epc_page = NULL;
256 	encl->secs_child_cnt--;
257 	sgx_encl_put_backing(backing);
258 
259 	if (!encl->secs_child_cnt && test_bit(SGX_ENCL_INITIALIZED, &encl->flags)) {
260 		ret = sgx_encl_get_backing(encl, PFN_DOWN(encl->size),
261 					   &secs_backing);
262 		if (ret)
263 			goto out;
264 
265 		sgx_encl_ewb(encl->secs.epc_page, &secs_backing);
266 
267 		sgx_encl_free_epc_page(encl->secs.epc_page);
268 		encl->secs.epc_page = NULL;
269 
270 		sgx_encl_put_backing(&secs_backing);
271 	}
272 
273 out:
274 	mutex_unlock(&encl->lock);
275 }
276 
277 /*
278  * Take a fixed number of pages from the head of the active page pool and
279  * reclaim them to the enclave's private shmem files. Skip the pages, which have
280  * been accessed since the last scan. Move those pages to the tail of active
281  * page pool so that the pages get scanned in LRU like fashion.
282  *
283  * Batch process a chunk of pages (at the moment 16) in order to degrade amount
284  * of IPI's and ETRACK's potentially required. sgx_encl_ewb() does degrade a bit
285  * among the HW threads with three stage EWB pipeline (EWB, ETRACK + EWB and IPI
286  * + EWB) but not sufficiently. Reclaiming one page at a time would also be
287  * problematic as it would increase the lock contention too much, which would
288  * halt forward progress.
289  */
290 static void sgx_reclaim_pages(void)
291 {
292 	struct sgx_epc_page *chunk[SGX_NR_TO_SCAN];
293 	struct sgx_backing backing[SGX_NR_TO_SCAN];
294 	struct sgx_encl_page *encl_page;
295 	struct sgx_epc_page *epc_page;
296 	pgoff_t page_index;
297 	int cnt = 0;
298 	int ret;
299 	int i;
300 
301 	spin_lock(&sgx_reclaimer_lock);
302 	for (i = 0; i < SGX_NR_TO_SCAN; i++) {
303 		if (list_empty(&sgx_active_page_list))
304 			break;
305 
306 		epc_page = list_first_entry(&sgx_active_page_list,
307 					    struct sgx_epc_page, list);
308 		list_del_init(&epc_page->list);
309 		encl_page = epc_page->owner;
310 
311 		if (kref_get_unless_zero(&encl_page->encl->refcount) != 0)
312 			chunk[cnt++] = epc_page;
313 		else
314 			/* The owner is freeing the page. No need to add the
315 			 * page back to the list of reclaimable pages.
316 			 */
317 			epc_page->flags &= ~SGX_EPC_PAGE_RECLAIMER_TRACKED;
318 	}
319 	spin_unlock(&sgx_reclaimer_lock);
320 
321 	for (i = 0; i < cnt; i++) {
322 		epc_page = chunk[i];
323 		encl_page = epc_page->owner;
324 
325 		if (!sgx_reclaimer_age(epc_page))
326 			goto skip;
327 
328 		page_index = PFN_DOWN(encl_page->desc - encl_page->encl->base);
329 
330 		mutex_lock(&encl_page->encl->lock);
331 		ret = sgx_encl_get_backing(encl_page->encl, page_index, &backing[i]);
332 		if (ret) {
333 			mutex_unlock(&encl_page->encl->lock);
334 			goto skip;
335 		}
336 
337 		encl_page->desc |= SGX_ENCL_PAGE_BEING_RECLAIMED;
338 		mutex_unlock(&encl_page->encl->lock);
339 		continue;
340 
341 skip:
342 		spin_lock(&sgx_reclaimer_lock);
343 		list_add_tail(&epc_page->list, &sgx_active_page_list);
344 		spin_unlock(&sgx_reclaimer_lock);
345 
346 		kref_put(&encl_page->encl->refcount, sgx_encl_release);
347 
348 		chunk[i] = NULL;
349 	}
350 
351 	for (i = 0; i < cnt; i++) {
352 		epc_page = chunk[i];
353 		if (epc_page)
354 			sgx_reclaimer_block(epc_page);
355 	}
356 
357 	for (i = 0; i < cnt; i++) {
358 		epc_page = chunk[i];
359 		if (!epc_page)
360 			continue;
361 
362 		encl_page = epc_page->owner;
363 		sgx_reclaimer_write(epc_page, &backing[i]);
364 
365 		kref_put(&encl_page->encl->refcount, sgx_encl_release);
366 		epc_page->flags &= ~SGX_EPC_PAGE_RECLAIMER_TRACKED;
367 
368 		sgx_free_epc_page(epc_page);
369 	}
370 }
371 
372 static bool sgx_should_reclaim(unsigned long watermark)
373 {
374 	return atomic_long_read(&sgx_nr_free_pages) < watermark &&
375 	       !list_empty(&sgx_active_page_list);
376 }
377 
378 static int ksgxd(void *p)
379 {
380 	set_freezable();
381 
382 	/*
383 	 * Sanitize pages in order to recover from kexec(). The 2nd pass is
384 	 * required for SECS pages, whose child pages blocked EREMOVE.
385 	 */
386 	__sgx_sanitize_pages(&sgx_dirty_page_list);
387 	__sgx_sanitize_pages(&sgx_dirty_page_list);
388 
389 	/* sanity check: */
390 	WARN_ON(!list_empty(&sgx_dirty_page_list));
391 
392 	while (!kthread_should_stop()) {
393 		if (try_to_freeze())
394 			continue;
395 
396 		wait_event_freezable(ksgxd_waitq,
397 				     kthread_should_stop() ||
398 				     sgx_should_reclaim(SGX_NR_HIGH_PAGES));
399 
400 		if (sgx_should_reclaim(SGX_NR_HIGH_PAGES))
401 			sgx_reclaim_pages();
402 
403 		cond_resched();
404 	}
405 
406 	return 0;
407 }
408 
409 static bool __init sgx_page_reclaimer_init(void)
410 {
411 	struct task_struct *tsk;
412 
413 	tsk = kthread_run(ksgxd, NULL, "ksgxd");
414 	if (IS_ERR(tsk))
415 		return false;
416 
417 	ksgxd_tsk = tsk;
418 
419 	return true;
420 }
421 
422 static struct sgx_epc_page *__sgx_alloc_epc_page_from_node(int nid)
423 {
424 	struct sgx_numa_node *node = &sgx_numa_nodes[nid];
425 	struct sgx_epc_page *page = NULL;
426 
427 	spin_lock(&node->lock);
428 
429 	if (list_empty(&node->free_page_list)) {
430 		spin_unlock(&node->lock);
431 		return NULL;
432 	}
433 
434 	page = list_first_entry(&node->free_page_list, struct sgx_epc_page, list);
435 	list_del_init(&page->list);
436 	page->flags = 0;
437 
438 	spin_unlock(&node->lock);
439 	atomic_long_dec(&sgx_nr_free_pages);
440 
441 	return page;
442 }
443 
444 /**
445  * __sgx_alloc_epc_page() - Allocate an EPC page
446  *
447  * Iterate through NUMA nodes and reserve ia free EPC page to the caller. Start
448  * from the NUMA node, where the caller is executing.
449  *
450  * Return:
451  * - an EPC page:	A borrowed EPC pages were available.
452  * - NULL:		Out of EPC pages.
453  */
454 struct sgx_epc_page *__sgx_alloc_epc_page(void)
455 {
456 	struct sgx_epc_page *page;
457 	int nid_of_current = numa_node_id();
458 	int nid = nid_of_current;
459 
460 	if (node_isset(nid_of_current, sgx_numa_mask)) {
461 		page = __sgx_alloc_epc_page_from_node(nid_of_current);
462 		if (page)
463 			return page;
464 	}
465 
466 	/* Fall back to the non-local NUMA nodes: */
467 	while (true) {
468 		nid = next_node_in(nid, sgx_numa_mask);
469 		if (nid == nid_of_current)
470 			break;
471 
472 		page = __sgx_alloc_epc_page_from_node(nid);
473 		if (page)
474 			return page;
475 	}
476 
477 	return ERR_PTR(-ENOMEM);
478 }
479 
480 /**
481  * sgx_mark_page_reclaimable() - Mark a page as reclaimable
482  * @page:	EPC page
483  *
484  * Mark a page as reclaimable and add it to the active page list. Pages
485  * are automatically removed from the active list when freed.
486  */
487 void sgx_mark_page_reclaimable(struct sgx_epc_page *page)
488 {
489 	spin_lock(&sgx_reclaimer_lock);
490 	page->flags |= SGX_EPC_PAGE_RECLAIMER_TRACKED;
491 	list_add_tail(&page->list, &sgx_active_page_list);
492 	spin_unlock(&sgx_reclaimer_lock);
493 }
494 
495 /**
496  * sgx_unmark_page_reclaimable() - Remove a page from the reclaim list
497  * @page:	EPC page
498  *
499  * Clear the reclaimable flag and remove the page from the active page list.
500  *
501  * Return:
502  *   0 on success,
503  *   -EBUSY if the page is in the process of being reclaimed
504  */
505 int sgx_unmark_page_reclaimable(struct sgx_epc_page *page)
506 {
507 	spin_lock(&sgx_reclaimer_lock);
508 	if (page->flags & SGX_EPC_PAGE_RECLAIMER_TRACKED) {
509 		/* The page is being reclaimed. */
510 		if (list_empty(&page->list)) {
511 			spin_unlock(&sgx_reclaimer_lock);
512 			return -EBUSY;
513 		}
514 
515 		list_del(&page->list);
516 		page->flags &= ~SGX_EPC_PAGE_RECLAIMER_TRACKED;
517 	}
518 	spin_unlock(&sgx_reclaimer_lock);
519 
520 	return 0;
521 }
522 
523 /**
524  * sgx_alloc_epc_page() - Allocate an EPC page
525  * @owner:	the owner of the EPC page
526  * @reclaim:	reclaim pages if necessary
527  *
528  * Iterate through EPC sections and borrow a free EPC page to the caller. When a
529  * page is no longer needed it must be released with sgx_free_epc_page(). If
530  * @reclaim is set to true, directly reclaim pages when we are out of pages. No
531  * mm's can be locked when @reclaim is set to true.
532  *
533  * Finally, wake up ksgxd when the number of pages goes below the watermark
534  * before returning back to the caller.
535  *
536  * Return:
537  *   an EPC page,
538  *   -errno on error
539  */
540 struct sgx_epc_page *sgx_alloc_epc_page(void *owner, bool reclaim)
541 {
542 	struct sgx_epc_page *page;
543 
544 	for ( ; ; ) {
545 		page = __sgx_alloc_epc_page();
546 		if (!IS_ERR(page)) {
547 			page->owner = owner;
548 			break;
549 		}
550 
551 		if (list_empty(&sgx_active_page_list))
552 			return ERR_PTR(-ENOMEM);
553 
554 		if (!reclaim) {
555 			page = ERR_PTR(-EBUSY);
556 			break;
557 		}
558 
559 		if (signal_pending(current)) {
560 			page = ERR_PTR(-ERESTARTSYS);
561 			break;
562 		}
563 
564 		sgx_reclaim_pages();
565 		cond_resched();
566 	}
567 
568 	if (sgx_should_reclaim(SGX_NR_LOW_PAGES))
569 		wake_up(&ksgxd_waitq);
570 
571 	return page;
572 }
573 
574 /**
575  * sgx_free_epc_page() - Free an EPC page
576  * @page:	an EPC page
577  *
578  * Put the EPC page back to the list of free pages. It's the caller's
579  * responsibility to make sure that the page is in uninitialized state. In other
580  * words, do EREMOVE, EWB or whatever operation is necessary before calling
581  * this function.
582  */
583 void sgx_free_epc_page(struct sgx_epc_page *page)
584 {
585 	struct sgx_epc_section *section = &sgx_epc_sections[page->section];
586 	struct sgx_numa_node *node = section->node;
587 
588 	spin_lock(&node->lock);
589 
590 	page->owner = NULL;
591 	if (page->poison)
592 		list_add(&page->list, &node->sgx_poison_page_list);
593 	else
594 		list_add_tail(&page->list, &node->free_page_list);
595 	page->flags = SGX_EPC_PAGE_IS_FREE;
596 
597 	spin_unlock(&node->lock);
598 	atomic_long_inc(&sgx_nr_free_pages);
599 }
600 
601 static bool __init sgx_setup_epc_section(u64 phys_addr, u64 size,
602 					 unsigned long index,
603 					 struct sgx_epc_section *section)
604 {
605 	unsigned long nr_pages = size >> PAGE_SHIFT;
606 	unsigned long i;
607 
608 	section->virt_addr = memremap(phys_addr, size, MEMREMAP_WB);
609 	if (!section->virt_addr)
610 		return false;
611 
612 	section->pages = vmalloc(nr_pages * sizeof(struct sgx_epc_page));
613 	if (!section->pages) {
614 		memunmap(section->virt_addr);
615 		return false;
616 	}
617 
618 	section->phys_addr = phys_addr;
619 	xa_store_range(&sgx_epc_address_space, section->phys_addr,
620 		       phys_addr + size - 1, section, GFP_KERNEL);
621 
622 	for (i = 0; i < nr_pages; i++) {
623 		section->pages[i].section = index;
624 		section->pages[i].flags = 0;
625 		section->pages[i].owner = NULL;
626 		section->pages[i].poison = 0;
627 		list_add_tail(&section->pages[i].list, &sgx_dirty_page_list);
628 	}
629 
630 	return true;
631 }
632 
633 bool arch_is_platform_page(u64 paddr)
634 {
635 	return !!xa_load(&sgx_epc_address_space, paddr);
636 }
637 EXPORT_SYMBOL_GPL(arch_is_platform_page);
638 
639 static struct sgx_epc_page *sgx_paddr_to_page(u64 paddr)
640 {
641 	struct sgx_epc_section *section;
642 
643 	section = xa_load(&sgx_epc_address_space, paddr);
644 	if (!section)
645 		return NULL;
646 
647 	return &section->pages[PFN_DOWN(paddr - section->phys_addr)];
648 }
649 
650 /*
651  * Called in process context to handle a hardware reported
652  * error in an SGX EPC page.
653  * If the MF_ACTION_REQUIRED bit is set in flags, then the
654  * context is the task that consumed the poison data. Otherwise
655  * this is called from a kernel thread unrelated to the page.
656  */
657 int arch_memory_failure(unsigned long pfn, int flags)
658 {
659 	struct sgx_epc_page *page = sgx_paddr_to_page(pfn << PAGE_SHIFT);
660 	struct sgx_epc_section *section;
661 	struct sgx_numa_node *node;
662 
663 	/*
664 	 * mm/memory-failure.c calls this routine for all errors
665 	 * where there isn't a "struct page" for the address. But that
666 	 * includes other address ranges besides SGX.
667 	 */
668 	if (!page)
669 		return -ENXIO;
670 
671 	/*
672 	 * If poison was consumed synchronously. Send a SIGBUS to
673 	 * the task. Hardware has already exited the SGX enclave and
674 	 * will not allow re-entry to an enclave that has a memory
675 	 * error. The signal may help the task understand why the
676 	 * enclave is broken.
677 	 */
678 	if (flags & MF_ACTION_REQUIRED)
679 		force_sig(SIGBUS);
680 
681 	section = &sgx_epc_sections[page->section];
682 	node = section->node;
683 
684 	spin_lock(&node->lock);
685 
686 	/* Already poisoned? Nothing more to do */
687 	if (page->poison)
688 		goto out;
689 
690 	page->poison = 1;
691 
692 	/*
693 	 * If the page is on a free list, move it to the per-node
694 	 * poison page list.
695 	 */
696 	if (page->flags & SGX_EPC_PAGE_IS_FREE) {
697 		list_move(&page->list, &node->sgx_poison_page_list);
698 		goto out;
699 	}
700 
701 	/*
702 	 * TBD: Add additional plumbing to enable pre-emptive
703 	 * action for asynchronous poison notification. Until
704 	 * then just hope that the poison:
705 	 * a) is not accessed - sgx_free_epc_page() will deal with it
706 	 *    when the user gives it back
707 	 * b) results in a recoverable machine check rather than
708 	 *    a fatal one
709 	 */
710 out:
711 	spin_unlock(&node->lock);
712 	return 0;
713 }
714 
715 /**
716  * A section metric is concatenated in a way that @low bits 12-31 define the
717  * bits 12-31 of the metric and @high bits 0-19 define the bits 32-51 of the
718  * metric.
719  */
720 static inline u64 __init sgx_calc_section_metric(u64 low, u64 high)
721 {
722 	return (low & GENMASK_ULL(31, 12)) +
723 	       ((high & GENMASK_ULL(19, 0)) << 32);
724 }
725 
726 #ifdef CONFIG_NUMA
727 static ssize_t sgx_total_bytes_show(struct device *dev, struct device_attribute *attr, char *buf)
728 {
729 	return sysfs_emit(buf, "%lu\n", sgx_numa_nodes[dev->id].size);
730 }
731 static DEVICE_ATTR_RO(sgx_total_bytes);
732 
733 static umode_t arch_node_attr_is_visible(struct kobject *kobj,
734 		struct attribute *attr, int idx)
735 {
736 	/* Make all x86/ attributes invisible when SGX is not initialized: */
737 	if (nodes_empty(sgx_numa_mask))
738 		return 0;
739 
740 	return attr->mode;
741 }
742 
743 static struct attribute *arch_node_dev_attrs[] = {
744 	&dev_attr_sgx_total_bytes.attr,
745 	NULL,
746 };
747 
748 const struct attribute_group arch_node_dev_group = {
749 	.name = "x86",
750 	.attrs = arch_node_dev_attrs,
751 	.is_visible = arch_node_attr_is_visible,
752 };
753 
754 static void __init arch_update_sysfs_visibility(int nid)
755 {
756 	struct node *node = node_devices[nid];
757 	int ret;
758 
759 	ret = sysfs_update_group(&node->dev.kobj, &arch_node_dev_group);
760 
761 	if (ret)
762 		pr_err("sysfs update failed (%d), files may be invisible", ret);
763 }
764 #else /* !CONFIG_NUMA */
765 static void __init arch_update_sysfs_visibility(int nid) {}
766 #endif
767 
768 static bool __init sgx_page_cache_init(void)
769 {
770 	u32 eax, ebx, ecx, edx, type;
771 	u64 pa, size;
772 	int nid;
773 	int i;
774 
775 	sgx_numa_nodes = kmalloc_array(num_possible_nodes(), sizeof(*sgx_numa_nodes), GFP_KERNEL);
776 	if (!sgx_numa_nodes)
777 		return false;
778 
779 	for (i = 0; i < ARRAY_SIZE(sgx_epc_sections); i++) {
780 		cpuid_count(SGX_CPUID, i + SGX_CPUID_EPC, &eax, &ebx, &ecx, &edx);
781 
782 		type = eax & SGX_CPUID_EPC_MASK;
783 		if (type == SGX_CPUID_EPC_INVALID)
784 			break;
785 
786 		if (type != SGX_CPUID_EPC_SECTION) {
787 			pr_err_once("Unknown EPC section type: %u\n", type);
788 			break;
789 		}
790 
791 		pa   = sgx_calc_section_metric(eax, ebx);
792 		size = sgx_calc_section_metric(ecx, edx);
793 
794 		pr_info("EPC section 0x%llx-0x%llx\n", pa, pa + size - 1);
795 
796 		if (!sgx_setup_epc_section(pa, size, i, &sgx_epc_sections[i])) {
797 			pr_err("No free memory for an EPC section\n");
798 			break;
799 		}
800 
801 		nid = numa_map_to_online_node(phys_to_target_node(pa));
802 		if (nid == NUMA_NO_NODE) {
803 			/* The physical address is already printed above. */
804 			pr_warn(FW_BUG "Unable to map EPC section to online node. Fallback to the NUMA node 0.\n");
805 			nid = 0;
806 		}
807 
808 		if (!node_isset(nid, sgx_numa_mask)) {
809 			spin_lock_init(&sgx_numa_nodes[nid].lock);
810 			INIT_LIST_HEAD(&sgx_numa_nodes[nid].free_page_list);
811 			INIT_LIST_HEAD(&sgx_numa_nodes[nid].sgx_poison_page_list);
812 			node_set(nid, sgx_numa_mask);
813 			sgx_numa_nodes[nid].size = 0;
814 
815 			/* Make SGX-specific node sysfs files visible: */
816 			arch_update_sysfs_visibility(nid);
817 		}
818 
819 		sgx_epc_sections[i].node =  &sgx_numa_nodes[nid];
820 		sgx_numa_nodes[nid].size += size;
821 
822 		sgx_nr_epc_sections++;
823 	}
824 
825 	if (!sgx_nr_epc_sections) {
826 		pr_err("There are zero EPC sections.\n");
827 		return false;
828 	}
829 
830 	return true;
831 }
832 
833 /*
834  * Update the SGX_LEPUBKEYHASH MSRs to the values specified by caller.
835  * Bare-metal driver requires to update them to hash of enclave's signer
836  * before EINIT. KVM needs to update them to guest's virtual MSR values
837  * before doing EINIT from guest.
838  */
839 void sgx_update_lepubkeyhash(u64 *lepubkeyhash)
840 {
841 	int i;
842 
843 	WARN_ON_ONCE(preemptible());
844 
845 	for (i = 0; i < 4; i++)
846 		wrmsrl(MSR_IA32_SGXLEPUBKEYHASH0 + i, lepubkeyhash[i]);
847 }
848 
849 const struct file_operations sgx_provision_fops = {
850 	.owner			= THIS_MODULE,
851 };
852 
853 static struct miscdevice sgx_dev_provision = {
854 	.minor = MISC_DYNAMIC_MINOR,
855 	.name = "sgx_provision",
856 	.nodename = "sgx_provision",
857 	.fops = &sgx_provision_fops,
858 };
859 
860 /**
861  * sgx_set_attribute() - Update allowed attributes given file descriptor
862  * @allowed_attributes:		Pointer to allowed enclave attributes
863  * @attribute_fd:		File descriptor for specific attribute
864  *
865  * Append enclave attribute indicated by file descriptor to allowed
866  * attributes. Currently only SGX_ATTR_PROVISIONKEY indicated by
867  * /dev/sgx_provision is supported.
868  *
869  * Return:
870  * -0:		SGX_ATTR_PROVISIONKEY is appended to allowed_attributes
871  * -EINVAL:	Invalid, or not supported file descriptor
872  */
873 int sgx_set_attribute(unsigned long *allowed_attributes,
874 		      unsigned int attribute_fd)
875 {
876 	struct file *file;
877 
878 	file = fget(attribute_fd);
879 	if (!file)
880 		return -EINVAL;
881 
882 	if (file->f_op != &sgx_provision_fops) {
883 		fput(file);
884 		return -EINVAL;
885 	}
886 
887 	*allowed_attributes |= SGX_ATTR_PROVISIONKEY;
888 
889 	fput(file);
890 	return 0;
891 }
892 EXPORT_SYMBOL_GPL(sgx_set_attribute);
893 
894 static int __init sgx_init(void)
895 {
896 	int ret;
897 	int i;
898 
899 	if (!cpu_feature_enabled(X86_FEATURE_SGX))
900 		return -ENODEV;
901 
902 	if (!sgx_page_cache_init())
903 		return -ENOMEM;
904 
905 	if (!sgx_page_reclaimer_init()) {
906 		ret = -ENOMEM;
907 		goto err_page_cache;
908 	}
909 
910 	ret = misc_register(&sgx_dev_provision);
911 	if (ret)
912 		goto err_kthread;
913 
914 	/*
915 	 * Always try to initialize the native *and* KVM drivers.
916 	 * The KVM driver is less picky than the native one and
917 	 * can function if the native one is not supported on the
918 	 * current system or fails to initialize.
919 	 *
920 	 * Error out only if both fail to initialize.
921 	 */
922 	ret = sgx_drv_init();
923 
924 	if (sgx_vepc_init() && ret)
925 		goto err_provision;
926 
927 	return 0;
928 
929 err_provision:
930 	misc_deregister(&sgx_dev_provision);
931 
932 err_kthread:
933 	kthread_stop(ksgxd_tsk);
934 
935 err_page_cache:
936 	for (i = 0; i < sgx_nr_epc_sections; i++) {
937 		vfree(sgx_epc_sections[i].pages);
938 		memunmap(sgx_epc_sections[i].virt_addr);
939 	}
940 
941 	return ret;
942 }
943 
944 device_initcall(sgx_init);
945