Lines Matching full:encl

11 #include "encl.h"
15 static int sgx_encl_lookup_backing(struct sgx_encl *encl, unsigned long page_index,
29 * @encl: Enclave to which PCMD page belongs
58 static int reclaimer_writing_to_pcmd(struct sgx_encl *encl, in reclaimer_writing_to_pcmd() argument
83 if (addr == encl->base + encl->size) in reclaimer_writing_to_pcmd()
86 entry = xa_load(&encl->page_array, PFN_DOWN(addr)); in reclaimer_writing_to_pcmd()
110 static inline pgoff_t sgx_encl_get_backing_page_pcmd_offset(struct sgx_encl *encl, in sgx_encl_get_backing_page_pcmd_offset() argument
113 pgoff_t epc_end_off = encl->size + sizeof(struct sgx_secs); in sgx_encl_get_backing_page_pcmd_offset()
121 static inline void sgx_encl_truncate_backing_page(struct sgx_encl *encl, unsigned long page_index) in sgx_encl_truncate_backing_page() argument
123 struct inode *inode = file_inode(encl->backing); in sgx_encl_truncate_backing_page()
137 struct sgx_encl *encl = encl_page->encl; in __sgx_encl_eldu() local
147 page_index = PFN_DOWN(encl_page->desc - encl_page->encl->base); in __sgx_encl_eldu()
149 page_index = PFN_DOWN(encl->size); in __sgx_encl_eldu()
154 pcmd_first_page = PFN_PHYS(page_index & ~PCMD_FIRST_MASK) + encl->base; in __sgx_encl_eldu()
156 page_pcmd_off = sgx_encl_get_backing_page_pcmd_offset(encl, page_index); in __sgx_encl_eldu()
158 ret = sgx_encl_lookup_backing(encl, page_index, &b); in __sgx_encl_eldu()
196 sgx_encl_truncate_backing_page(encl, page_index); in __sgx_encl_eldu()
198 if (pcmd_page_empty && !reclaimer_writing_to_pcmd(encl, pcmd_first_page)) { in __sgx_encl_eldu()
199 sgx_encl_truncate_backing_page(encl, PFN_DOWN(page_pcmd_off)); in __sgx_encl_eldu()
216 struct sgx_encl *encl = encl_page->encl; in sgx_encl_eldu() local
231 list_move(&encl_page->va_page->list, &encl->va_pages); in sgx_encl_eldu()
239 * Ensure the SECS page is not swapped out. Must be called with encl->lock
243 static struct sgx_epc_page *sgx_encl_load_secs(struct sgx_encl *encl) in sgx_encl_load_secs() argument
245 struct sgx_epc_page *epc_page = encl->secs.epc_page; in sgx_encl_load_secs()
248 epc_page = sgx_encl_eldu(&encl->secs, NULL); in sgx_encl_load_secs()
253 static struct sgx_encl_page *__sgx_encl_load_page(struct sgx_encl *encl, in __sgx_encl_load_page() argument
266 epc_page = sgx_encl_load_secs(encl); in __sgx_encl_load_page()
270 epc_page = sgx_encl_eldu(entry, encl->secs.epc_page); in __sgx_encl_load_page()
274 encl->secs_child_cnt++; in __sgx_encl_load_page()
280 static struct sgx_encl_page *sgx_encl_load_page_in_vma(struct sgx_encl *encl, in sgx_encl_load_page_in_vma() argument
287 entry = xa_load(&encl->page_array, PFN_DOWN(addr)); in sgx_encl_load_page_in_vma()
299 return __sgx_encl_load_page(encl, entry); in sgx_encl_load_page_in_vma()
302 struct sgx_encl_page *sgx_encl_load_page(struct sgx_encl *encl, in sgx_encl_load_page() argument
307 entry = xa_load(&encl->page_array, PFN_DOWN(addr)); in sgx_encl_load_page()
311 return __sgx_encl_load_page(encl, entry); in sgx_encl_load_page()
317 * @encl: enclave accessing the page
328 struct sgx_encl *encl, unsigned long addr) in sgx_encl_eaug_page() argument
339 if (!test_bit(SGX_ENCL_INITIALIZED, &encl->flags)) in sgx_encl_eaug_page()
349 encl_page = sgx_encl_page_alloc(encl, addr - encl->base, secinfo_flags); in sgx_encl_eaug_page()
353 mutex_lock(&encl->lock); in sgx_encl_eaug_page()
355 epc_page = sgx_encl_load_secs(encl); in sgx_encl_eaug_page()
369 va_page = sgx_encl_grow(encl, false); in sgx_encl_eaug_page()
377 list_add(&va_page->list, &encl->va_pages); in sgx_encl_eaug_page()
379 ret = xa_insert(&encl->page_array, PFN_DOWN(encl_page->desc), in sgx_encl_eaug_page()
383 * running without encl->lock in sgx_encl_eaug_page()
388 pginfo.secs = (unsigned long)sgx_get_epc_virt_addr(encl->secs.epc_page); in sgx_encl_eaug_page()
396 encl_page->encl = encl; in sgx_encl_eaug_page()
399 encl->secs_child_cnt++; in sgx_encl_eaug_page()
410 mutex_unlock(&encl->lock); in sgx_encl_eaug_page()
413 mutex_unlock(&encl->lock); in sgx_encl_eaug_page()
417 xa_erase(&encl->page_array, PFN_DOWN(encl_page->desc)); in sgx_encl_eaug_page()
420 sgx_encl_shrink(encl, va_page); in sgx_encl_eaug_page()
424 mutex_unlock(&encl->lock); in sgx_encl_eaug_page()
436 struct sgx_encl *encl; in sgx_vma_fault() local
439 encl = vma->vm_private_data; in sgx_vma_fault()
446 if (unlikely(!encl)) in sgx_vma_fault()
457 (!xa_load(&encl->page_array, PFN_DOWN(addr)))) in sgx_vma_fault()
458 return sgx_encl_eaug_page(vma, encl, addr); in sgx_vma_fault()
460 mutex_lock(&encl->lock); in sgx_vma_fault()
462 entry = sgx_encl_load_page_in_vma(encl, addr, vma->vm_flags); in sgx_vma_fault()
464 mutex_unlock(&encl->lock); in sgx_vma_fault()
476 mutex_unlock(&encl->lock); in sgx_vma_fault()
482 mutex_unlock(&encl->lock); in sgx_vma_fault()
489 struct sgx_encl *encl = vma->vm_private_data; in sgx_vma_open() local
496 if (unlikely(!encl)) in sgx_vma_open()
499 if (sgx_encl_mm_add(encl, vma->vm_mm)) in sgx_vma_open()
506 * @encl: an enclave pointer
522 int sgx_encl_may_map(struct sgx_encl *encl, unsigned long start, in sgx_encl_may_map() argument
530 XA_STATE(xas, &encl->page_array, PFN_DOWN(start)); in sgx_encl_may_map()
533 if (test_bit(SGX_ENCL_INITIALIZED, &encl->flags) && in sgx_encl_may_map()
534 (start < encl->base || end > encl->base + encl->size)) in sgx_encl_may_map()
544 mutex_lock(&encl->lock); in sgx_encl_may_map()
556 mutex_unlock(&encl->lock); in sgx_encl_may_map()
560 mutex_lock(&encl->lock); in sgx_encl_may_map()
565 mutex_unlock(&encl->lock); in sgx_encl_may_map()
576 static int sgx_encl_debug_read(struct sgx_encl *encl, struct sgx_encl_page *page, in sgx_encl_debug_read() argument
590 static int sgx_encl_debug_write(struct sgx_encl *encl, struct sgx_encl_page *page, in sgx_encl_debug_write() argument
604 * Load an enclave page to EPC if required, and take encl->lock.
606 static struct sgx_encl_page *sgx_encl_reserve_page(struct sgx_encl *encl, in sgx_encl_reserve_page() argument
613 mutex_lock(&encl->lock); in sgx_encl_reserve_page()
615 entry = sgx_encl_load_page_in_vma(encl, addr, vm_flags); in sgx_encl_reserve_page()
619 mutex_unlock(&encl->lock); in sgx_encl_reserve_page()
623 mutex_unlock(&encl->lock); in sgx_encl_reserve_page()
631 struct sgx_encl *encl = vma->vm_private_data; in sgx_vma_access() local
644 if (!encl) in sgx_vma_access()
647 if (!test_bit(SGX_ENCL_DEBUG, &encl->flags)) in sgx_vma_access()
651 entry = sgx_encl_reserve_page(encl, (addr + i) & PAGE_MASK, in sgx_vma_access()
663 ret = sgx_encl_debug_read(encl, entry, align, data); in sgx_vma_access()
669 ret = sgx_encl_debug_write(encl, entry, align, data); in sgx_vma_access()
677 mutex_unlock(&encl->lock); in sgx_vma_access()
702 struct sgx_encl *encl = container_of(ref, struct sgx_encl, refcount); in sgx_encl_release() local
703 unsigned long max_page_index = PFN_DOWN(encl->base + encl->size - 1); in sgx_encl_release()
708 XA_STATE(xas, &encl->page_array, PFN_DOWN(encl->base)); in sgx_encl_release()
721 encl->secs_child_cnt--; in sgx_encl_release()
741 xa_destroy(&encl->page_array); in sgx_encl_release()
743 if (!encl->secs_child_cnt && encl->secs.epc_page) { in sgx_encl_release()
744 sgx_encl_free_epc_page(encl->secs.epc_page); in sgx_encl_release()
745 encl->secs.epc_page = NULL; in sgx_encl_release()
748 while (!list_empty(&encl->va_pages)) { in sgx_encl_release()
749 va_page = list_first_entry(&encl->va_pages, struct sgx_va_page, in sgx_encl_release()
756 if (encl->backing) in sgx_encl_release()
757 fput(encl->backing); in sgx_encl_release()
759 cleanup_srcu_struct(&encl->srcu); in sgx_encl_release()
761 WARN_ON_ONCE(!list_empty(&encl->mm_list)); in sgx_encl_release()
764 WARN_ON_ONCE(encl->secs_child_cnt); in sgx_encl_release()
765 WARN_ON_ONCE(encl->secs.epc_page); in sgx_encl_release()
767 kfree(encl); in sgx_encl_release()
784 spin_lock(&encl_mm->encl->mm_lock); in sgx_mmu_notifier_release()
785 list_for_each_entry(tmp, &encl_mm->encl->mm_list, list) { in sgx_mmu_notifier_release()
792 spin_unlock(&encl_mm->encl->mm_lock); in sgx_mmu_notifier_release()
795 synchronize_srcu(&encl_mm->encl->srcu); in sgx_mmu_notifier_release()
804 /* 'encl_mm' is going away, put encl_mm->encl reference: */ in sgx_mmu_notifier_free()
805 kref_put(&encl_mm->encl->refcount, sgx_encl_release); in sgx_mmu_notifier_free()
815 static struct sgx_encl_mm *sgx_encl_find_mm(struct sgx_encl *encl, in sgx_encl_find_mm() argument
822 idx = srcu_read_lock(&encl->srcu); in sgx_encl_find_mm()
824 list_for_each_entry_rcu(tmp, &encl->mm_list, list) { in sgx_encl_find_mm()
831 srcu_read_unlock(&encl->srcu, idx); in sgx_encl_find_mm()
836 int sgx_encl_mm_add(struct sgx_encl *encl, struct mm_struct *mm) in sgx_encl_mm_add() argument
843 * each 'mm' only appears once on encl->mm_list. This is guaranteed by in sgx_encl_mm_add()
845 * remove to an encl->mm_list. in sgx_encl_mm_add()
853 if (sgx_encl_find_mm(encl, mm)) in sgx_encl_mm_add()
860 /* Grab a refcount for the encl_mm->encl reference: */ in sgx_encl_mm_add()
861 kref_get(&encl->refcount); in sgx_encl_mm_add()
862 encl_mm->encl = encl; in sgx_encl_mm_add()
872 spin_lock(&encl->mm_lock); in sgx_encl_mm_add()
873 list_add_rcu(&encl_mm->list, &encl->mm_list); in sgx_encl_mm_add()
876 encl->mm_list_version++; in sgx_encl_mm_add()
877 spin_unlock(&encl->mm_lock); in sgx_encl_mm_add()
884 * @encl: the enclave
923 * Return: cpumask of CPUs that might be accessing @encl
925 const cpumask_t *sgx_encl_cpumask(struct sgx_encl *encl) in sgx_encl_cpumask() argument
927 cpumask_t *cpumask = &encl->cpumask; in sgx_encl_cpumask()
933 idx = srcu_read_lock(&encl->srcu); in sgx_encl_cpumask()
935 list_for_each_entry_rcu(encl_mm, &encl->mm_list, list) { in sgx_encl_cpumask()
944 srcu_read_unlock(&encl->srcu, idx); in sgx_encl_cpumask()
949 static struct page *sgx_encl_get_backing_page(struct sgx_encl *encl, in sgx_encl_get_backing_page() argument
952 struct address_space *mapping = encl->backing->f_mapping; in sgx_encl_get_backing_page()
960 * @encl: an enclave pointer
971 static int __sgx_encl_get_backing(struct sgx_encl *encl, unsigned long page_index, in __sgx_encl_get_backing() argument
974 pgoff_t page_pcmd_off = sgx_encl_get_backing_page_pcmd_offset(encl, page_index); in __sgx_encl_get_backing()
978 contents = sgx_encl_get_backing_page(encl, page_index); in __sgx_encl_get_backing()
982 pcmd = sgx_encl_get_backing_page(encl, PFN_DOWN(page_pcmd_off)); in __sgx_encl_get_backing()
1000 static struct mem_cgroup *sgx_encl_get_mem_cgroup(struct sgx_encl *encl) in sgx_encl_get_mem_cgroup() argument
1018 idx = srcu_read_lock(&encl->srcu); in sgx_encl_get_mem_cgroup()
1020 list_for_each_entry_rcu(encl_mm, &encl->mm_list, list) { in sgx_encl_get_mem_cgroup()
1031 srcu_read_unlock(&encl->srcu, idx); in sgx_encl_get_mem_cgroup()
1047 * @encl: an enclave pointer
1062 int sgx_encl_alloc_backing(struct sgx_encl *encl, unsigned long page_index, in sgx_encl_alloc_backing() argument
1065 struct mem_cgroup *encl_memcg = sgx_encl_get_mem_cgroup(encl); in sgx_encl_alloc_backing()
1069 ret = __sgx_encl_get_backing(encl, page_index, backing); in sgx_encl_alloc_backing()
1079 * @encl: an enclave pointer
1094 static int sgx_encl_lookup_backing(struct sgx_encl *encl, unsigned long page_index, in sgx_encl_lookup_backing() argument
1097 return __sgx_encl_get_backing(encl, page_index, backing); in sgx_encl_lookup_backing()
1139 struct sgx_encl *encl = page->encl; in sgx_encl_test_and_clear_young() local
1147 if (encl != vma->vm_private_data) in sgx_encl_test_and_clear_young()
1158 struct sgx_encl_page *sgx_encl_page_alloc(struct sgx_encl *encl, in sgx_encl_page_alloc() argument
1169 encl_page->desc = encl->base + offset; in sgx_encl_page_alloc()
1170 encl_page->encl = encl; in sgx_encl_page_alloc()
1192 * @encl: the enclave
1199 void sgx_zap_enclave_ptes(struct sgx_encl *encl, unsigned long addr) in sgx_zap_enclave_ptes() argument
1207 mm_list_version = encl->mm_list_version; in sgx_zap_enclave_ptes()
1212 idx = srcu_read_lock(&encl->srcu); in sgx_zap_enclave_ptes()
1214 list_for_each_entry_rcu(encl_mm, &encl->mm_list, list) { in sgx_zap_enclave_ptes()
1221 if (!ret && encl == vma->vm_private_data) in sgx_zap_enclave_ptes()
1229 srcu_read_unlock(&encl->srcu, idx); in sgx_zap_enclave_ptes()
1230 } while (unlikely(encl->mm_list_version != mm_list_version)); in sgx_zap_enclave_ptes()