1caab277bSThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only
29031fefdSWill Deacon /*
30d747f65SVincenzo Frascino * VDSO implementations.
49031fefdSWill Deacon *
59031fefdSWill Deacon * Copyright (C) 2012 ARM Limited
69031fefdSWill Deacon *
79031fefdSWill Deacon * Author: Will Deacon <will.deacon@arm.com>
89031fefdSWill Deacon */
99031fefdSWill Deacon
105a9e3e15SJisheng Zhang #include <linux/cache.h>
119031fefdSWill Deacon #include <linux/clocksource.h>
129031fefdSWill Deacon #include <linux/elf.h>
139031fefdSWill Deacon #include <linux/err.h>
149031fefdSWill Deacon #include <linux/errno.h>
159031fefdSWill Deacon #include <linux/gfp.h>
165a9e3e15SJisheng Zhang #include <linux/kernel.h>
179031fefdSWill Deacon #include <linux/mm.h>
189031fefdSWill Deacon #include <linux/sched.h>
199031fefdSWill Deacon #include <linux/signal.h>
209031fefdSWill Deacon #include <linux/slab.h>
21ee3cda8eSAndrei Vagin #include <linux/time_namespace.h>
22c60b0c28SCatalin Marinas #include <linux/timekeeper_internal.h>
239031fefdSWill Deacon #include <linux/vmalloc.h>
2428b1a824SVincenzo Frascino #include <vdso/datapage.h>
2528b1a824SVincenzo Frascino #include <vdso/helpers.h>
2628b1a824SVincenzo Frascino #include <vdso/vsyscall.h>
279031fefdSWill Deacon
289031fefdSWill Deacon #include <asm/cacheflush.h>
299031fefdSWill Deacon #include <asm/signal32.h>
309031fefdSWill Deacon #include <asm/vdso.h>
319031fefdSWill Deacon
32d3418f38SMark Rutland enum vdso_abi {
33d3418f38SMark Rutland VDSO_ABI_AA64,
34d3418f38SMark Rutland VDSO_ABI_AA32,
35c7aa2d71SVincenzo Frascino };
36c7aa2d71SVincenzo Frascino
373503d56cSAndrei Vagin enum vvar_pages {
383503d56cSAndrei Vagin VVAR_DATA_PAGE_OFFSET,
393503d56cSAndrei Vagin VVAR_TIMENS_PAGE_OFFSET,
403503d56cSAndrei Vagin VVAR_NR_PAGES,
413503d56cSAndrei Vagin };
423503d56cSAndrei Vagin
43d3418f38SMark Rutland struct vdso_abi_info {
44c7aa2d71SVincenzo Frascino const char *name;
45c7aa2d71SVincenzo Frascino const char *vdso_code_start;
46c7aa2d71SVincenzo Frascino const char *vdso_code_end;
47c7aa2d71SVincenzo Frascino unsigned long vdso_pages;
48c7aa2d71SVincenzo Frascino /* Data Mapping */
49c7aa2d71SVincenzo Frascino struct vm_special_mapping *dm;
50c7aa2d71SVincenzo Frascino /* Code Mapping */
51c7aa2d71SVincenzo Frascino struct vm_special_mapping *cm;
52c7aa2d71SVincenzo Frascino };
53c7aa2d71SVincenzo Frascino
54d3418f38SMark Rutland static struct vdso_abi_info vdso_info[] __ro_after_init = {
55d3418f38SMark Rutland [VDSO_ABI_AA64] = {
56c7aa2d71SVincenzo Frascino .name = "vdso",
57c7aa2d71SVincenzo Frascino .vdso_code_start = vdso_start,
58c7aa2d71SVincenzo Frascino .vdso_code_end = vdso_end,
59c7aa2d71SVincenzo Frascino },
607c1deeebSVincenzo Frascino #ifdef CONFIG_COMPAT_VDSO
61d3418f38SMark Rutland [VDSO_ABI_AA32] = {
627c1deeebSVincenzo Frascino .name = "vdso32",
637c1deeebSVincenzo Frascino .vdso_code_start = vdso32_start,
647c1deeebSVincenzo Frascino .vdso_code_end = vdso32_end,
657c1deeebSVincenzo Frascino },
667c1deeebSVincenzo Frascino #endif /* CONFIG_COMPAT_VDSO */
67c7aa2d71SVincenzo Frascino };
689031fefdSWill Deacon
699031fefdSWill Deacon /*
709031fefdSWill Deacon * The vDSO data page.
719031fefdSWill Deacon */
729031fefdSWill Deacon static union {
7328b1a824SVincenzo Frascino struct vdso_data data[CS_BASES];
749031fefdSWill Deacon u8 page[PAGE_SIZE];
759031fefdSWill Deacon } vdso_data_store __page_aligned_data;
7628b1a824SVincenzo Frascino struct vdso_data *vdso_data = vdso_data_store.data;
779031fefdSWill Deacon
vdso_mremap(const struct vm_special_mapping * sm,struct vm_area_struct * new_vma)78871402e0SDmitry Safonov static int vdso_mremap(const struct vm_special_mapping *sm,
79c7aa2d71SVincenzo Frascino struct vm_area_struct *new_vma)
80c7aa2d71SVincenzo Frascino {
81c7aa2d71SVincenzo Frascino current->mm->context.vdso = (void *)new_vma->vm_start;
82c7aa2d71SVincenzo Frascino
83c7aa2d71SVincenzo Frascino return 0;
84c7aa2d71SVincenzo Frascino }
85c7aa2d71SVincenzo Frascino
__vdso_init(enum vdso_abi abi)86a7dcf58aSJisheng Zhang static int __init __vdso_init(enum vdso_abi abi)
87c7aa2d71SVincenzo Frascino {
88c7aa2d71SVincenzo Frascino int i;
89c7aa2d71SVincenzo Frascino struct page **vdso_pagelist;
90c7aa2d71SVincenzo Frascino unsigned long pfn;
91c7aa2d71SVincenzo Frascino
92d3418f38SMark Rutland if (memcmp(vdso_info[abi].vdso_code_start, "\177ELF", 4)) {
93c7aa2d71SVincenzo Frascino pr_err("vDSO is not a valid ELF object!\n");
94c7aa2d71SVincenzo Frascino return -EINVAL;
95c7aa2d71SVincenzo Frascino }
96c7aa2d71SVincenzo Frascino
97d3418f38SMark Rutland vdso_info[abi].vdso_pages = (
98d3418f38SMark Rutland vdso_info[abi].vdso_code_end -
99d3418f38SMark Rutland vdso_info[abi].vdso_code_start) >>
100c7aa2d71SVincenzo Frascino PAGE_SHIFT;
101c7aa2d71SVincenzo Frascino
102d53b5c01SAndrei Vagin vdso_pagelist = kcalloc(vdso_info[abi].vdso_pages,
103c7aa2d71SVincenzo Frascino sizeof(struct page *),
104c7aa2d71SVincenzo Frascino GFP_KERNEL);
105c7aa2d71SVincenzo Frascino if (vdso_pagelist == NULL)
106c7aa2d71SVincenzo Frascino return -ENOMEM;
107c7aa2d71SVincenzo Frascino
108c7aa2d71SVincenzo Frascino /* Grab the vDSO code pages. */
109d3418f38SMark Rutland pfn = sym_to_pfn(vdso_info[abi].vdso_code_start);
110c7aa2d71SVincenzo Frascino
111d3418f38SMark Rutland for (i = 0; i < vdso_info[abi].vdso_pages; i++)
112d53b5c01SAndrei Vagin vdso_pagelist[i] = pfn_to_page(pfn + i);
113c7aa2d71SVincenzo Frascino
114d53b5c01SAndrei Vagin vdso_info[abi].cm->pages = vdso_pagelist;
115c7aa2d71SVincenzo Frascino
116c7aa2d71SVincenzo Frascino return 0;
117c7aa2d71SVincenzo Frascino }
118c7aa2d71SVincenzo Frascino
1191b6867d2SAndrei Vagin #ifdef CONFIG_TIME_NS
arch_get_vdso_data(void * vvar_page)1203503d56cSAndrei Vagin struct vdso_data *arch_get_vdso_data(void *vvar_page)
1213503d56cSAndrei Vagin {
1223503d56cSAndrei Vagin return (struct vdso_data *)(vvar_page);
1233503d56cSAndrei Vagin }
1243503d56cSAndrei Vagin
1251b6867d2SAndrei Vagin /*
1261b6867d2SAndrei Vagin * The vvar mapping contains data for a specific time namespace, so when a task
1271b6867d2SAndrei Vagin * changes namespace we must unmap its vvar data for the old namespace.
1281b6867d2SAndrei Vagin * Subsequent faults will map in data for the new namespace.
1291b6867d2SAndrei Vagin *
1301b6867d2SAndrei Vagin * For more details see timens_setup_vdso_data().
1311b6867d2SAndrei Vagin */
vdso_join_timens(struct task_struct * task,struct time_namespace * ns)1321b6867d2SAndrei Vagin int vdso_join_timens(struct task_struct *task, struct time_namespace *ns)
1331b6867d2SAndrei Vagin {
1341b6867d2SAndrei Vagin struct mm_struct *mm = task->mm;
1351b6867d2SAndrei Vagin struct vm_area_struct *vma;
136de2b84d2SMatthew Wilcox (Oracle) VMA_ITERATOR(vmi, mm, 0);
1371b6867d2SAndrei Vagin
1381b6867d2SAndrei Vagin mmap_read_lock(mm);
1391b6867d2SAndrei Vagin
140de2b84d2SMatthew Wilcox (Oracle) for_each_vma(vmi, vma) {
1411b6867d2SAndrei Vagin if (vma_is_special_mapping(vma, vdso_info[VDSO_ABI_AA64].dm))
142e9adcfecSMike Kravetz zap_vma_pages(vma);
1431b6867d2SAndrei Vagin #ifdef CONFIG_COMPAT_VDSO
1441b6867d2SAndrei Vagin if (vma_is_special_mapping(vma, vdso_info[VDSO_ABI_AA32].dm))
145e9adcfecSMike Kravetz zap_vma_pages(vma);
1461b6867d2SAndrei Vagin #endif
1471b6867d2SAndrei Vagin }
1481b6867d2SAndrei Vagin
1491b6867d2SAndrei Vagin mmap_read_unlock(mm);
1501b6867d2SAndrei Vagin return 0;
1511b6867d2SAndrei Vagin }
1521b6867d2SAndrei Vagin #endif
1531b6867d2SAndrei Vagin
vvar_fault(const struct vm_special_mapping * sm,struct vm_area_struct * vma,struct vm_fault * vmf)154d53b5c01SAndrei Vagin static vm_fault_t vvar_fault(const struct vm_special_mapping *sm,
155d53b5c01SAndrei Vagin struct vm_area_struct *vma, struct vm_fault *vmf)
156d53b5c01SAndrei Vagin {
157ee3cda8eSAndrei Vagin struct page *timens_page = find_timens_vvar_page(vma);
158ee3cda8eSAndrei Vagin unsigned long pfn;
159ee3cda8eSAndrei Vagin
160ee3cda8eSAndrei Vagin switch (vmf->pgoff) {
161ee3cda8eSAndrei Vagin case VVAR_DATA_PAGE_OFFSET:
162ee3cda8eSAndrei Vagin if (timens_page)
163ee3cda8eSAndrei Vagin pfn = page_to_pfn(timens_page);
164ee3cda8eSAndrei Vagin else
165ee3cda8eSAndrei Vagin pfn = sym_to_pfn(vdso_data);
166ee3cda8eSAndrei Vagin break;
167ee3cda8eSAndrei Vagin #ifdef CONFIG_TIME_NS
168ee3cda8eSAndrei Vagin case VVAR_TIMENS_PAGE_OFFSET:
169ee3cda8eSAndrei Vagin /*
170ee3cda8eSAndrei Vagin * If a task belongs to a time namespace then a namespace
171ee3cda8eSAndrei Vagin * specific VVAR is mapped with the VVAR_DATA_PAGE_OFFSET and
172ee3cda8eSAndrei Vagin * the real VVAR page is mapped with the VVAR_TIMENS_PAGE_OFFSET
173ee3cda8eSAndrei Vagin * offset.
174ee3cda8eSAndrei Vagin * See also the comment near timens_setup_vdso_data().
175ee3cda8eSAndrei Vagin */
176ee3cda8eSAndrei Vagin if (!timens_page)
177d53b5c01SAndrei Vagin return VM_FAULT_SIGBUS;
178ee3cda8eSAndrei Vagin pfn = sym_to_pfn(vdso_data);
179ee3cda8eSAndrei Vagin break;
180ee3cda8eSAndrei Vagin #endif /* CONFIG_TIME_NS */
181ee3cda8eSAndrei Vagin default:
182ee3cda8eSAndrei Vagin return VM_FAULT_SIGBUS;
183ee3cda8eSAndrei Vagin }
184ee3cda8eSAndrei Vagin
185ee3cda8eSAndrei Vagin return vmf_insert_pfn(vma, vmf->address, pfn);
186d53b5c01SAndrei Vagin }
187d53b5c01SAndrei Vagin
__setup_additional_pages(enum vdso_abi abi,struct mm_struct * mm,struct linux_binprm * bprm,int uses_interp)188d3418f38SMark Rutland static int __setup_additional_pages(enum vdso_abi abi,
189c7aa2d71SVincenzo Frascino struct mm_struct *mm,
190c7aa2d71SVincenzo Frascino struct linux_binprm *bprm,
191c7aa2d71SVincenzo Frascino int uses_interp)
192c7aa2d71SVincenzo Frascino {
193c7aa2d71SVincenzo Frascino unsigned long vdso_base, vdso_text_len, vdso_mapping_len;
194bf740a90SMark Brown unsigned long gp_flags = 0;
195c7aa2d71SVincenzo Frascino void *ret;
196c7aa2d71SVincenzo Frascino
1973503d56cSAndrei Vagin BUILD_BUG_ON(VVAR_NR_PAGES != __VVAR_PAGES);
1983503d56cSAndrei Vagin
199d3418f38SMark Rutland vdso_text_len = vdso_info[abi].vdso_pages << PAGE_SHIFT;
200c7aa2d71SVincenzo Frascino /* Be sure to map the data page */
2013503d56cSAndrei Vagin vdso_mapping_len = vdso_text_len + VVAR_NR_PAGES * PAGE_SIZE;
202c7aa2d71SVincenzo Frascino
203c7aa2d71SVincenzo Frascino vdso_base = get_unmapped_area(NULL, 0, vdso_mapping_len, 0, 0);
204c7aa2d71SVincenzo Frascino if (IS_ERR_VALUE(vdso_base)) {
205c7aa2d71SVincenzo Frascino ret = ERR_PTR(vdso_base);
206c7aa2d71SVincenzo Frascino goto up_fail;
207c7aa2d71SVincenzo Frascino }
208c7aa2d71SVincenzo Frascino
2093503d56cSAndrei Vagin ret = _install_special_mapping(mm, vdso_base, VVAR_NR_PAGES * PAGE_SIZE,
210d53b5c01SAndrei Vagin VM_READ|VM_MAYREAD|VM_PFNMAP,
211d3418f38SMark Rutland vdso_info[abi].dm);
212c7aa2d71SVincenzo Frascino if (IS_ERR(ret))
213c7aa2d71SVincenzo Frascino goto up_fail;
214c7aa2d71SVincenzo Frascino
215bf740a90SMark Brown if (IS_ENABLED(CONFIG_ARM64_BTI_KERNEL) && system_supports_bti())
216bf740a90SMark Brown gp_flags = VM_ARM64_BTI;
217bf740a90SMark Brown
2183503d56cSAndrei Vagin vdso_base += VVAR_NR_PAGES * PAGE_SIZE;
219c7aa2d71SVincenzo Frascino mm->context.vdso = (void *)vdso_base;
220c7aa2d71SVincenzo Frascino ret = _install_special_mapping(mm, vdso_base, vdso_text_len,
221bf740a90SMark Brown VM_READ|VM_EXEC|gp_flags|
222c7aa2d71SVincenzo Frascino VM_MAYREAD|VM_MAYWRITE|VM_MAYEXEC,
223d3418f38SMark Rutland vdso_info[abi].cm);
224c7aa2d71SVincenzo Frascino if (IS_ERR(ret))
225c7aa2d71SVincenzo Frascino goto up_fail;
226c7aa2d71SVincenzo Frascino
227c7aa2d71SVincenzo Frascino return 0;
228c7aa2d71SVincenzo Frascino
229c7aa2d71SVincenzo Frascino up_fail:
230c7aa2d71SVincenzo Frascino mm->context.vdso = NULL;
231c7aa2d71SVincenzo Frascino return PTR_ERR(ret);
232c7aa2d71SVincenzo Frascino }
233c7aa2d71SVincenzo Frascino
2349031fefdSWill Deacon #ifdef CONFIG_COMPAT
2359031fefdSWill Deacon /*
2369031fefdSWill Deacon * Create and map the vectors page for AArch32 tasks.
2379031fefdSWill Deacon */
2381d09094aSMark Rutland enum aarch32_map {
2391d09094aSMark Rutland AA32_MAP_VECTORS, /* kuser helpers */
2400cbc2659SWill Deacon AA32_MAP_SIGPAGE,
2411d09094aSMark Rutland AA32_MAP_VVAR,
2421d09094aSMark Rutland AA32_MAP_VDSO,
2431d09094aSMark Rutland };
24474fc72e7SMark Rutland
24574fc72e7SMark Rutland static struct page *aarch32_vectors_page __ro_after_init;
24674fc72e7SMark Rutland static struct page *aarch32_sig_page __ro_after_init;
24774fc72e7SMark Rutland
aarch32_sigpage_mremap(const struct vm_special_mapping * sm,struct vm_area_struct * new_vma)2487adbf10eSWill Deacon static int aarch32_sigpage_mremap(const struct vm_special_mapping *sm,
2497adbf10eSWill Deacon struct vm_area_struct *new_vma)
2507adbf10eSWill Deacon {
2517adbf10eSWill Deacon current->mm->context.sigpage = (void *)new_vma->vm_start;
2527adbf10eSWill Deacon
2537adbf10eSWill Deacon return 0;
2547adbf10eSWill Deacon }
2557adbf10eSWill Deacon
2561d09094aSMark Rutland static struct vm_special_mapping aarch32_vdso_maps[] = {
2571d09094aSMark Rutland [AA32_MAP_VECTORS] = {
2580d747f65SVincenzo Frascino .name = "[vectors]", /* ABI */
25974fc72e7SMark Rutland .pages = &aarch32_vectors_page,
2600d747f65SVincenzo Frascino },
2610cbc2659SWill Deacon [AA32_MAP_SIGPAGE] = {
2620cbc2659SWill Deacon .name = "[sigpage]", /* ABI */
2630cbc2659SWill Deacon .pages = &aarch32_sig_page,
2647adbf10eSWill Deacon .mremap = aarch32_sigpage_mremap,
2650cbc2659SWill Deacon },
2661d09094aSMark Rutland [AA32_MAP_VVAR] = {
2677c1deeebSVincenzo Frascino .name = "[vvar]",
268d53b5c01SAndrei Vagin .fault = vvar_fault,
2697c1deeebSVincenzo Frascino },
2701d09094aSMark Rutland [AA32_MAP_VDSO] = {
2717c1deeebSVincenzo Frascino .name = "[vdso]",
272871402e0SDmitry Safonov .mremap = vdso_mremap,
2737c1deeebSVincenzo Frascino },
2740d747f65SVincenzo Frascino };
2759031fefdSWill Deacon
aarch32_alloc_kuser_vdso_page(void)2761255a734SVincenzo Frascino static int aarch32_alloc_kuser_vdso_page(void)
2779031fefdSWill Deacon {
2789031fefdSWill Deacon extern char __kuser_helper_start[], __kuser_helper_end[];
2799031fefdSWill Deacon int kuser_sz = __kuser_helper_end - __kuser_helper_start;
2801255a734SVincenzo Frascino unsigned long vdso_page;
2819031fefdSWill Deacon
282af1b3cf2SVincenzo Frascino if (!IS_ENABLED(CONFIG_KUSER_HELPERS))
283af1b3cf2SVincenzo Frascino return 0;
284af1b3cf2SVincenzo Frascino
2857cd6ca1dSWill Deacon vdso_page = get_zeroed_page(GFP_KERNEL);
2861255a734SVincenzo Frascino if (!vdso_page)
2879031fefdSWill Deacon return -ENOMEM;
2889031fefdSWill Deacon
2891255a734SVincenzo Frascino memcpy((void *)(vdso_page + 0x1000 - kuser_sz), __kuser_helper_start,
2901255a734SVincenzo Frascino kuser_sz);
291*b0abde80SLinus Walleij aarch32_vectors_page = virt_to_page((void *)vdso_page);
2921255a734SVincenzo Frascino return 0;
2930d747f65SVincenzo Frascino }
2940d747f65SVincenzo Frascino
2956e554abdSWill Deacon #define COMPAT_SIGPAGE_POISON_WORD 0xe7fddef1
aarch32_alloc_sigpage(void)296a39060b0SWill Deacon static int aarch32_alloc_sigpage(void)
2971255a734SVincenzo Frascino {
2981255a734SVincenzo Frascino extern char __aarch32_sigret_code_start[], __aarch32_sigret_code_end[];
2991255a734SVincenzo Frascino int sigret_sz = __aarch32_sigret_code_end - __aarch32_sigret_code_start;
3006e554abdSWill Deacon __le32 poison = cpu_to_le32(COMPAT_SIGPAGE_POISON_WORD);
3016e554abdSWill Deacon void *sigpage;
3029031fefdSWill Deacon
3036e554abdSWill Deacon sigpage = (void *)__get_free_page(GFP_KERNEL);
3041255a734SVincenzo Frascino if (!sigpage)
3051255a734SVincenzo Frascino return -ENOMEM;
3069031fefdSWill Deacon
3076e554abdSWill Deacon memset32(sigpage, (__force u32)poison, PAGE_SIZE / sizeof(poison));
3086e554abdSWill Deacon memcpy(sigpage, __aarch32_sigret_code_start, sigret_sz);
30974fc72e7SMark Rutland aarch32_sig_page = virt_to_page(sigpage);
310a39060b0SWill Deacon return 0;
311a39060b0SWill Deacon }
3120d747f65SVincenzo Frascino
__aarch32_alloc_vdso_pages(void)313a7dcf58aSJisheng Zhang static int __init __aarch32_alloc_vdso_pages(void)
314a39060b0SWill Deacon {
3150cbc2659SWill Deacon
3160cbc2659SWill Deacon if (!IS_ENABLED(CONFIG_COMPAT_VDSO))
3170cbc2659SWill Deacon return 0;
3180cbc2659SWill Deacon
319a39060b0SWill Deacon vdso_info[VDSO_ABI_AA32].dm = &aarch32_vdso_maps[AA32_MAP_VVAR];
320a39060b0SWill Deacon vdso_info[VDSO_ABI_AA32].cm = &aarch32_vdso_maps[AA32_MAP_VDSO];
3219031fefdSWill Deacon
322a39060b0SWill Deacon return __vdso_init(VDSO_ABI_AA32);
3239031fefdSWill Deacon }
3247c1deeebSVincenzo Frascino
aarch32_alloc_vdso_pages(void)3257c1deeebSVincenzo Frascino static int __init aarch32_alloc_vdso_pages(void)
3267c1deeebSVincenzo Frascino {
327a39060b0SWill Deacon int ret;
328a39060b0SWill Deacon
329a39060b0SWill Deacon ret = __aarch32_alloc_vdso_pages();
330a39060b0SWill Deacon if (ret)
331a39060b0SWill Deacon return ret;
332a39060b0SWill Deacon
333a39060b0SWill Deacon ret = aarch32_alloc_sigpage();
334a39060b0SWill Deacon if (ret)
335a39060b0SWill Deacon return ret;
336a39060b0SWill Deacon
337a39060b0SWill Deacon return aarch32_alloc_kuser_vdso_page();
3387c1deeebSVincenzo Frascino }
3390d747f65SVincenzo Frascino arch_initcall(aarch32_alloc_vdso_pages);
3409031fefdSWill Deacon
aarch32_kuser_helpers_setup(struct mm_struct * mm)3410d747f65SVincenzo Frascino static int aarch32_kuser_helpers_setup(struct mm_struct *mm)
3420d747f65SVincenzo Frascino {
3430d747f65SVincenzo Frascino void *ret;
3440d747f65SVincenzo Frascino
345af1b3cf2SVincenzo Frascino if (!IS_ENABLED(CONFIG_KUSER_HELPERS))
346af1b3cf2SVincenzo Frascino return 0;
347af1b3cf2SVincenzo Frascino
3480d747f65SVincenzo Frascino /*
3490d747f65SVincenzo Frascino * Avoid VM_MAYWRITE for compatibility with arch/arm/, where it's
3500d747f65SVincenzo Frascino * not safe to CoW the page containing the CPU exception vectors.
3510d747f65SVincenzo Frascino */
3520d747f65SVincenzo Frascino ret = _install_special_mapping(mm, AARCH32_VECTORS_BASE, PAGE_SIZE,
3530d747f65SVincenzo Frascino VM_READ | VM_EXEC |
3540d747f65SVincenzo Frascino VM_MAYREAD | VM_MAYEXEC,
3551d09094aSMark Rutland &aarch32_vdso_maps[AA32_MAP_VECTORS]);
3560d747f65SVincenzo Frascino
3570d747f65SVincenzo Frascino return PTR_ERR_OR_ZERO(ret);
3580d747f65SVincenzo Frascino }
3590d747f65SVincenzo Frascino
aarch32_sigreturn_setup(struct mm_struct * mm)3600d747f65SVincenzo Frascino static int aarch32_sigreturn_setup(struct mm_struct *mm)
3610d747f65SVincenzo Frascino {
3620d747f65SVincenzo Frascino unsigned long addr;
3630d747f65SVincenzo Frascino void *ret;
3640d747f65SVincenzo Frascino
3650d747f65SVincenzo Frascino addr = get_unmapped_area(NULL, 0, PAGE_SIZE, 0, 0);
3660d747f65SVincenzo Frascino if (IS_ERR_VALUE(addr)) {
3670d747f65SVincenzo Frascino ret = ERR_PTR(addr);
3680d747f65SVincenzo Frascino goto out;
3690d747f65SVincenzo Frascino }
3700d747f65SVincenzo Frascino
3710d747f65SVincenzo Frascino /*
3720d747f65SVincenzo Frascino * VM_MAYWRITE is required to allow gdb to Copy-on-Write and
3730d747f65SVincenzo Frascino * set breakpoints.
3740d747f65SVincenzo Frascino */
3750d747f65SVincenzo Frascino ret = _install_special_mapping(mm, addr, PAGE_SIZE,
3760d747f65SVincenzo Frascino VM_READ | VM_EXEC | VM_MAYREAD |
3770d747f65SVincenzo Frascino VM_MAYWRITE | VM_MAYEXEC,
3781d09094aSMark Rutland &aarch32_vdso_maps[AA32_MAP_SIGPAGE]);
3790d747f65SVincenzo Frascino if (IS_ERR(ret))
3800d747f65SVincenzo Frascino goto out;
3810d747f65SVincenzo Frascino
382a39060b0SWill Deacon mm->context.sigpage = (void *)addr;
3830d747f65SVincenzo Frascino
3840d747f65SVincenzo Frascino out:
3850d747f65SVincenzo Frascino return PTR_ERR_OR_ZERO(ret);
3860d747f65SVincenzo Frascino }
3870d747f65SVincenzo Frascino
aarch32_setup_additional_pages(struct linux_binprm * bprm,int uses_interp)3880d747f65SVincenzo Frascino int aarch32_setup_additional_pages(struct linux_binprm *bprm, int uses_interp)
3899031fefdSWill Deacon {
3909031fefdSWill Deacon struct mm_struct *mm = current->mm;
3910d747f65SVincenzo Frascino int ret;
3929031fefdSWill Deacon
393d8ed45c5SMichel Lespinasse if (mmap_write_lock_killable(mm))
39469048176SMichal Hocko return -EINTR;
3959031fefdSWill Deacon
3960d747f65SVincenzo Frascino ret = aarch32_kuser_helpers_setup(mm);
3970d747f65SVincenzo Frascino if (ret)
3980d747f65SVincenzo Frascino goto out;
3999031fefdSWill Deacon
4000cbc2659SWill Deacon if (IS_ENABLED(CONFIG_COMPAT_VDSO)) {
4012a30aca8SWill Deacon ret = __setup_additional_pages(VDSO_ABI_AA32, mm, bprm,
4027c1deeebSVincenzo Frascino uses_interp);
403a39060b0SWill Deacon if (ret)
404a39060b0SWill Deacon goto out;
4050cbc2659SWill Deacon }
4060d747f65SVincenzo Frascino
407a39060b0SWill Deacon ret = aarch32_sigreturn_setup(mm);
4080d747f65SVincenzo Frascino out:
409d8ed45c5SMichel Lespinasse mmap_write_unlock(mm);
4100d747f65SVincenzo Frascino return ret;
4119031fefdSWill Deacon }
4129031fefdSWill Deacon #endif /* CONFIG_COMPAT */
4139031fefdSWill Deacon
4141d09094aSMark Rutland enum aarch64_map {
4151d09094aSMark Rutland AA64_MAP_VVAR,
4161d09094aSMark Rutland AA64_MAP_VDSO,
4171d09094aSMark Rutland };
4181d09094aSMark Rutland
4191d09094aSMark Rutland static struct vm_special_mapping aarch64_vdso_maps[] __ro_after_init = {
4201d09094aSMark Rutland [AA64_MAP_VVAR] = {
4215a9e3e15SJisheng Zhang .name = "[vvar]",
422d53b5c01SAndrei Vagin .fault = vvar_fault,
4235a9e3e15SJisheng Zhang },
4241d09094aSMark Rutland [AA64_MAP_VDSO] = {
4255a9e3e15SJisheng Zhang .name = "[vdso]",
42673958695SDmitry Safonov .mremap = vdso_mremap,
4275a9e3e15SJisheng Zhang },
4285a9e3e15SJisheng Zhang };
4292fea7f6cSWill Deacon
vdso_init(void)4309031fefdSWill Deacon static int __init vdso_init(void)
4319031fefdSWill Deacon {
4321d09094aSMark Rutland vdso_info[VDSO_ABI_AA64].dm = &aarch64_vdso_maps[AA64_MAP_VVAR];
4331d09094aSMark Rutland vdso_info[VDSO_ABI_AA64].cm = &aarch64_vdso_maps[AA64_MAP_VDSO];
43416fb1a9bSNathan Lynch
435d3418f38SMark Rutland return __vdso_init(VDSO_ABI_AA64);
4369031fefdSWill Deacon }
4379031fefdSWill Deacon arch_initcall(vdso_init);
4389031fefdSWill Deacon
arch_setup_additional_pages(struct linux_binprm * bprm,int uses_interp)4392a30aca8SWill Deacon int arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp)
4409031fefdSWill Deacon {
4419031fefdSWill Deacon struct mm_struct *mm = current->mm;
442c7aa2d71SVincenzo Frascino int ret;
4439031fefdSWill Deacon
444d8ed45c5SMichel Lespinasse if (mmap_write_lock_killable(mm))
44569048176SMichal Hocko return -EINTR;
44687154938SWill Deacon
4472a30aca8SWill Deacon ret = __setup_additional_pages(VDSO_ABI_AA64, mm, bprm, uses_interp);
448d8ed45c5SMichel Lespinasse mmap_write_unlock(mm);
4499031fefdSWill Deacon
450c7aa2d71SVincenzo Frascino return ret;
4519031fefdSWill Deacon }
452