1caab277bSThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only
276d2a049SPalmer Dabbelt /*
376d2a049SPalmer Dabbelt * Copyright (C) 2004 Benjamin Herrenschmidt, IBM Corp.
476d2a049SPalmer Dabbelt * <benh@kernel.crashing.org>
576d2a049SPalmer Dabbelt * Copyright (C) 2012 ARM Limited
676d2a049SPalmer Dabbelt * Copyright (C) 2015 Regents of the University of California
776d2a049SPalmer Dabbelt */
876d2a049SPalmer Dabbelt
95ed881bcSPaul Walmsley #include <linux/elf.h>
1076d2a049SPalmer Dabbelt #include <linux/mm.h>
1176d2a049SPalmer Dabbelt #include <linux/slab.h>
1276d2a049SPalmer Dabbelt #include <linux/binfmts.h>
1376d2a049SPalmer Dabbelt #include <linux/err.h>
14ad5d1122SVincent Chen #include <asm/page.h>
15bb4a23c9STong Tiangen #include <asm/vdso.h>
16dffe11e2STong Tiangen #include <linux/time_namespace.h>
17ad5d1122SVincent Chen #include <vdso/datapage.h>
18*54cdede0SBen Dooks #include <vdso/vsyscall.h>
1976d2a049SPalmer Dabbelt
2078a743cdSTong Tiangen enum vvar_pages {
2178a743cdSTong Tiangen VVAR_DATA_PAGE_OFFSET,
22dffe11e2STong Tiangen VVAR_TIMENS_PAGE_OFFSET,
2378a743cdSTong Tiangen VVAR_NR_PAGES,
2478a743cdSTong Tiangen };
2578a743cdSTong Tiangen
263092eb45SGuo Ren enum rv_vdso_map {
273092eb45SGuo Ren RV_VDSO_MAP_VVAR,
283092eb45SGuo Ren RV_VDSO_MAP_VDSO,
293092eb45SGuo Ren };
303092eb45SGuo Ren
3178a743cdSTong Tiangen #define VVAR_SIZE (VVAR_NR_PAGES << PAGE_SHIFT)
3278a743cdSTong Tiangen
3376d2a049SPalmer Dabbelt /*
3476d2a049SPalmer Dabbelt * The vDSO data page.
3576d2a049SPalmer Dabbelt */
3676d2a049SPalmer Dabbelt static union {
3776d2a049SPalmer Dabbelt struct vdso_data data;
3876d2a049SPalmer Dabbelt u8 page[PAGE_SIZE];
3976d2a049SPalmer Dabbelt } vdso_data_store __page_aligned_data;
40ad5d1122SVincent Chen struct vdso_data *vdso_data = &vdso_data_store.data;
4176d2a049SPalmer Dabbelt
42dffe11e2STong Tiangen struct __vdso_info {
43dffe11e2STong Tiangen const char *name;
44dffe11e2STong Tiangen const char *vdso_code_start;
45dffe11e2STong Tiangen const char *vdso_code_end;
46dffe11e2STong Tiangen unsigned long vdso_pages;
47dffe11e2STong Tiangen /* Data Mapping */
48dffe11e2STong Tiangen struct vm_special_mapping *dm;
49dffe11e2STong Tiangen /* Code Mapping */
50dffe11e2STong Tiangen struct vm_special_mapping *cm;
51dffe11e2STong Tiangen };
52dffe11e2STong Tiangen
53a8616d2dSJisheng Zhang static struct __vdso_info vdso_info;
54a8616d2dSJisheng Zhang #ifdef CONFIG_COMPAT
55a8616d2dSJisheng Zhang static struct __vdso_info compat_vdso_info;
56a8616d2dSJisheng Zhang #endif
57a8616d2dSJisheng Zhang
vdso_mremap(const struct vm_special_mapping * sm,struct vm_area_struct * new_vma)58dffe11e2STong Tiangen static int vdso_mremap(const struct vm_special_mapping *sm,
59dffe11e2STong Tiangen struct vm_area_struct *new_vma)
6076d2a049SPalmer Dabbelt {
61dffe11e2STong Tiangen current->mm->context.vdso = (void *)new_vma->vm_start;
6276d2a049SPalmer Dabbelt
6376d2a049SPalmer Dabbelt return 0;
6476d2a049SPalmer Dabbelt }
65dffe11e2STong Tiangen
__vdso_init(struct __vdso_info * vdso_info)663092eb45SGuo Ren static void __init __vdso_init(struct __vdso_info *vdso_info)
67dffe11e2STong Tiangen {
68dffe11e2STong Tiangen unsigned int i;
69dffe11e2STong Tiangen struct page **vdso_pagelist;
70dffe11e2STong Tiangen unsigned long pfn;
71dffe11e2STong Tiangen
723092eb45SGuo Ren if (memcmp(vdso_info->vdso_code_start, "\177ELF", 4))
733092eb45SGuo Ren panic("vDSO is not a valid ELF object!\n");
74dffe11e2STong Tiangen
753092eb45SGuo Ren vdso_info->vdso_pages = (
763092eb45SGuo Ren vdso_info->vdso_code_end -
773092eb45SGuo Ren vdso_info->vdso_code_start) >>
78dffe11e2STong Tiangen PAGE_SHIFT;
79dffe11e2STong Tiangen
803092eb45SGuo Ren vdso_pagelist = kcalloc(vdso_info->vdso_pages,
81dffe11e2STong Tiangen sizeof(struct page *),
82dffe11e2STong Tiangen GFP_KERNEL);
83dffe11e2STong Tiangen if (vdso_pagelist == NULL)
843092eb45SGuo Ren panic("vDSO kcalloc failed!\n");
85dffe11e2STong Tiangen
86dffe11e2STong Tiangen /* Grab the vDSO code pages. */
873092eb45SGuo Ren pfn = sym_to_pfn(vdso_info->vdso_code_start);
88dffe11e2STong Tiangen
893092eb45SGuo Ren for (i = 0; i < vdso_info->vdso_pages; i++)
90dffe11e2STong Tiangen vdso_pagelist[i] = pfn_to_page(pfn + i);
91dffe11e2STong Tiangen
923092eb45SGuo Ren vdso_info->cm->pages = vdso_pagelist;
93dffe11e2STong Tiangen }
94dffe11e2STong Tiangen
95dffe11e2STong Tiangen #ifdef CONFIG_TIME_NS
arch_get_vdso_data(void * vvar_page)96dffe11e2STong Tiangen struct vdso_data *arch_get_vdso_data(void *vvar_page)
97dffe11e2STong Tiangen {
98dffe11e2STong Tiangen return (struct vdso_data *)(vvar_page);
99dffe11e2STong Tiangen }
100dffe11e2STong Tiangen
101dffe11e2STong Tiangen /*
102dffe11e2STong Tiangen * The vvar mapping contains data for a specific time namespace, so when a task
103dffe11e2STong Tiangen * changes namespace we must unmap its vvar data for the old namespace.
104dffe11e2STong Tiangen * Subsequent faults will map in data for the new namespace.
105dffe11e2STong Tiangen *
106dffe11e2STong Tiangen * For more details see timens_setup_vdso_data().
107dffe11e2STong Tiangen */
vdso_join_timens(struct task_struct * task,struct time_namespace * ns)108dffe11e2STong Tiangen int vdso_join_timens(struct task_struct *task, struct time_namespace *ns)
109dffe11e2STong Tiangen {
110dffe11e2STong Tiangen struct mm_struct *mm = task->mm;
111dffe11e2STong Tiangen struct vm_area_struct *vma;
1129b580a1dSLiam R. Howlett VMA_ITERATOR(vmi, mm, 0);
113dffe11e2STong Tiangen
114dffe11e2STong Tiangen mmap_read_lock(mm);
115dffe11e2STong Tiangen
1169b580a1dSLiam R. Howlett for_each_vma(vmi, vma) {
117a8616d2dSJisheng Zhang if (vma_is_special_mapping(vma, vdso_info.dm))
118e9adcfecSMike Kravetz zap_vma_pages(vma);
119a8616d2dSJisheng Zhang #ifdef CONFIG_COMPAT
120a8616d2dSJisheng Zhang if (vma_is_special_mapping(vma, compat_vdso_info.dm))
121e9adcfecSMike Kravetz zap_vma_pages(vma);
122a8616d2dSJisheng Zhang #endif
123dffe11e2STong Tiangen }
124dffe11e2STong Tiangen
125dffe11e2STong Tiangen mmap_read_unlock(mm);
126dffe11e2STong Tiangen return 0;
127dffe11e2STong Tiangen }
128dffe11e2STong Tiangen #endif
129dffe11e2STong Tiangen
vvar_fault(const struct vm_special_mapping * sm,struct vm_area_struct * vma,struct vm_fault * vmf)130dffe11e2STong Tiangen static vm_fault_t vvar_fault(const struct vm_special_mapping *sm,
131dffe11e2STong Tiangen struct vm_area_struct *vma, struct vm_fault *vmf)
132dffe11e2STong Tiangen {
133dffe11e2STong Tiangen struct page *timens_page = find_timens_vvar_page(vma);
134dffe11e2STong Tiangen unsigned long pfn;
135dffe11e2STong Tiangen
136dffe11e2STong Tiangen switch (vmf->pgoff) {
137dffe11e2STong Tiangen case VVAR_DATA_PAGE_OFFSET:
138dffe11e2STong Tiangen if (timens_page)
139dffe11e2STong Tiangen pfn = page_to_pfn(timens_page);
140dffe11e2STong Tiangen else
141dffe11e2STong Tiangen pfn = sym_to_pfn(vdso_data);
142dffe11e2STong Tiangen break;
143dffe11e2STong Tiangen #ifdef CONFIG_TIME_NS
144dffe11e2STong Tiangen case VVAR_TIMENS_PAGE_OFFSET:
145dffe11e2STong Tiangen /*
146dffe11e2STong Tiangen * If a task belongs to a time namespace then a namespace
147dffe11e2STong Tiangen * specific VVAR is mapped with the VVAR_DATA_PAGE_OFFSET and
148dffe11e2STong Tiangen * the real VVAR page is mapped with the VVAR_TIMENS_PAGE_OFFSET
149dffe11e2STong Tiangen * offset.
150dffe11e2STong Tiangen * See also the comment near timens_setup_vdso_data().
151dffe11e2STong Tiangen */
152dffe11e2STong Tiangen if (!timens_page)
153dffe11e2STong Tiangen return VM_FAULT_SIGBUS;
154dffe11e2STong Tiangen pfn = sym_to_pfn(vdso_data);
155dffe11e2STong Tiangen break;
156dffe11e2STong Tiangen #endif /* CONFIG_TIME_NS */
157dffe11e2STong Tiangen default:
158dffe11e2STong Tiangen return VM_FAULT_SIGBUS;
159dffe11e2STong Tiangen }
160dffe11e2STong Tiangen
161dffe11e2STong Tiangen return vmf_insert_pfn(vma, vmf->address, pfn);
162dffe11e2STong Tiangen }
163dffe11e2STong Tiangen
164dffe11e2STong Tiangen static struct vm_special_mapping rv_vdso_maps[] __ro_after_init = {
165dffe11e2STong Tiangen [RV_VDSO_MAP_VVAR] = {
166dffe11e2STong Tiangen .name = "[vvar]",
167dffe11e2STong Tiangen .fault = vvar_fault,
168dffe11e2STong Tiangen },
169dffe11e2STong Tiangen [RV_VDSO_MAP_VDSO] = {
170dffe11e2STong Tiangen .name = "[vdso]",
171dffe11e2STong Tiangen .mremap = vdso_mremap,
172dffe11e2STong Tiangen },
173dffe11e2STong Tiangen };
174dffe11e2STong Tiangen
1753092eb45SGuo Ren static struct __vdso_info vdso_info __ro_after_init = {
1763092eb45SGuo Ren .name = "vdso",
1773092eb45SGuo Ren .vdso_code_start = vdso_start,
1783092eb45SGuo Ren .vdso_code_end = vdso_end,
1793092eb45SGuo Ren .dm = &rv_vdso_maps[RV_VDSO_MAP_VVAR],
1803092eb45SGuo Ren .cm = &rv_vdso_maps[RV_VDSO_MAP_VDSO],
1813092eb45SGuo Ren };
1823092eb45SGuo Ren
1833092eb45SGuo Ren #ifdef CONFIG_COMPAT
184324373f4SGuo Ren static struct vm_special_mapping rv_compat_vdso_maps[] __ro_after_init = {
185324373f4SGuo Ren [RV_VDSO_MAP_VVAR] = {
186324373f4SGuo Ren .name = "[vvar]",
187324373f4SGuo Ren .fault = vvar_fault,
188324373f4SGuo Ren },
189324373f4SGuo Ren [RV_VDSO_MAP_VDSO] = {
190324373f4SGuo Ren .name = "[vdso]",
191324373f4SGuo Ren .mremap = vdso_mremap,
192324373f4SGuo Ren },
193324373f4SGuo Ren };
194324373f4SGuo Ren
1953092eb45SGuo Ren static struct __vdso_info compat_vdso_info __ro_after_init = {
1963092eb45SGuo Ren .name = "compat_vdso",
1973092eb45SGuo Ren .vdso_code_start = compat_vdso_start,
1983092eb45SGuo Ren .vdso_code_end = compat_vdso_end,
199324373f4SGuo Ren .dm = &rv_compat_vdso_maps[RV_VDSO_MAP_VVAR],
200324373f4SGuo Ren .cm = &rv_compat_vdso_maps[RV_VDSO_MAP_VDSO],
2013092eb45SGuo Ren };
2023092eb45SGuo Ren #endif
2033092eb45SGuo Ren
vdso_init(void)204dffe11e2STong Tiangen static int __init vdso_init(void)
205dffe11e2STong Tiangen {
2063092eb45SGuo Ren __vdso_init(&vdso_info);
2073092eb45SGuo Ren #ifdef CONFIG_COMPAT
2083092eb45SGuo Ren __vdso_init(&compat_vdso_info);
2093092eb45SGuo Ren #endif
210dffe11e2STong Tiangen
2113092eb45SGuo Ren return 0;
212dffe11e2STong Tiangen }
21376d2a049SPalmer Dabbelt arch_initcall(vdso_init);
21476d2a049SPalmer Dabbelt
__setup_additional_pages(struct mm_struct * mm,struct linux_binprm * bprm,int uses_interp,struct __vdso_info * vdso_info)215dffe11e2STong Tiangen static int __setup_additional_pages(struct mm_struct *mm,
216dffe11e2STong Tiangen struct linux_binprm *bprm,
2173092eb45SGuo Ren int uses_interp,
2183092eb45SGuo Ren struct __vdso_info *vdso_info)
21976d2a049SPalmer Dabbelt {
220dffe11e2STong Tiangen unsigned long vdso_base, vdso_text_len, vdso_mapping_len;
221dffe11e2STong Tiangen void *ret;
22276d2a049SPalmer Dabbelt
22378a743cdSTong Tiangen BUILD_BUG_ON(VVAR_NR_PAGES != __VVAR_PAGES);
22478a743cdSTong Tiangen
2253092eb45SGuo Ren vdso_text_len = vdso_info->vdso_pages << PAGE_SHIFT;
226dffe11e2STong Tiangen /* Be sure to map the data page */
227dffe11e2STong Tiangen vdso_mapping_len = vdso_text_len + VVAR_SIZE;
228dffe11e2STong Tiangen
229dffe11e2STong Tiangen vdso_base = get_unmapped_area(NULL, 0, vdso_mapping_len, 0, 0);
230dffe11e2STong Tiangen if (IS_ERR_VALUE(vdso_base)) {
231dffe11e2STong Tiangen ret = ERR_PTR(vdso_base);
232dffe11e2STong Tiangen goto up_fail;
233dffe11e2STong Tiangen }
234dffe11e2STong Tiangen
235dffe11e2STong Tiangen ret = _install_special_mapping(mm, vdso_base, VVAR_SIZE,
2363092eb45SGuo Ren (VM_READ | VM_MAYREAD | VM_PFNMAP), vdso_info->dm);
237dffe11e2STong Tiangen if (IS_ERR(ret))
238dffe11e2STong Tiangen goto up_fail;
239dffe11e2STong Tiangen
240dffe11e2STong Tiangen vdso_base += VVAR_SIZE;
241dffe11e2STong Tiangen mm->context.vdso = (void *)vdso_base;
2423092eb45SGuo Ren
243dffe11e2STong Tiangen ret =
244dffe11e2STong Tiangen _install_special_mapping(mm, vdso_base, vdso_text_len,
245dffe11e2STong Tiangen (VM_READ | VM_EXEC | VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC),
2463092eb45SGuo Ren vdso_info->cm);
247dffe11e2STong Tiangen
248dffe11e2STong Tiangen if (IS_ERR(ret))
249dffe11e2STong Tiangen goto up_fail;
250dffe11e2STong Tiangen
251dffe11e2STong Tiangen return 0;
252dffe11e2STong Tiangen
253dffe11e2STong Tiangen up_fail:
254dffe11e2STong Tiangen mm->context.vdso = NULL;
255dffe11e2STong Tiangen return PTR_ERR(ret);
256dffe11e2STong Tiangen }
257dffe11e2STong Tiangen
2583092eb45SGuo Ren #ifdef CONFIG_COMPAT
compat_arch_setup_additional_pages(struct linux_binprm * bprm,int uses_interp)2593092eb45SGuo Ren int compat_arch_setup_additional_pages(struct linux_binprm *bprm,
2603092eb45SGuo Ren int uses_interp)
2613092eb45SGuo Ren {
2623092eb45SGuo Ren struct mm_struct *mm = current->mm;
2633092eb45SGuo Ren int ret;
2643092eb45SGuo Ren
2653092eb45SGuo Ren if (mmap_write_lock_killable(mm))
2663092eb45SGuo Ren return -EINTR;
2673092eb45SGuo Ren
2683092eb45SGuo Ren ret = __setup_additional_pages(mm, bprm, uses_interp,
2693092eb45SGuo Ren &compat_vdso_info);
2703092eb45SGuo Ren mmap_write_unlock(mm);
2713092eb45SGuo Ren
2723092eb45SGuo Ren return ret;
2733092eb45SGuo Ren }
2743092eb45SGuo Ren #endif
2753092eb45SGuo Ren
arch_setup_additional_pages(struct linux_binprm * bprm,int uses_interp)276dffe11e2STong Tiangen int arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp)
277dffe11e2STong Tiangen {
278dffe11e2STong Tiangen struct mm_struct *mm = current->mm;
279dffe11e2STong Tiangen int ret;
28076d2a049SPalmer Dabbelt
2818bb0ab3aSTong Tiangen if (mmap_write_lock_killable(mm))
2828bb0ab3aSTong Tiangen return -EINTR;
2838bb0ab3aSTong Tiangen
2843092eb45SGuo Ren ret = __setup_additional_pages(mm, bprm, uses_interp, &vdso_info);
285d8ed45c5SMichel Lespinasse mmap_write_unlock(mm);
28676d2a049SPalmer Dabbelt
287dffe11e2STong Tiangen return ret;
28876d2a049SPalmer Dabbelt }
289