1caab277bSThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only 29031fefdSWill Deacon /* 30d747f65SVincenzo Frascino * VDSO implementations. 49031fefdSWill Deacon * 59031fefdSWill Deacon * Copyright (C) 2012 ARM Limited 69031fefdSWill Deacon * 79031fefdSWill Deacon * Author: Will Deacon <will.deacon@arm.com> 89031fefdSWill Deacon */ 99031fefdSWill Deacon 105a9e3e15SJisheng Zhang #include <linux/cache.h> 119031fefdSWill Deacon #include <linux/clocksource.h> 129031fefdSWill Deacon #include <linux/elf.h> 139031fefdSWill Deacon #include <linux/err.h> 149031fefdSWill Deacon #include <linux/errno.h> 159031fefdSWill Deacon #include <linux/gfp.h> 165a9e3e15SJisheng Zhang #include <linux/kernel.h> 179031fefdSWill Deacon #include <linux/mm.h> 189031fefdSWill Deacon #include <linux/sched.h> 199031fefdSWill Deacon #include <linux/signal.h> 209031fefdSWill Deacon #include <linux/slab.h> 21ee3cda8eSAndrei Vagin #include <linux/time_namespace.h> 22c60b0c28SCatalin Marinas #include <linux/timekeeper_internal.h> 239031fefdSWill Deacon #include <linux/vmalloc.h> 2428b1a824SVincenzo Frascino #include <vdso/datapage.h> 2528b1a824SVincenzo Frascino #include <vdso/helpers.h> 2628b1a824SVincenzo Frascino #include <vdso/vsyscall.h> 279031fefdSWill Deacon 289031fefdSWill Deacon #include <asm/cacheflush.h> 299031fefdSWill Deacon #include <asm/signal32.h> 309031fefdSWill Deacon #include <asm/vdso.h> 319031fefdSWill Deacon 32dbbb08f5SKees Cook extern char vdso_start[], vdso_end[]; 337c1deeebSVincenzo Frascino #ifdef CONFIG_COMPAT_VDSO 347c1deeebSVincenzo Frascino extern char vdso32_start[], vdso32_end[]; 357c1deeebSVincenzo Frascino #endif /* CONFIG_COMPAT_VDSO */ 36c7aa2d71SVincenzo Frascino 37d3418f38SMark Rutland enum vdso_abi { 38d3418f38SMark Rutland VDSO_ABI_AA64, 397c1deeebSVincenzo Frascino #ifdef CONFIG_COMPAT_VDSO 40d3418f38SMark Rutland VDSO_ABI_AA32, 417c1deeebSVincenzo Frascino #endif /* CONFIG_COMPAT_VDSO */ 42c7aa2d71SVincenzo Frascino }; 43c7aa2d71SVincenzo Frascino 443503d56cSAndrei Vagin enum vvar_pages { 453503d56cSAndrei Vagin VVAR_DATA_PAGE_OFFSET, 463503d56cSAndrei Vagin VVAR_TIMENS_PAGE_OFFSET, 473503d56cSAndrei Vagin VVAR_NR_PAGES, 483503d56cSAndrei Vagin }; 493503d56cSAndrei Vagin 50d3418f38SMark Rutland struct vdso_abi_info { 51c7aa2d71SVincenzo Frascino const char *name; 52c7aa2d71SVincenzo Frascino const char *vdso_code_start; 53c7aa2d71SVincenzo Frascino const char *vdso_code_end; 54c7aa2d71SVincenzo Frascino unsigned long vdso_pages; 55c7aa2d71SVincenzo Frascino /* Data Mapping */ 56c7aa2d71SVincenzo Frascino struct vm_special_mapping *dm; 57c7aa2d71SVincenzo Frascino /* Code Mapping */ 58c7aa2d71SVincenzo Frascino struct vm_special_mapping *cm; 59c7aa2d71SVincenzo Frascino }; 60c7aa2d71SVincenzo Frascino 61d3418f38SMark Rutland static struct vdso_abi_info vdso_info[] __ro_after_init = { 62d3418f38SMark Rutland [VDSO_ABI_AA64] = { 63c7aa2d71SVincenzo Frascino .name = "vdso", 64c7aa2d71SVincenzo Frascino .vdso_code_start = vdso_start, 65c7aa2d71SVincenzo Frascino .vdso_code_end = vdso_end, 66c7aa2d71SVincenzo Frascino }, 677c1deeebSVincenzo Frascino #ifdef CONFIG_COMPAT_VDSO 68d3418f38SMark Rutland [VDSO_ABI_AA32] = { 697c1deeebSVincenzo Frascino .name = "vdso32", 707c1deeebSVincenzo Frascino .vdso_code_start = vdso32_start, 717c1deeebSVincenzo Frascino .vdso_code_end = vdso32_end, 727c1deeebSVincenzo Frascino }, 737c1deeebSVincenzo Frascino #endif /* CONFIG_COMPAT_VDSO */ 74c7aa2d71SVincenzo Frascino }; 759031fefdSWill Deacon 769031fefdSWill Deacon /* 779031fefdSWill Deacon * The vDSO data page. 789031fefdSWill Deacon */ 799031fefdSWill Deacon static union { 8028b1a824SVincenzo Frascino struct vdso_data data[CS_BASES]; 819031fefdSWill Deacon u8 page[PAGE_SIZE]; 829031fefdSWill Deacon } vdso_data_store __page_aligned_data; 8328b1a824SVincenzo Frascino struct vdso_data *vdso_data = vdso_data_store.data; 849031fefdSWill Deacon 85d3418f38SMark Rutland static int __vdso_remap(enum vdso_abi abi, 86c7aa2d71SVincenzo Frascino const struct vm_special_mapping *sm, 87c7aa2d71SVincenzo Frascino struct vm_area_struct *new_vma) 88c7aa2d71SVincenzo Frascino { 89c7aa2d71SVincenzo Frascino unsigned long new_size = new_vma->vm_end - new_vma->vm_start; 90d3418f38SMark Rutland unsigned long vdso_size = vdso_info[abi].vdso_code_end - 91d3418f38SMark Rutland vdso_info[abi].vdso_code_start; 92c7aa2d71SVincenzo Frascino 93c7aa2d71SVincenzo Frascino if (vdso_size != new_size) 94c7aa2d71SVincenzo Frascino return -EINVAL; 95c7aa2d71SVincenzo Frascino 96c7aa2d71SVincenzo Frascino current->mm->context.vdso = (void *)new_vma->vm_start; 97c7aa2d71SVincenzo Frascino 98c7aa2d71SVincenzo Frascino return 0; 99c7aa2d71SVincenzo Frascino } 100c7aa2d71SVincenzo Frascino 101d3418f38SMark Rutland static int __vdso_init(enum vdso_abi abi) 102c7aa2d71SVincenzo Frascino { 103c7aa2d71SVincenzo Frascino int i; 104c7aa2d71SVincenzo Frascino struct page **vdso_pagelist; 105c7aa2d71SVincenzo Frascino unsigned long pfn; 106c7aa2d71SVincenzo Frascino 107d3418f38SMark Rutland if (memcmp(vdso_info[abi].vdso_code_start, "\177ELF", 4)) { 108c7aa2d71SVincenzo Frascino pr_err("vDSO is not a valid ELF object!\n"); 109c7aa2d71SVincenzo Frascino return -EINVAL; 110c7aa2d71SVincenzo Frascino } 111c7aa2d71SVincenzo Frascino 112d3418f38SMark Rutland vdso_info[abi].vdso_pages = ( 113d3418f38SMark Rutland vdso_info[abi].vdso_code_end - 114d3418f38SMark Rutland vdso_info[abi].vdso_code_start) >> 115c7aa2d71SVincenzo Frascino PAGE_SHIFT; 116c7aa2d71SVincenzo Frascino 117d53b5c01SAndrei Vagin vdso_pagelist = kcalloc(vdso_info[abi].vdso_pages, 118c7aa2d71SVincenzo Frascino sizeof(struct page *), 119c7aa2d71SVincenzo Frascino GFP_KERNEL); 120c7aa2d71SVincenzo Frascino if (vdso_pagelist == NULL) 121c7aa2d71SVincenzo Frascino return -ENOMEM; 122c7aa2d71SVincenzo Frascino 123c7aa2d71SVincenzo Frascino /* Grab the vDSO code pages. */ 124d3418f38SMark Rutland pfn = sym_to_pfn(vdso_info[abi].vdso_code_start); 125c7aa2d71SVincenzo Frascino 126d3418f38SMark Rutland for (i = 0; i < vdso_info[abi].vdso_pages; i++) 127d53b5c01SAndrei Vagin vdso_pagelist[i] = pfn_to_page(pfn + i); 128c7aa2d71SVincenzo Frascino 129d53b5c01SAndrei Vagin vdso_info[abi].cm->pages = vdso_pagelist; 130c7aa2d71SVincenzo Frascino 131c7aa2d71SVincenzo Frascino return 0; 132c7aa2d71SVincenzo Frascino } 133c7aa2d71SVincenzo Frascino 1341b6867d2SAndrei Vagin #ifdef CONFIG_TIME_NS 1353503d56cSAndrei Vagin struct vdso_data *arch_get_vdso_data(void *vvar_page) 1363503d56cSAndrei Vagin { 1373503d56cSAndrei Vagin return (struct vdso_data *)(vvar_page); 1383503d56cSAndrei Vagin } 1393503d56cSAndrei Vagin 1401b6867d2SAndrei Vagin /* 1411b6867d2SAndrei Vagin * The vvar mapping contains data for a specific time namespace, so when a task 1421b6867d2SAndrei Vagin * changes namespace we must unmap its vvar data for the old namespace. 1431b6867d2SAndrei Vagin * Subsequent faults will map in data for the new namespace. 1441b6867d2SAndrei Vagin * 1451b6867d2SAndrei Vagin * For more details see timens_setup_vdso_data(). 1461b6867d2SAndrei Vagin */ 1471b6867d2SAndrei Vagin int vdso_join_timens(struct task_struct *task, struct time_namespace *ns) 1481b6867d2SAndrei Vagin { 1491b6867d2SAndrei Vagin struct mm_struct *mm = task->mm; 1501b6867d2SAndrei Vagin struct vm_area_struct *vma; 1511b6867d2SAndrei Vagin 1521b6867d2SAndrei Vagin mmap_read_lock(mm); 1531b6867d2SAndrei Vagin 1541b6867d2SAndrei Vagin for (vma = mm->mmap; vma; vma = vma->vm_next) { 1551b6867d2SAndrei Vagin unsigned long size = vma->vm_end - vma->vm_start; 1561b6867d2SAndrei Vagin 1571b6867d2SAndrei Vagin if (vma_is_special_mapping(vma, vdso_info[VDSO_ABI_AA64].dm)) 1581b6867d2SAndrei Vagin zap_page_range(vma, vma->vm_start, size); 1591b6867d2SAndrei Vagin #ifdef CONFIG_COMPAT_VDSO 1601b6867d2SAndrei Vagin if (vma_is_special_mapping(vma, vdso_info[VDSO_ABI_AA32].dm)) 1611b6867d2SAndrei Vagin zap_page_range(vma, vma->vm_start, size); 1621b6867d2SAndrei Vagin #endif 1631b6867d2SAndrei Vagin } 1641b6867d2SAndrei Vagin 1651b6867d2SAndrei Vagin mmap_read_unlock(mm); 1661b6867d2SAndrei Vagin return 0; 1671b6867d2SAndrei Vagin } 168ee3cda8eSAndrei Vagin 169ee3cda8eSAndrei Vagin static struct page *find_timens_vvar_page(struct vm_area_struct *vma) 170ee3cda8eSAndrei Vagin { 171ee3cda8eSAndrei Vagin if (likely(vma->vm_mm == current->mm)) 172ee3cda8eSAndrei Vagin return current->nsproxy->time_ns->vvar_page; 173ee3cda8eSAndrei Vagin 174ee3cda8eSAndrei Vagin /* 175ee3cda8eSAndrei Vagin * VM_PFNMAP | VM_IO protect .fault() handler from being called 176ee3cda8eSAndrei Vagin * through interfaces like /proc/$pid/mem or 177ee3cda8eSAndrei Vagin * process_vm_{readv,writev}() as long as there's no .access() 178ee3cda8eSAndrei Vagin * in special_mapping_vmops. 179ee3cda8eSAndrei Vagin * For more details check_vma_flags() and __access_remote_vm() 180ee3cda8eSAndrei Vagin */ 181ee3cda8eSAndrei Vagin WARN(1, "vvar_page accessed remotely"); 182ee3cda8eSAndrei Vagin 183ee3cda8eSAndrei Vagin return NULL; 184ee3cda8eSAndrei Vagin } 185ee3cda8eSAndrei Vagin #else 186ee3cda8eSAndrei Vagin static struct page *find_timens_vvar_page(struct vm_area_struct *vma) 187ee3cda8eSAndrei Vagin { 188ee3cda8eSAndrei Vagin return NULL; 189ee3cda8eSAndrei Vagin } 1901b6867d2SAndrei Vagin #endif 1911b6867d2SAndrei Vagin 192d53b5c01SAndrei Vagin static vm_fault_t vvar_fault(const struct vm_special_mapping *sm, 193d53b5c01SAndrei Vagin struct vm_area_struct *vma, struct vm_fault *vmf) 194d53b5c01SAndrei Vagin { 195ee3cda8eSAndrei Vagin struct page *timens_page = find_timens_vvar_page(vma); 196ee3cda8eSAndrei Vagin unsigned long pfn; 197ee3cda8eSAndrei Vagin 198ee3cda8eSAndrei Vagin switch (vmf->pgoff) { 199ee3cda8eSAndrei Vagin case VVAR_DATA_PAGE_OFFSET: 200ee3cda8eSAndrei Vagin if (timens_page) 201ee3cda8eSAndrei Vagin pfn = page_to_pfn(timens_page); 202ee3cda8eSAndrei Vagin else 203ee3cda8eSAndrei Vagin pfn = sym_to_pfn(vdso_data); 204ee3cda8eSAndrei Vagin break; 205ee3cda8eSAndrei Vagin #ifdef CONFIG_TIME_NS 206ee3cda8eSAndrei Vagin case VVAR_TIMENS_PAGE_OFFSET: 207ee3cda8eSAndrei Vagin /* 208ee3cda8eSAndrei Vagin * If a task belongs to a time namespace then a namespace 209ee3cda8eSAndrei Vagin * specific VVAR is mapped with the VVAR_DATA_PAGE_OFFSET and 210ee3cda8eSAndrei Vagin * the real VVAR page is mapped with the VVAR_TIMENS_PAGE_OFFSET 211ee3cda8eSAndrei Vagin * offset. 212ee3cda8eSAndrei Vagin * See also the comment near timens_setup_vdso_data(). 213ee3cda8eSAndrei Vagin */ 214ee3cda8eSAndrei Vagin if (!timens_page) 215d53b5c01SAndrei Vagin return VM_FAULT_SIGBUS; 216ee3cda8eSAndrei Vagin pfn = sym_to_pfn(vdso_data); 217ee3cda8eSAndrei Vagin break; 218ee3cda8eSAndrei Vagin #endif /* CONFIG_TIME_NS */ 219ee3cda8eSAndrei Vagin default: 220ee3cda8eSAndrei Vagin return VM_FAULT_SIGBUS; 221ee3cda8eSAndrei Vagin } 222ee3cda8eSAndrei Vagin 223ee3cda8eSAndrei Vagin return vmf_insert_pfn(vma, vmf->address, pfn); 224d53b5c01SAndrei Vagin } 225d53b5c01SAndrei Vagin 226bcf99643SAndrei Vagin static int vvar_mremap(const struct vm_special_mapping *sm, 227bcf99643SAndrei Vagin struct vm_area_struct *new_vma) 228bcf99643SAndrei Vagin { 229bcf99643SAndrei Vagin unsigned long new_size = new_vma->vm_end - new_vma->vm_start; 230bcf99643SAndrei Vagin 231bcf99643SAndrei Vagin if (new_size != VVAR_NR_PAGES * PAGE_SIZE) 232bcf99643SAndrei Vagin return -EINVAL; 233bcf99643SAndrei Vagin 234bcf99643SAndrei Vagin return 0; 235bcf99643SAndrei Vagin } 236bcf99643SAndrei Vagin 237d3418f38SMark Rutland static int __setup_additional_pages(enum vdso_abi abi, 238c7aa2d71SVincenzo Frascino struct mm_struct *mm, 239c7aa2d71SVincenzo Frascino struct linux_binprm *bprm, 240c7aa2d71SVincenzo Frascino int uses_interp) 241c7aa2d71SVincenzo Frascino { 242c7aa2d71SVincenzo Frascino unsigned long vdso_base, vdso_text_len, vdso_mapping_len; 243bf740a90SMark Brown unsigned long gp_flags = 0; 244c7aa2d71SVincenzo Frascino void *ret; 245c7aa2d71SVincenzo Frascino 2463503d56cSAndrei Vagin BUILD_BUG_ON(VVAR_NR_PAGES != __VVAR_PAGES); 2473503d56cSAndrei Vagin 248d3418f38SMark Rutland vdso_text_len = vdso_info[abi].vdso_pages << PAGE_SHIFT; 249c7aa2d71SVincenzo Frascino /* Be sure to map the data page */ 2503503d56cSAndrei Vagin vdso_mapping_len = vdso_text_len + VVAR_NR_PAGES * PAGE_SIZE; 251c7aa2d71SVincenzo Frascino 252c7aa2d71SVincenzo Frascino vdso_base = get_unmapped_area(NULL, 0, vdso_mapping_len, 0, 0); 253c7aa2d71SVincenzo Frascino if (IS_ERR_VALUE(vdso_base)) { 254c7aa2d71SVincenzo Frascino ret = ERR_PTR(vdso_base); 255c7aa2d71SVincenzo Frascino goto up_fail; 256c7aa2d71SVincenzo Frascino } 257c7aa2d71SVincenzo Frascino 2583503d56cSAndrei Vagin ret = _install_special_mapping(mm, vdso_base, VVAR_NR_PAGES * PAGE_SIZE, 259d53b5c01SAndrei Vagin VM_READ|VM_MAYREAD|VM_PFNMAP, 260d3418f38SMark Rutland vdso_info[abi].dm); 261c7aa2d71SVincenzo Frascino if (IS_ERR(ret)) 262c7aa2d71SVincenzo Frascino goto up_fail; 263c7aa2d71SVincenzo Frascino 264bf740a90SMark Brown if (IS_ENABLED(CONFIG_ARM64_BTI_KERNEL) && system_supports_bti()) 265bf740a90SMark Brown gp_flags = VM_ARM64_BTI; 266bf740a90SMark Brown 2673503d56cSAndrei Vagin vdso_base += VVAR_NR_PAGES * PAGE_SIZE; 268c7aa2d71SVincenzo Frascino mm->context.vdso = (void *)vdso_base; 269c7aa2d71SVincenzo Frascino ret = _install_special_mapping(mm, vdso_base, vdso_text_len, 270bf740a90SMark Brown VM_READ|VM_EXEC|gp_flags| 271c7aa2d71SVincenzo Frascino VM_MAYREAD|VM_MAYWRITE|VM_MAYEXEC, 272d3418f38SMark Rutland vdso_info[abi].cm); 273c7aa2d71SVincenzo Frascino if (IS_ERR(ret)) 274c7aa2d71SVincenzo Frascino goto up_fail; 275c7aa2d71SVincenzo Frascino 276c7aa2d71SVincenzo Frascino return 0; 277c7aa2d71SVincenzo Frascino 278c7aa2d71SVincenzo Frascino up_fail: 279c7aa2d71SVincenzo Frascino mm->context.vdso = NULL; 280c7aa2d71SVincenzo Frascino return PTR_ERR(ret); 281c7aa2d71SVincenzo Frascino } 282c7aa2d71SVincenzo Frascino 2839031fefdSWill Deacon #ifdef CONFIG_COMPAT 2849031fefdSWill Deacon /* 2859031fefdSWill Deacon * Create and map the vectors page for AArch32 tasks. 2869031fefdSWill Deacon */ 2877c1deeebSVincenzo Frascino #ifdef CONFIG_COMPAT_VDSO 2887c1deeebSVincenzo Frascino static int aarch32_vdso_mremap(const struct vm_special_mapping *sm, 2897c1deeebSVincenzo Frascino struct vm_area_struct *new_vma) 2907c1deeebSVincenzo Frascino { 291d3418f38SMark Rutland return __vdso_remap(VDSO_ABI_AA32, sm, new_vma); 2927c1deeebSVincenzo Frascino } 2937c1deeebSVincenzo Frascino #endif /* CONFIG_COMPAT_VDSO */ 2947c1deeebSVincenzo Frascino 2951d09094aSMark Rutland enum aarch32_map { 2961d09094aSMark Rutland AA32_MAP_VECTORS, /* kuser helpers */ 2977c1deeebSVincenzo Frascino #ifdef CONFIG_COMPAT_VDSO 2981d09094aSMark Rutland AA32_MAP_VVAR, 2991d09094aSMark Rutland AA32_MAP_VDSO, 3001d09094aSMark Rutland #endif 301a39060b0SWill Deacon AA32_MAP_SIGPAGE 3021d09094aSMark Rutland }; 30374fc72e7SMark Rutland 30474fc72e7SMark Rutland static struct page *aarch32_vectors_page __ro_after_init; 30574fc72e7SMark Rutland static struct page *aarch32_sig_page __ro_after_init; 30674fc72e7SMark Rutland 3071d09094aSMark Rutland static struct vm_special_mapping aarch32_vdso_maps[] = { 3081d09094aSMark Rutland [AA32_MAP_VECTORS] = { 3090d747f65SVincenzo Frascino .name = "[vectors]", /* ABI */ 31074fc72e7SMark Rutland .pages = &aarch32_vectors_page, 3110d747f65SVincenzo Frascino }, 3127c1deeebSVincenzo Frascino #ifdef CONFIG_COMPAT_VDSO 3131d09094aSMark Rutland [AA32_MAP_VVAR] = { 3147c1deeebSVincenzo Frascino .name = "[vvar]", 315d53b5c01SAndrei Vagin .fault = vvar_fault, 316bcf99643SAndrei Vagin .mremap = vvar_mremap, 3177c1deeebSVincenzo Frascino }, 3181d09094aSMark Rutland [AA32_MAP_VDSO] = { 3197c1deeebSVincenzo Frascino .name = "[vdso]", 3207c1deeebSVincenzo Frascino .mremap = aarch32_vdso_mremap, 3217c1deeebSVincenzo Frascino }, 322a39060b0SWill Deacon #endif /* CONFIG_COMPAT_VDSO */ 3231d09094aSMark Rutland [AA32_MAP_SIGPAGE] = { 3240d747f65SVincenzo Frascino .name = "[sigpage]", /* ABI */ 32574fc72e7SMark Rutland .pages = &aarch32_sig_page, 3260d747f65SVincenzo Frascino }, 3270d747f65SVincenzo Frascino }; 3289031fefdSWill Deacon 3291255a734SVincenzo Frascino static int aarch32_alloc_kuser_vdso_page(void) 3309031fefdSWill Deacon { 3319031fefdSWill Deacon extern char __kuser_helper_start[], __kuser_helper_end[]; 3329031fefdSWill Deacon int kuser_sz = __kuser_helper_end - __kuser_helper_start; 3331255a734SVincenzo Frascino unsigned long vdso_page; 3349031fefdSWill Deacon 335af1b3cf2SVincenzo Frascino if (!IS_ENABLED(CONFIG_KUSER_HELPERS)) 336af1b3cf2SVincenzo Frascino return 0; 337af1b3cf2SVincenzo Frascino 3381255a734SVincenzo Frascino vdso_page = get_zeroed_page(GFP_ATOMIC); 3391255a734SVincenzo Frascino if (!vdso_page) 3409031fefdSWill Deacon return -ENOMEM; 3419031fefdSWill Deacon 3421255a734SVincenzo Frascino memcpy((void *)(vdso_page + 0x1000 - kuser_sz), __kuser_helper_start, 3431255a734SVincenzo Frascino kuser_sz); 34474fc72e7SMark Rutland aarch32_vectors_page = virt_to_page(vdso_page); 34574fc72e7SMark Rutland flush_dcache_page(aarch32_vectors_page); 3461255a734SVincenzo Frascino return 0; 3470d747f65SVincenzo Frascino } 3480d747f65SVincenzo Frascino 349a39060b0SWill Deacon static int aarch32_alloc_sigpage(void) 3501255a734SVincenzo Frascino { 3511255a734SVincenzo Frascino extern char __aarch32_sigret_code_start[], __aarch32_sigret_code_end[]; 3521255a734SVincenzo Frascino int sigret_sz = __aarch32_sigret_code_end - __aarch32_sigret_code_start; 3531255a734SVincenzo Frascino unsigned long sigpage; 3549031fefdSWill Deacon 3551255a734SVincenzo Frascino sigpage = get_zeroed_page(GFP_ATOMIC); 3561255a734SVincenzo Frascino if (!sigpage) 3571255a734SVincenzo Frascino return -ENOMEM; 3589031fefdSWill Deacon 3591255a734SVincenzo Frascino memcpy((void *)sigpage, __aarch32_sigret_code_start, sigret_sz); 36074fc72e7SMark Rutland aarch32_sig_page = virt_to_page(sigpage); 36174fc72e7SMark Rutland flush_dcache_page(aarch32_sig_page); 362a39060b0SWill Deacon return 0; 363a39060b0SWill Deacon } 3640d747f65SVincenzo Frascino 365a39060b0SWill Deacon #ifdef CONFIG_COMPAT_VDSO 366a39060b0SWill Deacon static int __aarch32_alloc_vdso_pages(void) 367a39060b0SWill Deacon { 368a39060b0SWill Deacon vdso_info[VDSO_ABI_AA32].dm = &aarch32_vdso_maps[AA32_MAP_VVAR]; 369a39060b0SWill Deacon vdso_info[VDSO_ABI_AA32].cm = &aarch32_vdso_maps[AA32_MAP_VDSO]; 3709031fefdSWill Deacon 371a39060b0SWill Deacon return __vdso_init(VDSO_ABI_AA32); 3729031fefdSWill Deacon } 3737c1deeebSVincenzo Frascino #endif /* CONFIG_COMPAT_VDSO */ 3747c1deeebSVincenzo Frascino 3757c1deeebSVincenzo Frascino static int __init aarch32_alloc_vdso_pages(void) 3767c1deeebSVincenzo Frascino { 377a39060b0SWill Deacon int ret; 378a39060b0SWill Deacon 379a39060b0SWill Deacon #ifdef CONFIG_COMPAT_VDSO 380a39060b0SWill Deacon ret = __aarch32_alloc_vdso_pages(); 381a39060b0SWill Deacon if (ret) 382a39060b0SWill Deacon return ret; 383a39060b0SWill Deacon #endif 384a39060b0SWill Deacon 385a39060b0SWill Deacon ret = aarch32_alloc_sigpage(); 386a39060b0SWill Deacon if (ret) 387a39060b0SWill Deacon return ret; 388a39060b0SWill Deacon 389a39060b0SWill Deacon return aarch32_alloc_kuser_vdso_page(); 3907c1deeebSVincenzo Frascino } 3910d747f65SVincenzo Frascino arch_initcall(aarch32_alloc_vdso_pages); 3929031fefdSWill Deacon 3930d747f65SVincenzo Frascino static int aarch32_kuser_helpers_setup(struct mm_struct *mm) 3940d747f65SVincenzo Frascino { 3950d747f65SVincenzo Frascino void *ret; 3960d747f65SVincenzo Frascino 397af1b3cf2SVincenzo Frascino if (!IS_ENABLED(CONFIG_KUSER_HELPERS)) 398af1b3cf2SVincenzo Frascino return 0; 399af1b3cf2SVincenzo Frascino 4000d747f65SVincenzo Frascino /* 4010d747f65SVincenzo Frascino * Avoid VM_MAYWRITE for compatibility with arch/arm/, where it's 4020d747f65SVincenzo Frascino * not safe to CoW the page containing the CPU exception vectors. 4030d747f65SVincenzo Frascino */ 4040d747f65SVincenzo Frascino ret = _install_special_mapping(mm, AARCH32_VECTORS_BASE, PAGE_SIZE, 4050d747f65SVincenzo Frascino VM_READ | VM_EXEC | 4060d747f65SVincenzo Frascino VM_MAYREAD | VM_MAYEXEC, 4071d09094aSMark Rutland &aarch32_vdso_maps[AA32_MAP_VECTORS]); 4080d747f65SVincenzo Frascino 4090d747f65SVincenzo Frascino return PTR_ERR_OR_ZERO(ret); 4100d747f65SVincenzo Frascino } 4110d747f65SVincenzo Frascino 4120d747f65SVincenzo Frascino static int aarch32_sigreturn_setup(struct mm_struct *mm) 4130d747f65SVincenzo Frascino { 4140d747f65SVincenzo Frascino unsigned long addr; 4150d747f65SVincenzo Frascino void *ret; 4160d747f65SVincenzo Frascino 4170d747f65SVincenzo Frascino addr = get_unmapped_area(NULL, 0, PAGE_SIZE, 0, 0); 4180d747f65SVincenzo Frascino if (IS_ERR_VALUE(addr)) { 4190d747f65SVincenzo Frascino ret = ERR_PTR(addr); 4200d747f65SVincenzo Frascino goto out; 4210d747f65SVincenzo Frascino } 4220d747f65SVincenzo Frascino 4230d747f65SVincenzo Frascino /* 4240d747f65SVincenzo Frascino * VM_MAYWRITE is required to allow gdb to Copy-on-Write and 4250d747f65SVincenzo Frascino * set breakpoints. 4260d747f65SVincenzo Frascino */ 4270d747f65SVincenzo Frascino ret = _install_special_mapping(mm, addr, PAGE_SIZE, 4280d747f65SVincenzo Frascino VM_READ | VM_EXEC | VM_MAYREAD | 4290d747f65SVincenzo Frascino VM_MAYWRITE | VM_MAYEXEC, 4301d09094aSMark Rutland &aarch32_vdso_maps[AA32_MAP_SIGPAGE]); 4310d747f65SVincenzo Frascino if (IS_ERR(ret)) 4320d747f65SVincenzo Frascino goto out; 4330d747f65SVincenzo Frascino 434a39060b0SWill Deacon mm->context.sigpage = (void *)addr; 4350d747f65SVincenzo Frascino 4360d747f65SVincenzo Frascino out: 4370d747f65SVincenzo Frascino return PTR_ERR_OR_ZERO(ret); 4380d747f65SVincenzo Frascino } 4390d747f65SVincenzo Frascino 4400d747f65SVincenzo Frascino int aarch32_setup_additional_pages(struct linux_binprm *bprm, int uses_interp) 4419031fefdSWill Deacon { 4429031fefdSWill Deacon struct mm_struct *mm = current->mm; 4430d747f65SVincenzo Frascino int ret; 4449031fefdSWill Deacon 445d8ed45c5SMichel Lespinasse if (mmap_write_lock_killable(mm)) 44669048176SMichal Hocko return -EINTR; 4479031fefdSWill Deacon 4480d747f65SVincenzo Frascino ret = aarch32_kuser_helpers_setup(mm); 4490d747f65SVincenzo Frascino if (ret) 4500d747f65SVincenzo Frascino goto out; 4519031fefdSWill Deacon 4527c1deeebSVincenzo Frascino #ifdef CONFIG_COMPAT_VDSO 453d3418f38SMark Rutland ret = __setup_additional_pages(VDSO_ABI_AA32, 4547c1deeebSVincenzo Frascino mm, 4557c1deeebSVincenzo Frascino bprm, 4567c1deeebSVincenzo Frascino uses_interp); 457a39060b0SWill Deacon if (ret) 458a39060b0SWill Deacon goto out; 4597c1deeebSVincenzo Frascino #endif /* CONFIG_COMPAT_VDSO */ 4600d747f65SVincenzo Frascino 461a39060b0SWill Deacon ret = aarch32_sigreturn_setup(mm); 4620d747f65SVincenzo Frascino out: 463d8ed45c5SMichel Lespinasse mmap_write_unlock(mm); 4640d747f65SVincenzo Frascino return ret; 4659031fefdSWill Deacon } 4669031fefdSWill Deacon #endif /* CONFIG_COMPAT */ 4679031fefdSWill Deacon 46873958695SDmitry Safonov static int vdso_mremap(const struct vm_special_mapping *sm, 46973958695SDmitry Safonov struct vm_area_struct *new_vma) 47073958695SDmitry Safonov { 471d3418f38SMark Rutland return __vdso_remap(VDSO_ABI_AA64, sm, new_vma); 47273958695SDmitry Safonov } 47373958695SDmitry Safonov 4741d09094aSMark Rutland enum aarch64_map { 4751d09094aSMark Rutland AA64_MAP_VVAR, 4761d09094aSMark Rutland AA64_MAP_VDSO, 4771d09094aSMark Rutland }; 4781d09094aSMark Rutland 4791d09094aSMark Rutland static struct vm_special_mapping aarch64_vdso_maps[] __ro_after_init = { 4801d09094aSMark Rutland [AA64_MAP_VVAR] = { 4815a9e3e15SJisheng Zhang .name = "[vvar]", 482d53b5c01SAndrei Vagin .fault = vvar_fault, 483bcf99643SAndrei Vagin .mremap = vvar_mremap, 4845a9e3e15SJisheng Zhang }, 4851d09094aSMark Rutland [AA64_MAP_VDSO] = { 4865a9e3e15SJisheng Zhang .name = "[vdso]", 48773958695SDmitry Safonov .mremap = vdso_mremap, 4885a9e3e15SJisheng Zhang }, 4895a9e3e15SJisheng Zhang }; 4902fea7f6cSWill Deacon 4919031fefdSWill Deacon static int __init vdso_init(void) 4929031fefdSWill Deacon { 4931d09094aSMark Rutland vdso_info[VDSO_ABI_AA64].dm = &aarch64_vdso_maps[AA64_MAP_VVAR]; 4941d09094aSMark Rutland vdso_info[VDSO_ABI_AA64].cm = &aarch64_vdso_maps[AA64_MAP_VDSO]; 49516fb1a9bSNathan Lynch 496d3418f38SMark Rutland return __vdso_init(VDSO_ABI_AA64); 4979031fefdSWill Deacon } 4989031fefdSWill Deacon arch_initcall(vdso_init); 4999031fefdSWill Deacon 5009031fefdSWill Deacon int arch_setup_additional_pages(struct linux_binprm *bprm, 5019031fefdSWill Deacon int uses_interp) 5029031fefdSWill Deacon { 5039031fefdSWill Deacon struct mm_struct *mm = current->mm; 504c7aa2d71SVincenzo Frascino int ret; 5059031fefdSWill Deacon 506d8ed45c5SMichel Lespinasse if (mmap_write_lock_killable(mm)) 50769048176SMichal Hocko return -EINTR; 50887154938SWill Deacon 509d3418f38SMark Rutland ret = __setup_additional_pages(VDSO_ABI_AA64, 510c7aa2d71SVincenzo Frascino mm, 511c7aa2d71SVincenzo Frascino bprm, 512c7aa2d71SVincenzo Frascino uses_interp); 513601255aeSWill Deacon 514d8ed45c5SMichel Lespinasse mmap_write_unlock(mm); 5159031fefdSWill Deacon 516c7aa2d71SVincenzo Frascino return ret; 5179031fefdSWill Deacon } 518