1b2441318SGreg Kroah-Hartman // SPDX-License-Identifier: GPL-2.0
20d55303cSDeepa Dinamani #include <linux/compat.h>
3250c2277SThomas Gleixner #include <linux/errno.h>
4250c2277SThomas Gleixner #include <linux/sched.h>
501042607SIngo Molnar #include <linux/sched/mm.h>
6250c2277SThomas Gleixner #include <linux/syscalls.h>
7250c2277SThomas Gleixner #include <linux/mm.h>
8250c2277SThomas Gleixner #include <linux/fs.h>
9250c2277SThomas Gleixner #include <linux/smp.h>
10250c2277SThomas Gleixner #include <linux/sem.h>
11250c2277SThomas Gleixner #include <linux/msg.h>
12250c2277SThomas Gleixner #include <linux/shm.h>
13250c2277SThomas Gleixner #include <linux/stat.h>
14250c2277SThomas Gleixner #include <linux/mman.h>
15250c2277SThomas Gleixner #include <linux/file.h>
16250c2277SThomas Gleixner #include <linux/utsname.h>
17250c2277SThomas Gleixner #include <linux/personality.h>
18cc503c1bSJiri Kosina #include <linux/random.h>
19e9c8abb6SGustavo F. Padovan #include <linux/uaccess.h>
20910b2c51SStephen Rothwell #include <linux/elf.h>
21250c2277SThomas Gleixner
221b028f78SDmitry Safonov #include <asm/elf.h>
23250c2277SThomas Gleixner #include <asm/ia32.h>
24250c2277SThomas Gleixner
25dfb09f9bSBorislav Petkov /*
26dfb09f9bSBorislav Petkov * Align a virtual address to avoid aliasing in the I$ on AMD F15h.
27dfb09f9bSBorislav Petkov */
get_align_mask(void)28f9902472SMichel Lespinasse static unsigned long get_align_mask(void)
29dfb09f9bSBorislav Petkov {
30dfb09f9bSBorislav Petkov /* handle 32- and 64-bit case with a single conditional */
31dfb09f9bSBorislav Petkov if (va_align.flags < 0 || !(va_align.flags & (2 - mmap_is_ia32())))
32f9902472SMichel Lespinasse return 0;
33dfb09f9bSBorislav Petkov
34dfb09f9bSBorislav Petkov if (!(current->flags & PF_RANDOMIZE))
35f9902472SMichel Lespinasse return 0;
36dfb09f9bSBorislav Petkov
37f9902472SMichel Lespinasse return va_align.mask;
38f9902472SMichel Lespinasse }
39dfb09f9bSBorislav Petkov
404e26d11fSHector Marco-Gisbert /*
414e26d11fSHector Marco-Gisbert * To avoid aliasing in the I$ on AMD F15h, the bits defined by the
424e26d11fSHector Marco-Gisbert * va_align.bits, [12:upper_bit), are set to a random value instead of
434e26d11fSHector Marco-Gisbert * zeroing them. This random value is computed once per boot. This form
444e26d11fSHector Marco-Gisbert * of ASLR is known as "per-boot ASLR".
454e26d11fSHector Marco-Gisbert *
464e26d11fSHector Marco-Gisbert * To achieve this, the random value is added to the info.align_offset
474e26d11fSHector Marco-Gisbert * value before calling vm_unmapped_area() or ORed directly to the
484e26d11fSHector Marco-Gisbert * address.
494e26d11fSHector Marco-Gisbert */
get_align_bits(void)504e26d11fSHector Marco-Gisbert static unsigned long get_align_bits(void)
514e26d11fSHector Marco-Gisbert {
524e26d11fSHector Marco-Gisbert return va_align.bits & get_align_mask();
534e26d11fSHector Marco-Gisbert }
544e26d11fSHector Marco-Gisbert
align_vdso_addr(unsigned long addr)55f9902472SMichel Lespinasse unsigned long align_vdso_addr(unsigned long addr)
56f9902472SMichel Lespinasse {
57f9902472SMichel Lespinasse unsigned long align_mask = get_align_mask();
584e26d11fSHector Marco-Gisbert addr = (addr + align_mask) & ~align_mask;
594e26d11fSHector Marco-Gisbert return addr | get_align_bits();
60dfb09f9bSBorislav Petkov }
61dfb09f9bSBorislav Petkov
control_va_addr_alignment(char * str)62dfb09f9bSBorislav Petkov static int __init control_va_addr_alignment(char *str)
63dfb09f9bSBorislav Petkov {
64dfb09f9bSBorislav Petkov /* guard against enabling this on other CPU families */
65dfb09f9bSBorislav Petkov if (va_align.flags < 0)
66dfb09f9bSBorislav Petkov return 1;
67dfb09f9bSBorislav Petkov
68dfb09f9bSBorislav Petkov if (*str == 0)
69dfb09f9bSBorislav Petkov return 1;
70dfb09f9bSBorislav Petkov
71dfb09f9bSBorislav Petkov if (!strcmp(str, "32"))
72dfb09f9bSBorislav Petkov va_align.flags = ALIGN_VA_32;
73dfb09f9bSBorislav Petkov else if (!strcmp(str, "64"))
74dfb09f9bSBorislav Petkov va_align.flags = ALIGN_VA_64;
75dfb09f9bSBorislav Petkov else if (!strcmp(str, "off"))
76dfb09f9bSBorislav Petkov va_align.flags = 0;
77dfb09f9bSBorislav Petkov else if (!strcmp(str, "on"))
78dfb09f9bSBorislav Petkov va_align.flags = ALIGN_VA_32 | ALIGN_VA_64;
79dfb09f9bSBorislav Petkov else
801ef64b1eSRandy Dunlap pr_warn("invalid option value: 'align_va_addr=%s'\n", str);
81dfb09f9bSBorislav Petkov
82dfb09f9bSBorislav Petkov return 1;
83dfb09f9bSBorislav Petkov }
841ef64b1eSRandy Dunlap __setup("align_va_addr=", control_va_addr_alignment);
85dfb09f9bSBorislav Petkov
SYSCALL_DEFINE6(mmap,unsigned long,addr,unsigned long,len,unsigned long,prot,unsigned long,flags,unsigned long,fd,unsigned long,off)860ac676fbSJason Baron SYSCALL_DEFINE6(mmap, unsigned long, addr, unsigned long, len,
870ac676fbSJason Baron unsigned long, prot, unsigned long, flags,
880ac676fbSJason Baron unsigned long, fd, unsigned long, off)
89250c2277SThomas Gleixner {
90250c2277SThomas Gleixner if (off & ~PAGE_MASK)
9191a8f6cbSAdrian Huang return -EINVAL;
92250c2277SThomas Gleixner
9391a8f6cbSAdrian Huang return ksys_mmap_pgoff(addr, len, prot, flags, fd, off >> PAGE_SHIFT);
94250c2277SThomas Gleixner }
95250c2277SThomas Gleixner
find_start_end(unsigned long addr,unsigned long flags,unsigned long * begin,unsigned long * end)96b569bab7SKirill A. Shutemov static void find_start_end(unsigned long addr, unsigned long flags,
97b569bab7SKirill A. Shutemov unsigned long *begin, unsigned long *end)
98250c2277SThomas Gleixner {
99a846446bSDmitry Safonov if (!in_32bit_syscall() && (flags & MAP_32BIT)) {
100250c2277SThomas Gleixner /* This is usually used needed to map code in small
101250c2277SThomas Gleixner model, so it needs to be in the first 31bit. Limit
102250c2277SThomas Gleixner it to that. This means we need to move the
103250c2277SThomas Gleixner unmapped base down for this case. This can give
104250c2277SThomas Gleixner conflicts with the heap, but we assume that glibc
105250c2277SThomas Gleixner malloc knows how to fall back to mmap. Give it 1GB
106250c2277SThomas Gleixner of playground for now. -AK */
107250c2277SThomas Gleixner *begin = 0x40000000;
108250c2277SThomas Gleixner *end = 0x80000000;
109cc503c1bSJiri Kosina if (current->flags & PF_RANDOMIZE) {
1109c6f0902SJason Cooper *begin = randomize_page(*begin, 0x02000000);
111cc503c1bSJiri Kosina }
1121b028f78SDmitry Safonov return;
113250c2277SThomas Gleixner }
1141b028f78SDmitry Safonov
1151b028f78SDmitry Safonov *begin = get_mmap_base(1);
116a846446bSDmitry Safonov if (in_32bit_syscall())
117b569bab7SKirill A. Shutemov *end = task_size_32bit();
118b569bab7SKirill A. Shutemov else
119b569bab7SKirill A. Shutemov *end = task_size_64bit(addr > DEFAULT_MAP_WINDOW);
120250c2277SThomas Gleixner }
121250c2277SThomas Gleixner
122250c2277SThomas Gleixner unsigned long
arch_get_unmapped_area(struct file * filp,unsigned long addr,unsigned long len,unsigned long pgoff,unsigned long flags)123250c2277SThomas Gleixner arch_get_unmapped_area(struct file *filp, unsigned long addr,
124250c2277SThomas Gleixner unsigned long len, unsigned long pgoff, unsigned long flags)
125250c2277SThomas Gleixner {
126250c2277SThomas Gleixner struct mm_struct *mm = current->mm;
127250c2277SThomas Gleixner struct vm_area_struct *vma;
128f9902472SMichel Lespinasse struct vm_unmapped_area_info info;
129250c2277SThomas Gleixner unsigned long begin, end;
130250c2277SThomas Gleixner
131250c2277SThomas Gleixner if (flags & MAP_FIXED)
132250c2277SThomas Gleixner return addr;
133250c2277SThomas Gleixner
134b569bab7SKirill A. Shutemov find_start_end(addr, flags, &begin, &end);
135250c2277SThomas Gleixner
136250c2277SThomas Gleixner if (len > end)
137250c2277SThomas Gleixner return -ENOMEM;
138250c2277SThomas Gleixner
139250c2277SThomas Gleixner if (addr) {
140250c2277SThomas Gleixner addr = PAGE_ALIGN(addr);
141250c2277SThomas Gleixner vma = find_vma(mm, addr);
142250c2277SThomas Gleixner if (end - len >= addr &&
1431be7107fSHugh Dickins (!vma || addr + len <= vm_start_gap(vma)))
144250c2277SThomas Gleixner return addr;
145250c2277SThomas Gleixner }
146250c2277SThomas Gleixner
147f9902472SMichel Lespinasse info.flags = 0;
148f9902472SMichel Lespinasse info.length = len;
149f9902472SMichel Lespinasse info.low_limit = begin;
150f9902472SMichel Lespinasse info.high_limit = end;
1514e26d11fSHector Marco-Gisbert info.align_mask = 0;
1527d025059SMichel Lespinasse info.align_offset = pgoff << PAGE_SHIFT;
1534e26d11fSHector Marco-Gisbert if (filp) {
1544e26d11fSHector Marco-Gisbert info.align_mask = get_align_mask();
1554e26d11fSHector Marco-Gisbert info.align_offset += get_align_bits();
1564e26d11fSHector Marco-Gisbert }
157f9902472SMichel Lespinasse return vm_unmapped_area(&info);
158250c2277SThomas Gleixner }
159cc503c1bSJiri Kosina
160cc503c1bSJiri Kosina unsigned long
arch_get_unmapped_area_topdown(struct file * filp,const unsigned long addr0,const unsigned long len,const unsigned long pgoff,const unsigned long flags)161cc503c1bSJiri Kosina arch_get_unmapped_area_topdown(struct file *filp, const unsigned long addr0,
162cc503c1bSJiri Kosina const unsigned long len, const unsigned long pgoff,
163cc503c1bSJiri Kosina const unsigned long flags)
164cc503c1bSJiri Kosina {
165cc503c1bSJiri Kosina struct vm_area_struct *vma;
166cc503c1bSJiri Kosina struct mm_struct *mm = current->mm;
167f9902472SMichel Lespinasse unsigned long addr = addr0;
168f9902472SMichel Lespinasse struct vm_unmapped_area_info info;
169cc503c1bSJiri Kosina
170cc503c1bSJiri Kosina /* requested length too big for entire address space */
171cc503c1bSJiri Kosina if (len > TASK_SIZE)
172cc503c1bSJiri Kosina return -ENOMEM;
173cc503c1bSJiri Kosina
1741e0f25dbSKirill A. Shutemov /* No address checking. See comment at mmap_address_hint_valid() */
175cc503c1bSJiri Kosina if (flags & MAP_FIXED)
176cc503c1bSJiri Kosina return addr;
177cc503c1bSJiri Kosina
178e3e81acaSYuanhan Liu /* for MAP_32BIT mappings we force the legacy mmap base */
179a846446bSDmitry Safonov if (!in_32bit_syscall() && (flags & MAP_32BIT))
180cc503c1bSJiri Kosina goto bottomup;
181cc503c1bSJiri Kosina
182cc503c1bSJiri Kosina /* requesting a specific address */
183cc503c1bSJiri Kosina if (addr) {
1841e0f25dbSKirill A. Shutemov addr &= PAGE_MASK;
1851e0f25dbSKirill A. Shutemov if (!mmap_address_hint_valid(addr, len))
1861e0f25dbSKirill A. Shutemov goto get_unmapped_area;
1871e0f25dbSKirill A. Shutemov
188cc503c1bSJiri Kosina vma = find_vma(mm, addr);
1891e0f25dbSKirill A. Shutemov if (!vma || addr + len <= vm_start_gap(vma))
190cc503c1bSJiri Kosina return addr;
191cc503c1bSJiri Kosina }
1921e0f25dbSKirill A. Shutemov get_unmapped_area:
193cc503c1bSJiri Kosina
194f9902472SMichel Lespinasse info.flags = VM_UNMAPPED_AREA_TOPDOWN;
195f9902472SMichel Lespinasse info.length = len;
196*29f890d1SRick Edgecombe if (!in_32bit_syscall() && (flags & MAP_ABOVE4G))
197*29f890d1SRick Edgecombe info.low_limit = SZ_4G;
198*29f890d1SRick Edgecombe else
199f9902472SMichel Lespinasse info.low_limit = PAGE_SIZE;
200*29f890d1SRick Edgecombe
2011b028f78SDmitry Safonov info.high_limit = get_mmap_base(0);
202b569bab7SKirill A. Shutemov
203b569bab7SKirill A. Shutemov /*
204b569bab7SKirill A. Shutemov * If hint address is above DEFAULT_MAP_WINDOW, look for unmapped area
205b569bab7SKirill A. Shutemov * in the full address space.
206b569bab7SKirill A. Shutemov *
207a846446bSDmitry Safonov * !in_32bit_syscall() check to avoid high addresses for x32
208a846446bSDmitry Safonov * (and make it no op on native i386).
209b569bab7SKirill A. Shutemov */
210a846446bSDmitry Safonov if (addr > DEFAULT_MAP_WINDOW && !in_32bit_syscall())
211b569bab7SKirill A. Shutemov info.high_limit += TASK_SIZE_MAX - DEFAULT_MAP_WINDOW;
212b569bab7SKirill A. Shutemov
2134e26d11fSHector Marco-Gisbert info.align_mask = 0;
2147d025059SMichel Lespinasse info.align_offset = pgoff << PAGE_SHIFT;
2154e26d11fSHector Marco-Gisbert if (filp) {
2164e26d11fSHector Marco-Gisbert info.align_mask = get_align_mask();
2174e26d11fSHector Marco-Gisbert info.align_offset += get_align_bits();
2184e26d11fSHector Marco-Gisbert }
219f9902472SMichel Lespinasse addr = vm_unmapped_area(&info);
220f9902472SMichel Lespinasse if (!(addr & ~PAGE_MASK))
221f9902472SMichel Lespinasse return addr;
222f9902472SMichel Lespinasse VM_BUG_ON(addr != -ENOMEM);
223b716ad95SXiao Guangrong
224cc503c1bSJiri Kosina bottomup:
225cc503c1bSJiri Kosina /*
226cc503c1bSJiri Kosina * A failed mmap() very likely causes application failure,
227cc503c1bSJiri Kosina * so fall back to the bottom-up function here. This scenario
228cc503c1bSJiri Kosina * can happen with large stack limits and large mmap()
229cc503c1bSJiri Kosina * allocations.
230cc503c1bSJiri Kosina */
231f9902472SMichel Lespinasse return arch_get_unmapped_area(filp, addr0, len, pgoff, flags);
232cc503c1bSJiri Kosina }
233