1250c2277SThomas Gleixner #include <linux/errno.h> 2250c2277SThomas Gleixner #include <linux/sched.h> 3250c2277SThomas Gleixner #include <linux/syscalls.h> 4250c2277SThomas Gleixner #include <linux/mm.h> 5250c2277SThomas Gleixner #include <linux/fs.h> 6250c2277SThomas Gleixner #include <linux/smp.h> 7250c2277SThomas Gleixner #include <linux/sem.h> 8250c2277SThomas Gleixner #include <linux/msg.h> 9250c2277SThomas Gleixner #include <linux/shm.h> 10250c2277SThomas Gleixner #include <linux/stat.h> 11250c2277SThomas Gleixner #include <linux/mman.h> 12250c2277SThomas Gleixner #include <linux/file.h> 13250c2277SThomas Gleixner #include <linux/utsname.h> 14250c2277SThomas Gleixner #include <linux/personality.h> 15cc503c1bSJiri Kosina #include <linux/random.h> 16e9c8abb6SGustavo F. Padovan #include <linux/uaccess.h> 17910b2c51SStephen Rothwell #include <linux/elf.h> 18250c2277SThomas Gleixner 19250c2277SThomas Gleixner #include <asm/ia32.h> 20bbc1f698SJaswinder Singh #include <asm/syscalls.h> 21250c2277SThomas Gleixner 22dfb09f9bSBorislav Petkov /* 23dfb09f9bSBorislav Petkov * Align a virtual address to avoid aliasing in the I$ on AMD F15h. 24dfb09f9bSBorislav Petkov */ 25f9902472SMichel Lespinasse static unsigned long get_align_mask(void) 26dfb09f9bSBorislav Petkov { 27dfb09f9bSBorislav Petkov /* handle 32- and 64-bit case with a single conditional */ 28dfb09f9bSBorislav Petkov if (va_align.flags < 0 || !(va_align.flags & (2 - mmap_is_ia32()))) 29f9902472SMichel Lespinasse return 0; 30dfb09f9bSBorislav Petkov 31dfb09f9bSBorislav Petkov if (!(current->flags & PF_RANDOMIZE)) 32f9902472SMichel Lespinasse return 0; 33dfb09f9bSBorislav Petkov 34f9902472SMichel Lespinasse return va_align.mask; 35f9902472SMichel Lespinasse } 36dfb09f9bSBorislav Petkov 374e26d11fSHector Marco-Gisbert /* 384e26d11fSHector Marco-Gisbert * To avoid aliasing in the I$ on AMD F15h, the bits defined by the 394e26d11fSHector Marco-Gisbert * va_align.bits, [12:upper_bit), are set to a random value instead of 404e26d11fSHector Marco-Gisbert * zeroing them. This random value is computed once per boot. This form 414e26d11fSHector Marco-Gisbert * of ASLR is known as "per-boot ASLR". 424e26d11fSHector Marco-Gisbert * 434e26d11fSHector Marco-Gisbert * To achieve this, the random value is added to the info.align_offset 444e26d11fSHector Marco-Gisbert * value before calling vm_unmapped_area() or ORed directly to the 454e26d11fSHector Marco-Gisbert * address. 464e26d11fSHector Marco-Gisbert */ 474e26d11fSHector Marco-Gisbert static unsigned long get_align_bits(void) 484e26d11fSHector Marco-Gisbert { 494e26d11fSHector Marco-Gisbert return va_align.bits & get_align_mask(); 504e26d11fSHector Marco-Gisbert } 514e26d11fSHector Marco-Gisbert 52f9902472SMichel Lespinasse unsigned long align_vdso_addr(unsigned long addr) 53f9902472SMichel Lespinasse { 54f9902472SMichel Lespinasse unsigned long align_mask = get_align_mask(); 554e26d11fSHector Marco-Gisbert addr = (addr + align_mask) & ~align_mask; 564e26d11fSHector Marco-Gisbert return addr | get_align_bits(); 57dfb09f9bSBorislav Petkov } 58dfb09f9bSBorislav Petkov 59dfb09f9bSBorislav Petkov static int __init control_va_addr_alignment(char *str) 60dfb09f9bSBorislav Petkov { 61dfb09f9bSBorislav Petkov /* guard against enabling this on other CPU families */ 62dfb09f9bSBorislav Petkov if (va_align.flags < 0) 63dfb09f9bSBorislav Petkov return 1; 64dfb09f9bSBorislav Petkov 65dfb09f9bSBorislav Petkov if (*str == 0) 66dfb09f9bSBorislav Petkov return 1; 67dfb09f9bSBorislav Petkov 68dfb09f9bSBorislav Petkov if (*str == '=') 69dfb09f9bSBorislav Petkov str++; 70dfb09f9bSBorislav Petkov 71dfb09f9bSBorislav Petkov if (!strcmp(str, "32")) 72dfb09f9bSBorislav Petkov va_align.flags = ALIGN_VA_32; 73dfb09f9bSBorislav Petkov else if (!strcmp(str, "64")) 74dfb09f9bSBorislav Petkov va_align.flags = ALIGN_VA_64; 75dfb09f9bSBorislav Petkov else if (!strcmp(str, "off")) 76dfb09f9bSBorislav Petkov va_align.flags = 0; 77dfb09f9bSBorislav Petkov else if (!strcmp(str, "on")) 78dfb09f9bSBorislav Petkov va_align.flags = ALIGN_VA_32 | ALIGN_VA_64; 79dfb09f9bSBorislav Petkov else 80dfb09f9bSBorislav Petkov return 0; 81dfb09f9bSBorislav Petkov 82dfb09f9bSBorislav Petkov return 1; 83dfb09f9bSBorislav Petkov } 84dfb09f9bSBorislav Petkov __setup("align_va_addr", control_va_addr_alignment); 85dfb09f9bSBorislav Petkov 860ac676fbSJason Baron SYSCALL_DEFINE6(mmap, unsigned long, addr, unsigned long, len, 870ac676fbSJason Baron unsigned long, prot, unsigned long, flags, 880ac676fbSJason Baron unsigned long, fd, unsigned long, off) 89250c2277SThomas Gleixner { 90250c2277SThomas Gleixner long error; 91250c2277SThomas Gleixner error = -EINVAL; 92250c2277SThomas Gleixner if (off & ~PAGE_MASK) 93250c2277SThomas Gleixner goto out; 94250c2277SThomas Gleixner 95f8b72560SAl Viro error = sys_mmap_pgoff(addr, len, prot, flags, fd, off >> PAGE_SHIFT); 96250c2277SThomas Gleixner out: 97250c2277SThomas Gleixner return error; 98250c2277SThomas Gleixner } 99250c2277SThomas Gleixner 100250c2277SThomas Gleixner static void find_start_end(unsigned long flags, unsigned long *begin, 101250c2277SThomas Gleixner unsigned long *end) 102250c2277SThomas Gleixner { 1036bd33008SH. Peter Anvin if (!test_thread_flag(TIF_ADDR32) && (flags & MAP_32BIT)) { 104cc503c1bSJiri Kosina unsigned long new_begin; 105250c2277SThomas Gleixner /* This is usually used needed to map code in small 106250c2277SThomas Gleixner model, so it needs to be in the first 31bit. Limit 107250c2277SThomas Gleixner it to that. This means we need to move the 108250c2277SThomas Gleixner unmapped base down for this case. This can give 109250c2277SThomas Gleixner conflicts with the heap, but we assume that glibc 110250c2277SThomas Gleixner malloc knows how to fall back to mmap. Give it 1GB 111250c2277SThomas Gleixner of playground for now. -AK */ 112250c2277SThomas Gleixner *begin = 0x40000000; 113250c2277SThomas Gleixner *end = 0x80000000; 114cc503c1bSJiri Kosina if (current->flags & PF_RANDOMIZE) { 115cc503c1bSJiri Kosina new_begin = randomize_range(*begin, *begin + 0x02000000, 0); 116cc503c1bSJiri Kosina if (new_begin) 117cc503c1bSJiri Kosina *begin = new_begin; 118cc503c1bSJiri Kosina } 119250c2277SThomas Gleixner } else { 12041aacc1eSRadu Caragea *begin = current->mm->mmap_legacy_base; 121250c2277SThomas Gleixner *end = TASK_SIZE; 122250c2277SThomas Gleixner } 123250c2277SThomas Gleixner } 124250c2277SThomas Gleixner 125250c2277SThomas Gleixner unsigned long 126250c2277SThomas Gleixner arch_get_unmapped_area(struct file *filp, unsigned long addr, 127250c2277SThomas Gleixner unsigned long len, unsigned long pgoff, unsigned long flags) 128250c2277SThomas Gleixner { 129250c2277SThomas Gleixner struct mm_struct *mm = current->mm; 130250c2277SThomas Gleixner struct vm_area_struct *vma; 131f9902472SMichel Lespinasse struct vm_unmapped_area_info info; 132250c2277SThomas Gleixner unsigned long begin, end; 133250c2277SThomas Gleixner 134250c2277SThomas Gleixner if (flags & MAP_FIXED) 135250c2277SThomas Gleixner return addr; 136250c2277SThomas Gleixner 137250c2277SThomas Gleixner find_start_end(flags, &begin, &end); 138250c2277SThomas Gleixner 139250c2277SThomas Gleixner if (len > end) 140250c2277SThomas Gleixner return -ENOMEM; 141250c2277SThomas Gleixner 142250c2277SThomas Gleixner if (addr) { 143250c2277SThomas Gleixner addr = PAGE_ALIGN(addr); 144250c2277SThomas Gleixner vma = find_vma(mm, addr); 145250c2277SThomas Gleixner if (end - len >= addr && 146250c2277SThomas Gleixner (!vma || addr + len <= vma->vm_start)) 147250c2277SThomas Gleixner return addr; 148250c2277SThomas Gleixner } 149250c2277SThomas Gleixner 150f9902472SMichel Lespinasse info.flags = 0; 151f9902472SMichel Lespinasse info.length = len; 152f9902472SMichel Lespinasse info.low_limit = begin; 153f9902472SMichel Lespinasse info.high_limit = end; 1544e26d11fSHector Marco-Gisbert info.align_mask = 0; 1557d025059SMichel Lespinasse info.align_offset = pgoff << PAGE_SHIFT; 1564e26d11fSHector Marco-Gisbert if (filp) { 1574e26d11fSHector Marco-Gisbert info.align_mask = get_align_mask(); 1584e26d11fSHector Marco-Gisbert info.align_offset += get_align_bits(); 1594e26d11fSHector Marco-Gisbert } 160f9902472SMichel Lespinasse return vm_unmapped_area(&info); 161250c2277SThomas Gleixner } 162cc503c1bSJiri Kosina 163cc503c1bSJiri Kosina unsigned long 164cc503c1bSJiri Kosina arch_get_unmapped_area_topdown(struct file *filp, const unsigned long addr0, 165cc503c1bSJiri Kosina const unsigned long len, const unsigned long pgoff, 166cc503c1bSJiri Kosina const unsigned long flags) 167cc503c1bSJiri Kosina { 168cc503c1bSJiri Kosina struct vm_area_struct *vma; 169cc503c1bSJiri Kosina struct mm_struct *mm = current->mm; 170f9902472SMichel Lespinasse unsigned long addr = addr0; 171f9902472SMichel Lespinasse struct vm_unmapped_area_info info; 172cc503c1bSJiri Kosina 173cc503c1bSJiri Kosina /* requested length too big for entire address space */ 174cc503c1bSJiri Kosina if (len > TASK_SIZE) 175cc503c1bSJiri Kosina return -ENOMEM; 176cc503c1bSJiri Kosina 177cc503c1bSJiri Kosina if (flags & MAP_FIXED) 178cc503c1bSJiri Kosina return addr; 179cc503c1bSJiri Kosina 180e3e81acaSYuanhan Liu /* for MAP_32BIT mappings we force the legacy mmap base */ 1816bd33008SH. Peter Anvin if (!test_thread_flag(TIF_ADDR32) && (flags & MAP_32BIT)) 182cc503c1bSJiri Kosina goto bottomup; 183cc503c1bSJiri Kosina 184cc503c1bSJiri Kosina /* requesting a specific address */ 185cc503c1bSJiri Kosina if (addr) { 186cc503c1bSJiri Kosina addr = PAGE_ALIGN(addr); 187cc503c1bSJiri Kosina vma = find_vma(mm, addr); 188cc503c1bSJiri Kosina if (TASK_SIZE - len >= addr && 189cc503c1bSJiri Kosina (!vma || addr + len <= vma->vm_start)) 190cc503c1bSJiri Kosina return addr; 191cc503c1bSJiri Kosina } 192cc503c1bSJiri Kosina 193f9902472SMichel Lespinasse info.flags = VM_UNMAPPED_AREA_TOPDOWN; 194f9902472SMichel Lespinasse info.length = len; 195f9902472SMichel Lespinasse info.low_limit = PAGE_SIZE; 196f9902472SMichel Lespinasse info.high_limit = mm->mmap_base; 1974e26d11fSHector Marco-Gisbert info.align_mask = 0; 1987d025059SMichel Lespinasse info.align_offset = pgoff << PAGE_SHIFT; 1994e26d11fSHector Marco-Gisbert if (filp) { 2004e26d11fSHector Marco-Gisbert info.align_mask = get_align_mask(); 2014e26d11fSHector Marco-Gisbert info.align_offset += get_align_bits(); 2024e26d11fSHector Marco-Gisbert } 203f9902472SMichel Lespinasse addr = vm_unmapped_area(&info); 204f9902472SMichel Lespinasse if (!(addr & ~PAGE_MASK)) 205f9902472SMichel Lespinasse return addr; 206f9902472SMichel Lespinasse VM_BUG_ON(addr != -ENOMEM); 207b716ad95SXiao Guangrong 208cc503c1bSJiri Kosina bottomup: 209cc503c1bSJiri Kosina /* 210cc503c1bSJiri Kosina * A failed mmap() very likely causes application failure, 211cc503c1bSJiri Kosina * so fall back to the bottom-up function here. This scenario 212cc503c1bSJiri Kosina * can happen with large stack limits and large mmap() 213cc503c1bSJiri Kosina * allocations. 214cc503c1bSJiri Kosina */ 215f9902472SMichel Lespinasse return arch_get_unmapped_area(filp, addr0, len, pgoff, flags); 216cc503c1bSJiri Kosina } 217