1 /* 2 * mm/mprotect.c 3 * 4 * (C) Copyright 1994 Linus Torvalds 5 * (C) Copyright 2002 Christoph Hellwig 6 * 7 * Address space accounting code <alan@lxorguk.ukuu.org.uk> 8 * (C) Copyright 2002 Red Hat Inc, All Rights Reserved 9 */ 10 11 #include <linux/mm.h> 12 #include <linux/hugetlb.h> 13 #include <linux/shm.h> 14 #include <linux/mman.h> 15 #include <linux/fs.h> 16 #include <linux/highmem.h> 17 #include <linux/security.h> 18 #include <linux/mempolicy.h> 19 #include <linux/personality.h> 20 #include <linux/syscalls.h> 21 #include <linux/swap.h> 22 #include <linux/swapops.h> 23 #include <linux/mmu_notifier.h> 24 #include <linux/migrate.h> 25 #include <linux/perf_event.h> 26 #include <asm/uaccess.h> 27 #include <asm/pgtable.h> 28 #include <asm/cacheflush.h> 29 #include <asm/tlbflush.h> 30 31 #ifndef pgprot_modify 32 static inline pgprot_t pgprot_modify(pgprot_t oldprot, pgprot_t newprot) 33 { 34 return newprot; 35 } 36 #endif 37 38 static unsigned long change_pte_range(struct vm_area_struct *vma, pmd_t *pmd, 39 unsigned long addr, unsigned long end, pgprot_t newprot, 40 int dirty_accountable, int prot_numa) 41 { 42 struct mm_struct *mm = vma->vm_mm; 43 pte_t *pte, oldpte; 44 spinlock_t *ptl; 45 unsigned long pages = 0; 46 47 pte = pte_offset_map_lock(mm, pmd, addr, &ptl); 48 arch_enter_lazy_mmu_mode(); 49 do { 50 oldpte = *pte; 51 if (pte_present(oldpte)) { 52 pte_t ptent; 53 bool updated = false; 54 55 ptent = ptep_modify_prot_start(mm, addr, pte); 56 if (!prot_numa) { 57 ptent = pte_modify(ptent, newprot); 58 updated = true; 59 } else { 60 struct page *page; 61 62 page = vm_normal_page(vma, addr, oldpte); 63 if (page) { 64 if (!pte_numa(oldpte)) { 65 ptent = pte_mknuma(ptent); 66 updated = true; 67 } 68 } 69 } 70 71 /* 72 * Avoid taking write faults for pages we know to be 73 * dirty. 74 */ 75 if (dirty_accountable && pte_dirty(ptent)) { 76 ptent = pte_mkwrite(ptent); 77 updated = true; 78 } 79 80 if (updated) 81 pages++; 82 ptep_modify_prot_commit(mm, addr, pte, ptent); 83 } else if (IS_ENABLED(CONFIG_MIGRATION) && !pte_file(oldpte)) { 84 swp_entry_t entry = pte_to_swp_entry(oldpte); 85 86 if (is_write_migration_entry(entry)) { 87 pte_t newpte; 88 /* 89 * A protection check is difficult so 90 * just be safe and disable write 91 */ 92 make_migration_entry_read(&entry); 93 newpte = swp_entry_to_pte(entry); 94 if (pte_swp_soft_dirty(oldpte)) 95 newpte = pte_swp_mksoft_dirty(newpte); 96 set_pte_at(mm, addr, pte, newpte); 97 98 pages++; 99 } 100 } 101 } while (pte++, addr += PAGE_SIZE, addr != end); 102 arch_leave_lazy_mmu_mode(); 103 pte_unmap_unlock(pte - 1, ptl); 104 105 return pages; 106 } 107 108 static inline unsigned long change_pmd_range(struct vm_area_struct *vma, 109 pud_t *pud, unsigned long addr, unsigned long end, 110 pgprot_t newprot, int dirty_accountable, int prot_numa) 111 { 112 pmd_t *pmd; 113 unsigned long next; 114 unsigned long pages = 0; 115 unsigned long nr_huge_updates = 0; 116 117 pmd = pmd_offset(pud, addr); 118 do { 119 unsigned long this_pages; 120 121 next = pmd_addr_end(addr, end); 122 if (pmd_trans_huge(*pmd)) { 123 if (next - addr != HPAGE_PMD_SIZE) 124 split_huge_page_pmd(vma, addr, pmd); 125 else { 126 int nr_ptes = change_huge_pmd(vma, pmd, addr, 127 newprot, prot_numa); 128 129 if (nr_ptes) { 130 if (nr_ptes == HPAGE_PMD_NR) { 131 pages += HPAGE_PMD_NR; 132 nr_huge_updates++; 133 } 134 continue; 135 } 136 } 137 /* fall through */ 138 } 139 if (pmd_none_or_clear_bad(pmd)) 140 continue; 141 this_pages = change_pte_range(vma, pmd, addr, next, newprot, 142 dirty_accountable, prot_numa); 143 pages += this_pages; 144 } while (pmd++, addr = next, addr != end); 145 146 if (nr_huge_updates) 147 count_vm_numa_events(NUMA_HUGE_PTE_UPDATES, nr_huge_updates); 148 return pages; 149 } 150 151 static inline unsigned long change_pud_range(struct vm_area_struct *vma, 152 pgd_t *pgd, unsigned long addr, unsigned long end, 153 pgprot_t newprot, int dirty_accountable, int prot_numa) 154 { 155 pud_t *pud; 156 unsigned long next; 157 unsigned long pages = 0; 158 159 pud = pud_offset(pgd, addr); 160 do { 161 next = pud_addr_end(addr, end); 162 if (pud_none_or_clear_bad(pud)) 163 continue; 164 pages += change_pmd_range(vma, pud, addr, next, newprot, 165 dirty_accountable, prot_numa); 166 } while (pud++, addr = next, addr != end); 167 168 return pages; 169 } 170 171 static unsigned long change_protection_range(struct vm_area_struct *vma, 172 unsigned long addr, unsigned long end, pgprot_t newprot, 173 int dirty_accountable, int prot_numa) 174 { 175 struct mm_struct *mm = vma->vm_mm; 176 pgd_t *pgd; 177 unsigned long next; 178 unsigned long start = addr; 179 unsigned long pages = 0; 180 181 BUG_ON(addr >= end); 182 pgd = pgd_offset(mm, addr); 183 flush_cache_range(vma, addr, end); 184 do { 185 next = pgd_addr_end(addr, end); 186 if (pgd_none_or_clear_bad(pgd)) 187 continue; 188 pages += change_pud_range(vma, pgd, addr, next, newprot, 189 dirty_accountable, prot_numa); 190 } while (pgd++, addr = next, addr != end); 191 192 /* Only flush the TLB if we actually modified any entries: */ 193 if (pages) 194 flush_tlb_range(vma, start, end); 195 196 return pages; 197 } 198 199 unsigned long change_protection(struct vm_area_struct *vma, unsigned long start, 200 unsigned long end, pgprot_t newprot, 201 int dirty_accountable, int prot_numa) 202 { 203 struct mm_struct *mm = vma->vm_mm; 204 unsigned long pages; 205 206 mmu_notifier_invalidate_range_start(mm, start, end); 207 if (is_vm_hugetlb_page(vma)) 208 pages = hugetlb_change_protection(vma, start, end, newprot); 209 else 210 pages = change_protection_range(vma, start, end, newprot, dirty_accountable, prot_numa); 211 mmu_notifier_invalidate_range_end(mm, start, end); 212 213 return pages; 214 } 215 216 int 217 mprotect_fixup(struct vm_area_struct *vma, struct vm_area_struct **pprev, 218 unsigned long start, unsigned long end, unsigned long newflags) 219 { 220 struct mm_struct *mm = vma->vm_mm; 221 unsigned long oldflags = vma->vm_flags; 222 long nrpages = (end - start) >> PAGE_SHIFT; 223 unsigned long charged = 0; 224 pgoff_t pgoff; 225 int error; 226 int dirty_accountable = 0; 227 228 if (newflags == oldflags) { 229 *pprev = vma; 230 return 0; 231 } 232 233 /* 234 * If we make a private mapping writable we increase our commit; 235 * but (without finer accounting) cannot reduce our commit if we 236 * make it unwritable again. hugetlb mapping were accounted for 237 * even if read-only so there is no need to account for them here 238 */ 239 if (newflags & VM_WRITE) { 240 if (!(oldflags & (VM_ACCOUNT|VM_WRITE|VM_HUGETLB| 241 VM_SHARED|VM_NORESERVE))) { 242 charged = nrpages; 243 if (security_vm_enough_memory_mm(mm, charged)) 244 return -ENOMEM; 245 newflags |= VM_ACCOUNT; 246 } 247 } 248 249 /* 250 * First try to merge with previous and/or next vma. 251 */ 252 pgoff = vma->vm_pgoff + ((start - vma->vm_start) >> PAGE_SHIFT); 253 *pprev = vma_merge(mm, *pprev, start, end, newflags, 254 vma->anon_vma, vma->vm_file, pgoff, vma_policy(vma)); 255 if (*pprev) { 256 vma = *pprev; 257 goto success; 258 } 259 260 *pprev = vma; 261 262 if (start != vma->vm_start) { 263 error = split_vma(mm, vma, start, 1); 264 if (error) 265 goto fail; 266 } 267 268 if (end != vma->vm_end) { 269 error = split_vma(mm, vma, end, 0); 270 if (error) 271 goto fail; 272 } 273 274 success: 275 /* 276 * vm_flags and vm_page_prot are protected by the mmap_sem 277 * held in write mode. 278 */ 279 vma->vm_flags = newflags; 280 vma->vm_page_prot = pgprot_modify(vma->vm_page_prot, 281 vm_get_page_prot(newflags)); 282 283 if (vma_wants_writenotify(vma)) { 284 vma->vm_page_prot = vm_get_page_prot(newflags & ~VM_SHARED); 285 dirty_accountable = 1; 286 } 287 288 change_protection(vma, start, end, vma->vm_page_prot, 289 dirty_accountable, 0); 290 291 vm_stat_account(mm, oldflags, vma->vm_file, -nrpages); 292 vm_stat_account(mm, newflags, vma->vm_file, nrpages); 293 perf_event_mmap(vma); 294 return 0; 295 296 fail: 297 vm_unacct_memory(charged); 298 return error; 299 } 300 301 SYSCALL_DEFINE3(mprotect, unsigned long, start, size_t, len, 302 unsigned long, prot) 303 { 304 unsigned long vm_flags, nstart, end, tmp, reqprot; 305 struct vm_area_struct *vma, *prev; 306 int error = -EINVAL; 307 const int grows = prot & (PROT_GROWSDOWN|PROT_GROWSUP); 308 prot &= ~(PROT_GROWSDOWN|PROT_GROWSUP); 309 if (grows == (PROT_GROWSDOWN|PROT_GROWSUP)) /* can't be both */ 310 return -EINVAL; 311 312 if (start & ~PAGE_MASK) 313 return -EINVAL; 314 if (!len) 315 return 0; 316 len = PAGE_ALIGN(len); 317 end = start + len; 318 if (end <= start) 319 return -ENOMEM; 320 if (!arch_validate_prot(prot)) 321 return -EINVAL; 322 323 reqprot = prot; 324 /* 325 * Does the application expect PROT_READ to imply PROT_EXEC: 326 */ 327 if ((prot & PROT_READ) && (current->personality & READ_IMPLIES_EXEC)) 328 prot |= PROT_EXEC; 329 330 vm_flags = calc_vm_prot_bits(prot); 331 332 down_write(¤t->mm->mmap_sem); 333 334 vma = find_vma(current->mm, start); 335 error = -ENOMEM; 336 if (!vma) 337 goto out; 338 prev = vma->vm_prev; 339 if (unlikely(grows & PROT_GROWSDOWN)) { 340 if (vma->vm_start >= end) 341 goto out; 342 start = vma->vm_start; 343 error = -EINVAL; 344 if (!(vma->vm_flags & VM_GROWSDOWN)) 345 goto out; 346 } else { 347 if (vma->vm_start > start) 348 goto out; 349 if (unlikely(grows & PROT_GROWSUP)) { 350 end = vma->vm_end; 351 error = -EINVAL; 352 if (!(vma->vm_flags & VM_GROWSUP)) 353 goto out; 354 } 355 } 356 if (start > vma->vm_start) 357 prev = vma; 358 359 for (nstart = start ; ; ) { 360 unsigned long newflags; 361 362 /* Here we know that vma->vm_start <= nstart < vma->vm_end. */ 363 364 newflags = vm_flags; 365 newflags |= (vma->vm_flags & ~(VM_READ | VM_WRITE | VM_EXEC)); 366 367 /* newflags >> 4 shift VM_MAY% in place of VM_% */ 368 if ((newflags & ~(newflags >> 4)) & (VM_READ | VM_WRITE | VM_EXEC)) { 369 error = -EACCES; 370 goto out; 371 } 372 373 error = security_file_mprotect(vma, reqprot, prot); 374 if (error) 375 goto out; 376 377 tmp = vma->vm_end; 378 if (tmp > end) 379 tmp = end; 380 error = mprotect_fixup(vma, &prev, nstart, tmp, newflags); 381 if (error) 382 goto out; 383 nstart = tmp; 384 385 if (nstart < prev->vm_end) 386 nstart = prev->vm_end; 387 if (nstart >= end) 388 goto out; 389 390 vma = prev->vm_next; 391 if (!vma || vma->vm_start != nstart) { 392 error = -ENOMEM; 393 goto out; 394 } 395 } 396 out: 397 up_write(¤t->mm->mmap_sem); 398 return error; 399 } 400