1 /* 2 * mm/mprotect.c 3 * 4 * (C) Copyright 1994 Linus Torvalds 5 * (C) Copyright 2002 Christoph Hellwig 6 * 7 * Address space accounting code <alan@lxorguk.ukuu.org.uk> 8 * (C) Copyright 2002 Red Hat Inc, All Rights Reserved 9 */ 10 11 #include <linux/mm.h> 12 #include <linux/hugetlb.h> 13 #include <linux/shm.h> 14 #include <linux/mman.h> 15 #include <linux/fs.h> 16 #include <linux/highmem.h> 17 #include <linux/security.h> 18 #include <linux/mempolicy.h> 19 #include <linux/personality.h> 20 #include <linux/syscalls.h> 21 #include <linux/swap.h> 22 #include <linux/swapops.h> 23 #include <linux/mmu_notifier.h> 24 #include <linux/migrate.h> 25 #include <linux/perf_event.h> 26 #include <linux/ksm.h> 27 #include <asm/uaccess.h> 28 #include <asm/pgtable.h> 29 #include <asm/cacheflush.h> 30 #include <asm/tlbflush.h> 31 32 #include "internal.h" 33 34 /* 35 * For a prot_numa update we only hold mmap_sem for read so there is a 36 * potential race with faulting where a pmd was temporarily none. This 37 * function checks for a transhuge pmd under the appropriate lock. It 38 * returns a pte if it was successfully locked or NULL if it raced with 39 * a transhuge insertion. 40 */ 41 static pte_t *lock_pte_protection(struct vm_area_struct *vma, pmd_t *pmd, 42 unsigned long addr, int prot_numa, spinlock_t **ptl) 43 { 44 pte_t *pte; 45 spinlock_t *pmdl; 46 47 /* !prot_numa is protected by mmap_sem held for write */ 48 if (!prot_numa) 49 return pte_offset_map_lock(vma->vm_mm, pmd, addr, ptl); 50 51 pmdl = pmd_lock(vma->vm_mm, pmd); 52 if (unlikely(pmd_trans_huge(*pmd) || pmd_none(*pmd))) { 53 spin_unlock(pmdl); 54 return NULL; 55 } 56 57 pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, ptl); 58 spin_unlock(pmdl); 59 return pte; 60 } 61 62 static unsigned long change_pte_range(struct vm_area_struct *vma, pmd_t *pmd, 63 unsigned long addr, unsigned long end, pgprot_t newprot, 64 int dirty_accountable, int prot_numa) 65 { 66 struct mm_struct *mm = vma->vm_mm; 67 pte_t *pte, oldpte; 68 spinlock_t *ptl; 69 unsigned long pages = 0; 70 71 pte = lock_pte_protection(vma, pmd, addr, prot_numa, &ptl); 72 if (!pte) 73 return 0; 74 75 arch_enter_lazy_mmu_mode(); 76 do { 77 oldpte = *pte; 78 if (pte_present(oldpte)) { 79 pte_t ptent; 80 bool preserve_write = prot_numa && pte_write(oldpte); 81 82 /* 83 * Avoid trapping faults against the zero or KSM 84 * pages. See similar comment in change_huge_pmd. 85 */ 86 if (prot_numa) { 87 struct page *page; 88 89 page = vm_normal_page(vma, addr, oldpte); 90 if (!page || PageKsm(page)) 91 continue; 92 93 /* Avoid TLB flush if possible */ 94 if (pte_protnone(oldpte)) 95 continue; 96 } 97 98 ptent = ptep_modify_prot_start(mm, addr, pte); 99 ptent = pte_modify(ptent, newprot); 100 if (preserve_write) 101 ptent = pte_mkwrite(ptent); 102 103 /* Avoid taking write faults for known dirty pages */ 104 if (dirty_accountable && pte_dirty(ptent) && 105 (pte_soft_dirty(ptent) || 106 !(vma->vm_flags & VM_SOFTDIRTY))) { 107 ptent = pte_mkwrite(ptent); 108 } 109 ptep_modify_prot_commit(mm, addr, pte, ptent); 110 pages++; 111 } else if (IS_ENABLED(CONFIG_MIGRATION)) { 112 swp_entry_t entry = pte_to_swp_entry(oldpte); 113 114 if (is_write_migration_entry(entry)) { 115 pte_t newpte; 116 /* 117 * A protection check is difficult so 118 * just be safe and disable write 119 */ 120 make_migration_entry_read(&entry); 121 newpte = swp_entry_to_pte(entry); 122 if (pte_swp_soft_dirty(oldpte)) 123 newpte = pte_swp_mksoft_dirty(newpte); 124 set_pte_at(mm, addr, pte, newpte); 125 126 pages++; 127 } 128 } 129 } while (pte++, addr += PAGE_SIZE, addr != end); 130 arch_leave_lazy_mmu_mode(); 131 pte_unmap_unlock(pte - 1, ptl); 132 133 return pages; 134 } 135 136 static inline unsigned long change_pmd_range(struct vm_area_struct *vma, 137 pud_t *pud, unsigned long addr, unsigned long end, 138 pgprot_t newprot, int dirty_accountable, int prot_numa) 139 { 140 pmd_t *pmd; 141 struct mm_struct *mm = vma->vm_mm; 142 unsigned long next; 143 unsigned long pages = 0; 144 unsigned long nr_huge_updates = 0; 145 unsigned long mni_start = 0; 146 147 pmd = pmd_offset(pud, addr); 148 do { 149 unsigned long this_pages; 150 151 next = pmd_addr_end(addr, end); 152 if (!pmd_trans_huge(*pmd) && !pmd_devmap(*pmd) 153 && pmd_none_or_clear_bad(pmd)) 154 continue; 155 156 /* invoke the mmu notifier if the pmd is populated */ 157 if (!mni_start) { 158 mni_start = addr; 159 mmu_notifier_invalidate_range_start(mm, mni_start, end); 160 } 161 162 if (pmd_trans_huge(*pmd) || pmd_devmap(*pmd)) { 163 if (next - addr != HPAGE_PMD_SIZE) { 164 split_huge_pmd(vma, pmd, addr); 165 if (pmd_none(*pmd)) 166 continue; 167 } else { 168 int nr_ptes = change_huge_pmd(vma, pmd, addr, 169 newprot, prot_numa); 170 171 if (nr_ptes) { 172 if (nr_ptes == HPAGE_PMD_NR) { 173 pages += HPAGE_PMD_NR; 174 nr_huge_updates++; 175 } 176 177 /* huge pmd was handled */ 178 continue; 179 } 180 } 181 /* fall through, the trans huge pmd just split */ 182 } 183 this_pages = change_pte_range(vma, pmd, addr, next, newprot, 184 dirty_accountable, prot_numa); 185 pages += this_pages; 186 } while (pmd++, addr = next, addr != end); 187 188 if (mni_start) 189 mmu_notifier_invalidate_range_end(mm, mni_start, end); 190 191 if (nr_huge_updates) 192 count_vm_numa_events(NUMA_HUGE_PTE_UPDATES, nr_huge_updates); 193 return pages; 194 } 195 196 static inline unsigned long change_pud_range(struct vm_area_struct *vma, 197 pgd_t *pgd, unsigned long addr, unsigned long end, 198 pgprot_t newprot, int dirty_accountable, int prot_numa) 199 { 200 pud_t *pud; 201 unsigned long next; 202 unsigned long pages = 0; 203 204 pud = pud_offset(pgd, addr); 205 do { 206 next = pud_addr_end(addr, end); 207 if (pud_none_or_clear_bad(pud)) 208 continue; 209 pages += change_pmd_range(vma, pud, addr, next, newprot, 210 dirty_accountable, prot_numa); 211 } while (pud++, addr = next, addr != end); 212 213 return pages; 214 } 215 216 static unsigned long change_protection_range(struct vm_area_struct *vma, 217 unsigned long addr, unsigned long end, pgprot_t newprot, 218 int dirty_accountable, int prot_numa) 219 { 220 struct mm_struct *mm = vma->vm_mm; 221 pgd_t *pgd; 222 unsigned long next; 223 unsigned long start = addr; 224 unsigned long pages = 0; 225 226 BUG_ON(addr >= end); 227 pgd = pgd_offset(mm, addr); 228 flush_cache_range(vma, addr, end); 229 set_tlb_flush_pending(mm); 230 do { 231 next = pgd_addr_end(addr, end); 232 if (pgd_none_or_clear_bad(pgd)) 233 continue; 234 pages += change_pud_range(vma, pgd, addr, next, newprot, 235 dirty_accountable, prot_numa); 236 } while (pgd++, addr = next, addr != end); 237 238 /* Only flush the TLB if we actually modified any entries: */ 239 if (pages) 240 flush_tlb_range(vma, start, end); 241 clear_tlb_flush_pending(mm); 242 243 return pages; 244 } 245 246 unsigned long change_protection(struct vm_area_struct *vma, unsigned long start, 247 unsigned long end, pgprot_t newprot, 248 int dirty_accountable, int prot_numa) 249 { 250 unsigned long pages; 251 252 if (is_vm_hugetlb_page(vma)) 253 pages = hugetlb_change_protection(vma, start, end, newprot); 254 else 255 pages = change_protection_range(vma, start, end, newprot, dirty_accountable, prot_numa); 256 257 return pages; 258 } 259 260 int 261 mprotect_fixup(struct vm_area_struct *vma, struct vm_area_struct **pprev, 262 unsigned long start, unsigned long end, unsigned long newflags) 263 { 264 struct mm_struct *mm = vma->vm_mm; 265 unsigned long oldflags = vma->vm_flags; 266 long nrpages = (end - start) >> PAGE_SHIFT; 267 unsigned long charged = 0; 268 pgoff_t pgoff; 269 int error; 270 int dirty_accountable = 0; 271 272 if (newflags == oldflags) { 273 *pprev = vma; 274 return 0; 275 } 276 277 /* 278 * If we make a private mapping writable we increase our commit; 279 * but (without finer accounting) cannot reduce our commit if we 280 * make it unwritable again. hugetlb mapping were accounted for 281 * even if read-only so there is no need to account for them here 282 */ 283 if (newflags & VM_WRITE) { 284 /* Check space limits when area turns into data. */ 285 if (!may_expand_vm(mm, newflags, nrpages) && 286 may_expand_vm(mm, oldflags, nrpages)) 287 return -ENOMEM; 288 if (!(oldflags & (VM_ACCOUNT|VM_WRITE|VM_HUGETLB| 289 VM_SHARED|VM_NORESERVE))) { 290 charged = nrpages; 291 if (security_vm_enough_memory_mm(mm, charged)) 292 return -ENOMEM; 293 newflags |= VM_ACCOUNT; 294 } 295 } 296 297 /* 298 * First try to merge with previous and/or next vma. 299 */ 300 pgoff = vma->vm_pgoff + ((start - vma->vm_start) >> PAGE_SHIFT); 301 *pprev = vma_merge(mm, *pprev, start, end, newflags, 302 vma->anon_vma, vma->vm_file, pgoff, vma_policy(vma), 303 vma->vm_userfaultfd_ctx); 304 if (*pprev) { 305 vma = *pprev; 306 goto success; 307 } 308 309 *pprev = vma; 310 311 if (start != vma->vm_start) { 312 error = split_vma(mm, vma, start, 1); 313 if (error) 314 goto fail; 315 } 316 317 if (end != vma->vm_end) { 318 error = split_vma(mm, vma, end, 0); 319 if (error) 320 goto fail; 321 } 322 323 success: 324 /* 325 * vm_flags and vm_page_prot are protected by the mmap_sem 326 * held in write mode. 327 */ 328 vma->vm_flags = newflags; 329 dirty_accountable = vma_wants_writenotify(vma); 330 vma_set_page_prot(vma); 331 332 change_protection(vma, start, end, vma->vm_page_prot, 333 dirty_accountable, 0); 334 335 /* 336 * Private VM_LOCKED VMA becoming writable: trigger COW to avoid major 337 * fault on access. 338 */ 339 if ((oldflags & (VM_WRITE | VM_SHARED | VM_LOCKED)) == VM_LOCKED && 340 (newflags & VM_WRITE)) { 341 populate_vma_page_range(vma, start, end, NULL); 342 } 343 344 vm_stat_account(mm, oldflags, -nrpages); 345 vm_stat_account(mm, newflags, nrpages); 346 perf_event_mmap(vma); 347 return 0; 348 349 fail: 350 vm_unacct_memory(charged); 351 return error; 352 } 353 354 SYSCALL_DEFINE3(mprotect, unsigned long, start, size_t, len, 355 unsigned long, prot) 356 { 357 unsigned long vm_flags, nstart, end, tmp, reqprot; 358 struct vm_area_struct *vma, *prev; 359 int error = -EINVAL; 360 const int grows = prot & (PROT_GROWSDOWN|PROT_GROWSUP); 361 prot &= ~(PROT_GROWSDOWN|PROT_GROWSUP); 362 if (grows == (PROT_GROWSDOWN|PROT_GROWSUP)) /* can't be both */ 363 return -EINVAL; 364 365 if (start & ~PAGE_MASK) 366 return -EINVAL; 367 if (!len) 368 return 0; 369 len = PAGE_ALIGN(len); 370 end = start + len; 371 if (end <= start) 372 return -ENOMEM; 373 if (!arch_validate_prot(prot)) 374 return -EINVAL; 375 376 reqprot = prot; 377 /* 378 * Does the application expect PROT_READ to imply PROT_EXEC: 379 */ 380 if ((prot & PROT_READ) && (current->personality & READ_IMPLIES_EXEC)) 381 prot |= PROT_EXEC; 382 383 vm_flags = calc_vm_prot_bits(prot); 384 385 down_write(¤t->mm->mmap_sem); 386 387 vma = find_vma(current->mm, start); 388 error = -ENOMEM; 389 if (!vma) 390 goto out; 391 prev = vma->vm_prev; 392 if (unlikely(grows & PROT_GROWSDOWN)) { 393 if (vma->vm_start >= end) 394 goto out; 395 start = vma->vm_start; 396 error = -EINVAL; 397 if (!(vma->vm_flags & VM_GROWSDOWN)) 398 goto out; 399 } else { 400 if (vma->vm_start > start) 401 goto out; 402 if (unlikely(grows & PROT_GROWSUP)) { 403 end = vma->vm_end; 404 error = -EINVAL; 405 if (!(vma->vm_flags & VM_GROWSUP)) 406 goto out; 407 } 408 } 409 if (start > vma->vm_start) 410 prev = vma; 411 412 for (nstart = start ; ; ) { 413 unsigned long newflags; 414 415 /* Here we know that vma->vm_start <= nstart < vma->vm_end. */ 416 417 newflags = vm_flags; 418 newflags |= (vma->vm_flags & ~(VM_READ | VM_WRITE | VM_EXEC)); 419 420 /* newflags >> 4 shift VM_MAY% in place of VM_% */ 421 if ((newflags & ~(newflags >> 4)) & (VM_READ | VM_WRITE | VM_EXEC)) { 422 error = -EACCES; 423 goto out; 424 } 425 426 error = security_file_mprotect(vma, reqprot, prot); 427 if (error) 428 goto out; 429 430 tmp = vma->vm_end; 431 if (tmp > end) 432 tmp = end; 433 error = mprotect_fixup(vma, &prev, nstart, tmp, newflags); 434 if (error) 435 goto out; 436 nstart = tmp; 437 438 if (nstart < prev->vm_end) 439 nstart = prev->vm_end; 440 if (nstart >= end) 441 goto out; 442 443 vma = prev->vm_next; 444 if (!vma || vma->vm_start != nstart) { 445 error = -ENOMEM; 446 goto out; 447 } 448 } 449 out: 450 up_write(¤t->mm->mmap_sem); 451 return error; 452 } 453