1 /* 2 * mm/mprotect.c 3 * 4 * (C) Copyright 1994 Linus Torvalds 5 * (C) Copyright 2002 Christoph Hellwig 6 * 7 * Address space accounting code <alan@lxorguk.ukuu.org.uk> 8 * (C) Copyright 2002 Red Hat Inc, All Rights Reserved 9 */ 10 11 #include <linux/mm.h> 12 #include <linux/hugetlb.h> 13 #include <linux/shm.h> 14 #include <linux/mman.h> 15 #include <linux/fs.h> 16 #include <linux/highmem.h> 17 #include <linux/security.h> 18 #include <linux/mempolicy.h> 19 #include <linux/personality.h> 20 #include <linux/syscalls.h> 21 #include <linux/swap.h> 22 #include <linux/swapops.h> 23 #include <linux/mmu_notifier.h> 24 #include <linux/migrate.h> 25 #include <linux/perf_event.h> 26 #include <asm/uaccess.h> 27 #include <asm/pgtable.h> 28 #include <asm/cacheflush.h> 29 #include <asm/tlbflush.h> 30 31 #ifndef pgprot_modify 32 static inline pgprot_t pgprot_modify(pgprot_t oldprot, pgprot_t newprot) 33 { 34 return newprot; 35 } 36 #endif 37 38 static unsigned long change_pte_range(struct vm_area_struct *vma, pmd_t *pmd, 39 unsigned long addr, unsigned long end, pgprot_t newprot, 40 int dirty_accountable, int prot_numa) 41 { 42 struct mm_struct *mm = vma->vm_mm; 43 pte_t *pte, oldpte; 44 spinlock_t *ptl; 45 unsigned long pages = 0; 46 47 pte = pte_offset_map_lock(mm, pmd, addr, &ptl); 48 arch_enter_lazy_mmu_mode(); 49 do { 50 oldpte = *pte; 51 if (pte_present(oldpte)) { 52 pte_t ptent; 53 bool updated = false; 54 55 if (!prot_numa) { 56 ptent = ptep_modify_prot_start(mm, addr, pte); 57 if (pte_numa(ptent)) 58 ptent = pte_mknonnuma(ptent); 59 ptent = pte_modify(ptent, newprot); 60 updated = true; 61 } else { 62 struct page *page; 63 64 ptent = *pte; 65 page = vm_normal_page(vma, addr, oldpte); 66 if (page) { 67 if (!pte_numa(oldpte)) { 68 ptent = pte_mknuma(ptent); 69 set_pte_at(mm, addr, pte, ptent); 70 updated = true; 71 } 72 } 73 } 74 75 /* 76 * Avoid taking write faults for pages we know to be 77 * dirty. 78 */ 79 if (dirty_accountable && pte_dirty(ptent)) { 80 ptent = pte_mkwrite(ptent); 81 updated = true; 82 } 83 84 if (updated) 85 pages++; 86 87 /* Only !prot_numa always clears the pte */ 88 if (!prot_numa) 89 ptep_modify_prot_commit(mm, addr, pte, ptent); 90 } else if (IS_ENABLED(CONFIG_MIGRATION) && !pte_file(oldpte)) { 91 swp_entry_t entry = pte_to_swp_entry(oldpte); 92 93 if (is_write_migration_entry(entry)) { 94 pte_t newpte; 95 /* 96 * A protection check is difficult so 97 * just be safe and disable write 98 */ 99 make_migration_entry_read(&entry); 100 newpte = swp_entry_to_pte(entry); 101 if (pte_swp_soft_dirty(oldpte)) 102 newpte = pte_swp_mksoft_dirty(newpte); 103 set_pte_at(mm, addr, pte, newpte); 104 105 pages++; 106 } 107 } 108 } while (pte++, addr += PAGE_SIZE, addr != end); 109 arch_leave_lazy_mmu_mode(); 110 pte_unmap_unlock(pte - 1, ptl); 111 112 return pages; 113 } 114 115 static inline unsigned long change_pmd_range(struct vm_area_struct *vma, 116 pud_t *pud, unsigned long addr, unsigned long end, 117 pgprot_t newprot, int dirty_accountable, int prot_numa) 118 { 119 pmd_t *pmd; 120 unsigned long next; 121 unsigned long pages = 0; 122 unsigned long nr_huge_updates = 0; 123 124 pmd = pmd_offset(pud, addr); 125 do { 126 unsigned long this_pages; 127 128 next = pmd_addr_end(addr, end); 129 if (pmd_trans_huge(*pmd)) { 130 if (next - addr != HPAGE_PMD_SIZE) 131 split_huge_page_pmd(vma, addr, pmd); 132 else { 133 int nr_ptes = change_huge_pmd(vma, pmd, addr, 134 newprot, prot_numa); 135 136 if (nr_ptes) { 137 if (nr_ptes == HPAGE_PMD_NR) { 138 pages += HPAGE_PMD_NR; 139 nr_huge_updates++; 140 } 141 continue; 142 } 143 } 144 /* fall through */ 145 } 146 if (pmd_none_or_clear_bad(pmd)) 147 continue; 148 this_pages = change_pte_range(vma, pmd, addr, next, newprot, 149 dirty_accountable, prot_numa); 150 pages += this_pages; 151 } while (pmd++, addr = next, addr != end); 152 153 if (nr_huge_updates) 154 count_vm_numa_events(NUMA_HUGE_PTE_UPDATES, nr_huge_updates); 155 return pages; 156 } 157 158 static inline unsigned long change_pud_range(struct vm_area_struct *vma, 159 pgd_t *pgd, unsigned long addr, unsigned long end, 160 pgprot_t newprot, int dirty_accountable, int prot_numa) 161 { 162 pud_t *pud; 163 unsigned long next; 164 unsigned long pages = 0; 165 166 pud = pud_offset(pgd, addr); 167 do { 168 next = pud_addr_end(addr, end); 169 if (pud_none_or_clear_bad(pud)) 170 continue; 171 pages += change_pmd_range(vma, pud, addr, next, newprot, 172 dirty_accountable, prot_numa); 173 } while (pud++, addr = next, addr != end); 174 175 return pages; 176 } 177 178 static unsigned long change_protection_range(struct vm_area_struct *vma, 179 unsigned long addr, unsigned long end, pgprot_t newprot, 180 int dirty_accountable, int prot_numa) 181 { 182 struct mm_struct *mm = vma->vm_mm; 183 pgd_t *pgd; 184 unsigned long next; 185 unsigned long start = addr; 186 unsigned long pages = 0; 187 188 BUG_ON(addr >= end); 189 pgd = pgd_offset(mm, addr); 190 flush_cache_range(vma, addr, end); 191 set_tlb_flush_pending(mm); 192 do { 193 next = pgd_addr_end(addr, end); 194 if (pgd_none_or_clear_bad(pgd)) 195 continue; 196 pages += change_pud_range(vma, pgd, addr, next, newprot, 197 dirty_accountable, prot_numa); 198 } while (pgd++, addr = next, addr != end); 199 200 /* Only flush the TLB if we actually modified any entries: */ 201 if (pages) 202 flush_tlb_range(vma, start, end); 203 clear_tlb_flush_pending(mm); 204 205 return pages; 206 } 207 208 unsigned long change_protection(struct vm_area_struct *vma, unsigned long start, 209 unsigned long end, pgprot_t newprot, 210 int dirty_accountable, int prot_numa) 211 { 212 struct mm_struct *mm = vma->vm_mm; 213 unsigned long pages; 214 215 mmu_notifier_invalidate_range_start(mm, start, end); 216 if (is_vm_hugetlb_page(vma)) 217 pages = hugetlb_change_protection(vma, start, end, newprot); 218 else 219 pages = change_protection_range(vma, start, end, newprot, dirty_accountable, prot_numa); 220 mmu_notifier_invalidate_range_end(mm, start, end); 221 222 return pages; 223 } 224 225 int 226 mprotect_fixup(struct vm_area_struct *vma, struct vm_area_struct **pprev, 227 unsigned long start, unsigned long end, unsigned long newflags) 228 { 229 struct mm_struct *mm = vma->vm_mm; 230 unsigned long oldflags = vma->vm_flags; 231 long nrpages = (end - start) >> PAGE_SHIFT; 232 unsigned long charged = 0; 233 pgoff_t pgoff; 234 int error; 235 int dirty_accountable = 0; 236 237 if (newflags == oldflags) { 238 *pprev = vma; 239 return 0; 240 } 241 242 /* 243 * If we make a private mapping writable we increase our commit; 244 * but (without finer accounting) cannot reduce our commit if we 245 * make it unwritable again. hugetlb mapping were accounted for 246 * even if read-only so there is no need to account for them here 247 */ 248 if (newflags & VM_WRITE) { 249 if (!(oldflags & (VM_ACCOUNT|VM_WRITE|VM_HUGETLB| 250 VM_SHARED|VM_NORESERVE))) { 251 charged = nrpages; 252 if (security_vm_enough_memory_mm(mm, charged)) 253 return -ENOMEM; 254 newflags |= VM_ACCOUNT; 255 } 256 } 257 258 /* 259 * First try to merge with previous and/or next vma. 260 */ 261 pgoff = vma->vm_pgoff + ((start - vma->vm_start) >> PAGE_SHIFT); 262 *pprev = vma_merge(mm, *pprev, start, end, newflags, 263 vma->anon_vma, vma->vm_file, pgoff, vma_policy(vma)); 264 if (*pprev) { 265 vma = *pprev; 266 goto success; 267 } 268 269 *pprev = vma; 270 271 if (start != vma->vm_start) { 272 error = split_vma(mm, vma, start, 1); 273 if (error) 274 goto fail; 275 } 276 277 if (end != vma->vm_end) { 278 error = split_vma(mm, vma, end, 0); 279 if (error) 280 goto fail; 281 } 282 283 success: 284 /* 285 * vm_flags and vm_page_prot are protected by the mmap_sem 286 * held in write mode. 287 */ 288 vma->vm_flags = newflags; 289 vma->vm_page_prot = pgprot_modify(vma->vm_page_prot, 290 vm_get_page_prot(newflags)); 291 292 if (vma_wants_writenotify(vma)) { 293 vma->vm_page_prot = vm_get_page_prot(newflags & ~VM_SHARED); 294 dirty_accountable = 1; 295 } 296 297 change_protection(vma, start, end, vma->vm_page_prot, 298 dirty_accountable, 0); 299 300 vm_stat_account(mm, oldflags, vma->vm_file, -nrpages); 301 vm_stat_account(mm, newflags, vma->vm_file, nrpages); 302 perf_event_mmap(vma); 303 return 0; 304 305 fail: 306 vm_unacct_memory(charged); 307 return error; 308 } 309 310 SYSCALL_DEFINE3(mprotect, unsigned long, start, size_t, len, 311 unsigned long, prot) 312 { 313 unsigned long vm_flags, nstart, end, tmp, reqprot; 314 struct vm_area_struct *vma, *prev; 315 int error = -EINVAL; 316 const int grows = prot & (PROT_GROWSDOWN|PROT_GROWSUP); 317 prot &= ~(PROT_GROWSDOWN|PROT_GROWSUP); 318 if (grows == (PROT_GROWSDOWN|PROT_GROWSUP)) /* can't be both */ 319 return -EINVAL; 320 321 if (start & ~PAGE_MASK) 322 return -EINVAL; 323 if (!len) 324 return 0; 325 len = PAGE_ALIGN(len); 326 end = start + len; 327 if (end <= start) 328 return -ENOMEM; 329 if (!arch_validate_prot(prot)) 330 return -EINVAL; 331 332 reqprot = prot; 333 /* 334 * Does the application expect PROT_READ to imply PROT_EXEC: 335 */ 336 if ((prot & PROT_READ) && (current->personality & READ_IMPLIES_EXEC)) 337 prot |= PROT_EXEC; 338 339 vm_flags = calc_vm_prot_bits(prot); 340 341 down_write(¤t->mm->mmap_sem); 342 343 vma = find_vma(current->mm, start); 344 error = -ENOMEM; 345 if (!vma) 346 goto out; 347 prev = vma->vm_prev; 348 if (unlikely(grows & PROT_GROWSDOWN)) { 349 if (vma->vm_start >= end) 350 goto out; 351 start = vma->vm_start; 352 error = -EINVAL; 353 if (!(vma->vm_flags & VM_GROWSDOWN)) 354 goto out; 355 } else { 356 if (vma->vm_start > start) 357 goto out; 358 if (unlikely(grows & PROT_GROWSUP)) { 359 end = vma->vm_end; 360 error = -EINVAL; 361 if (!(vma->vm_flags & VM_GROWSUP)) 362 goto out; 363 } 364 } 365 if (start > vma->vm_start) 366 prev = vma; 367 368 for (nstart = start ; ; ) { 369 unsigned long newflags; 370 371 /* Here we know that vma->vm_start <= nstart < vma->vm_end. */ 372 373 newflags = vm_flags; 374 newflags |= (vma->vm_flags & ~(VM_READ | VM_WRITE | VM_EXEC)); 375 376 /* newflags >> 4 shift VM_MAY% in place of VM_% */ 377 if ((newflags & ~(newflags >> 4)) & (VM_READ | VM_WRITE | VM_EXEC)) { 378 error = -EACCES; 379 goto out; 380 } 381 382 error = security_file_mprotect(vma, reqprot, prot); 383 if (error) 384 goto out; 385 386 tmp = vma->vm_end; 387 if (tmp > end) 388 tmp = end; 389 error = mprotect_fixup(vma, &prev, nstart, tmp, newflags); 390 if (error) 391 goto out; 392 nstart = tmp; 393 394 if (nstart < prev->vm_end) 395 nstart = prev->vm_end; 396 if (nstart >= end) 397 goto out; 398 399 vma = prev->vm_next; 400 if (!vma || vma->vm_start != nstart) { 401 error = -ENOMEM; 402 goto out; 403 } 404 } 405 out: 406 up_write(¤t->mm->mmap_sem); 407 return error; 408 } 409