1 /* 2 * mm/mprotect.c 3 * 4 * (C) Copyright 1994 Linus Torvalds 5 * (C) Copyright 2002 Christoph Hellwig 6 * 7 * Address space accounting code <alan@lxorguk.ukuu.org.uk> 8 * (C) Copyright 2002 Red Hat Inc, All Rights Reserved 9 */ 10 11 #include <linux/mm.h> 12 #include <linux/hugetlb.h> 13 #include <linux/shm.h> 14 #include <linux/mman.h> 15 #include <linux/fs.h> 16 #include <linux/highmem.h> 17 #include <linux/security.h> 18 #include <linux/mempolicy.h> 19 #include <linux/personality.h> 20 #include <linux/syscalls.h> 21 #include <linux/swap.h> 22 #include <linux/swapops.h> 23 #include <linux/mmu_notifier.h> 24 #include <linux/migrate.h> 25 #include <linux/perf_event.h> 26 #include <asm/uaccess.h> 27 #include <asm/pgtable.h> 28 #include <asm/cacheflush.h> 29 #include <asm/tlbflush.h> 30 31 #ifndef pgprot_modify 32 static inline pgprot_t pgprot_modify(pgprot_t oldprot, pgprot_t newprot) 33 { 34 return newprot; 35 } 36 #endif 37 38 static unsigned long change_pte_range(struct vm_area_struct *vma, pmd_t *pmd, 39 unsigned long addr, unsigned long end, pgprot_t newprot, 40 int dirty_accountable, int prot_numa, bool *ret_all_same_node) 41 { 42 struct mm_struct *mm = vma->vm_mm; 43 pte_t *pte, oldpte; 44 spinlock_t *ptl; 45 unsigned long pages = 0; 46 bool all_same_node = true; 47 int last_nid = -1; 48 49 pte = pte_offset_map_lock(mm, pmd, addr, &ptl); 50 arch_enter_lazy_mmu_mode(); 51 do { 52 oldpte = *pte; 53 if (pte_present(oldpte)) { 54 pte_t ptent; 55 bool updated = false; 56 57 ptent = ptep_modify_prot_start(mm, addr, pte); 58 if (!prot_numa) { 59 ptent = pte_modify(ptent, newprot); 60 updated = true; 61 } else { 62 struct page *page; 63 64 page = vm_normal_page(vma, addr, oldpte); 65 if (page) { 66 int this_nid = page_to_nid(page); 67 if (last_nid == -1) 68 last_nid = this_nid; 69 if (last_nid != this_nid) 70 all_same_node = false; 71 72 /* only check non-shared pages */ 73 if (!pte_numa(oldpte) && 74 page_mapcount(page) == 1) { 75 ptent = pte_mknuma(ptent); 76 updated = true; 77 } 78 } 79 } 80 81 /* 82 * Avoid taking write faults for pages we know to be 83 * dirty. 84 */ 85 if (dirty_accountable && pte_dirty(ptent)) { 86 ptent = pte_mkwrite(ptent); 87 updated = true; 88 } 89 90 if (updated) 91 pages++; 92 ptep_modify_prot_commit(mm, addr, pte, ptent); 93 } else if (IS_ENABLED(CONFIG_MIGRATION) && !pte_file(oldpte)) { 94 swp_entry_t entry = pte_to_swp_entry(oldpte); 95 96 if (is_write_migration_entry(entry)) { 97 pte_t newpte; 98 /* 99 * A protection check is difficult so 100 * just be safe and disable write 101 */ 102 make_migration_entry_read(&entry); 103 newpte = swp_entry_to_pte(entry); 104 if (pte_swp_soft_dirty(oldpte)) 105 newpte = pte_swp_mksoft_dirty(newpte); 106 set_pte_at(mm, addr, pte, newpte); 107 } 108 pages++; 109 } 110 } while (pte++, addr += PAGE_SIZE, addr != end); 111 arch_leave_lazy_mmu_mode(); 112 pte_unmap_unlock(pte - 1, ptl); 113 114 *ret_all_same_node = all_same_node; 115 return pages; 116 } 117 118 #ifdef CONFIG_NUMA_BALANCING 119 static inline void change_pmd_protnuma(struct mm_struct *mm, unsigned long addr, 120 pmd_t *pmd) 121 { 122 spin_lock(&mm->page_table_lock); 123 set_pmd_at(mm, addr & PMD_MASK, pmd, pmd_mknuma(*pmd)); 124 spin_unlock(&mm->page_table_lock); 125 } 126 #else 127 static inline void change_pmd_protnuma(struct mm_struct *mm, unsigned long addr, 128 pmd_t *pmd) 129 { 130 BUG(); 131 } 132 #endif /* CONFIG_NUMA_BALANCING */ 133 134 static inline unsigned long change_pmd_range(struct vm_area_struct *vma, 135 pud_t *pud, unsigned long addr, unsigned long end, 136 pgprot_t newprot, int dirty_accountable, int prot_numa) 137 { 138 pmd_t *pmd; 139 unsigned long next; 140 unsigned long pages = 0; 141 bool all_same_node; 142 143 pmd = pmd_offset(pud, addr); 144 do { 145 next = pmd_addr_end(addr, end); 146 if (pmd_trans_huge(*pmd)) { 147 if (next - addr != HPAGE_PMD_SIZE) 148 split_huge_page_pmd(vma, addr, pmd); 149 else if (change_huge_pmd(vma, pmd, addr, newprot, 150 prot_numa)) { 151 pages += HPAGE_PMD_NR; 152 continue; 153 } 154 /* fall through */ 155 } 156 if (pmd_none_or_clear_bad(pmd)) 157 continue; 158 pages += change_pte_range(vma, pmd, addr, next, newprot, 159 dirty_accountable, prot_numa, &all_same_node); 160 161 /* 162 * If we are changing protections for NUMA hinting faults then 163 * set pmd_numa if the examined pages were all on the same 164 * node. This allows a regular PMD to be handled as one fault 165 * and effectively batches the taking of the PTL 166 */ 167 if (prot_numa && all_same_node) 168 change_pmd_protnuma(vma->vm_mm, addr, pmd); 169 } while (pmd++, addr = next, addr != end); 170 171 return pages; 172 } 173 174 static inline unsigned long change_pud_range(struct vm_area_struct *vma, 175 pgd_t *pgd, unsigned long addr, unsigned long end, 176 pgprot_t newprot, int dirty_accountable, int prot_numa) 177 { 178 pud_t *pud; 179 unsigned long next; 180 unsigned long pages = 0; 181 182 pud = pud_offset(pgd, addr); 183 do { 184 next = pud_addr_end(addr, end); 185 if (pud_none_or_clear_bad(pud)) 186 continue; 187 pages += change_pmd_range(vma, pud, addr, next, newprot, 188 dirty_accountable, prot_numa); 189 } while (pud++, addr = next, addr != end); 190 191 return pages; 192 } 193 194 static unsigned long change_protection_range(struct vm_area_struct *vma, 195 unsigned long addr, unsigned long end, pgprot_t newprot, 196 int dirty_accountable, int prot_numa) 197 { 198 struct mm_struct *mm = vma->vm_mm; 199 pgd_t *pgd; 200 unsigned long next; 201 unsigned long start = addr; 202 unsigned long pages = 0; 203 204 BUG_ON(addr >= end); 205 pgd = pgd_offset(mm, addr); 206 flush_cache_range(vma, addr, end); 207 do { 208 next = pgd_addr_end(addr, end); 209 if (pgd_none_or_clear_bad(pgd)) 210 continue; 211 pages += change_pud_range(vma, pgd, addr, next, newprot, 212 dirty_accountable, prot_numa); 213 } while (pgd++, addr = next, addr != end); 214 215 /* Only flush the TLB if we actually modified any entries: */ 216 if (pages) 217 flush_tlb_range(vma, start, end); 218 219 return pages; 220 } 221 222 unsigned long change_protection(struct vm_area_struct *vma, unsigned long start, 223 unsigned long end, pgprot_t newprot, 224 int dirty_accountable, int prot_numa) 225 { 226 struct mm_struct *mm = vma->vm_mm; 227 unsigned long pages; 228 229 mmu_notifier_invalidate_range_start(mm, start, end); 230 if (is_vm_hugetlb_page(vma)) 231 pages = hugetlb_change_protection(vma, start, end, newprot); 232 else 233 pages = change_protection_range(vma, start, end, newprot, dirty_accountable, prot_numa); 234 mmu_notifier_invalidate_range_end(mm, start, end); 235 236 return pages; 237 } 238 239 int 240 mprotect_fixup(struct vm_area_struct *vma, struct vm_area_struct **pprev, 241 unsigned long start, unsigned long end, unsigned long newflags) 242 { 243 struct mm_struct *mm = vma->vm_mm; 244 unsigned long oldflags = vma->vm_flags; 245 long nrpages = (end - start) >> PAGE_SHIFT; 246 unsigned long charged = 0; 247 pgoff_t pgoff; 248 int error; 249 int dirty_accountable = 0; 250 251 if (newflags == oldflags) { 252 *pprev = vma; 253 return 0; 254 } 255 256 /* 257 * If we make a private mapping writable we increase our commit; 258 * but (without finer accounting) cannot reduce our commit if we 259 * make it unwritable again. hugetlb mapping were accounted for 260 * even if read-only so there is no need to account for them here 261 */ 262 if (newflags & VM_WRITE) { 263 if (!(oldflags & (VM_ACCOUNT|VM_WRITE|VM_HUGETLB| 264 VM_SHARED|VM_NORESERVE))) { 265 charged = nrpages; 266 if (security_vm_enough_memory_mm(mm, charged)) 267 return -ENOMEM; 268 newflags |= VM_ACCOUNT; 269 } 270 } 271 272 /* 273 * First try to merge with previous and/or next vma. 274 */ 275 pgoff = vma->vm_pgoff + ((start - vma->vm_start) >> PAGE_SHIFT); 276 *pprev = vma_merge(mm, *pprev, start, end, newflags, 277 vma->anon_vma, vma->vm_file, pgoff, vma_policy(vma)); 278 if (*pprev) { 279 vma = *pprev; 280 goto success; 281 } 282 283 *pprev = vma; 284 285 if (start != vma->vm_start) { 286 error = split_vma(mm, vma, start, 1); 287 if (error) 288 goto fail; 289 } 290 291 if (end != vma->vm_end) { 292 error = split_vma(mm, vma, end, 0); 293 if (error) 294 goto fail; 295 } 296 297 success: 298 /* 299 * vm_flags and vm_page_prot are protected by the mmap_sem 300 * held in write mode. 301 */ 302 vma->vm_flags = newflags; 303 vma->vm_page_prot = pgprot_modify(vma->vm_page_prot, 304 vm_get_page_prot(newflags)); 305 306 if (vma_wants_writenotify(vma)) { 307 vma->vm_page_prot = vm_get_page_prot(newflags & ~VM_SHARED); 308 dirty_accountable = 1; 309 } 310 311 change_protection(vma, start, end, vma->vm_page_prot, 312 dirty_accountable, 0); 313 314 vm_stat_account(mm, oldflags, vma->vm_file, -nrpages); 315 vm_stat_account(mm, newflags, vma->vm_file, nrpages); 316 perf_event_mmap(vma); 317 return 0; 318 319 fail: 320 vm_unacct_memory(charged); 321 return error; 322 } 323 324 SYSCALL_DEFINE3(mprotect, unsigned long, start, size_t, len, 325 unsigned long, prot) 326 { 327 unsigned long vm_flags, nstart, end, tmp, reqprot; 328 struct vm_area_struct *vma, *prev; 329 int error = -EINVAL; 330 const int grows = prot & (PROT_GROWSDOWN|PROT_GROWSUP); 331 prot &= ~(PROT_GROWSDOWN|PROT_GROWSUP); 332 if (grows == (PROT_GROWSDOWN|PROT_GROWSUP)) /* can't be both */ 333 return -EINVAL; 334 335 if (start & ~PAGE_MASK) 336 return -EINVAL; 337 if (!len) 338 return 0; 339 len = PAGE_ALIGN(len); 340 end = start + len; 341 if (end <= start) 342 return -ENOMEM; 343 if (!arch_validate_prot(prot)) 344 return -EINVAL; 345 346 reqprot = prot; 347 /* 348 * Does the application expect PROT_READ to imply PROT_EXEC: 349 */ 350 if ((prot & PROT_READ) && (current->personality & READ_IMPLIES_EXEC)) 351 prot |= PROT_EXEC; 352 353 vm_flags = calc_vm_prot_bits(prot); 354 355 down_write(¤t->mm->mmap_sem); 356 357 vma = find_vma(current->mm, start); 358 error = -ENOMEM; 359 if (!vma) 360 goto out; 361 prev = vma->vm_prev; 362 if (unlikely(grows & PROT_GROWSDOWN)) { 363 if (vma->vm_start >= end) 364 goto out; 365 start = vma->vm_start; 366 error = -EINVAL; 367 if (!(vma->vm_flags & VM_GROWSDOWN)) 368 goto out; 369 } else { 370 if (vma->vm_start > start) 371 goto out; 372 if (unlikely(grows & PROT_GROWSUP)) { 373 end = vma->vm_end; 374 error = -EINVAL; 375 if (!(vma->vm_flags & VM_GROWSUP)) 376 goto out; 377 } 378 } 379 if (start > vma->vm_start) 380 prev = vma; 381 382 for (nstart = start ; ; ) { 383 unsigned long newflags; 384 385 /* Here we know that vma->vm_start <= nstart < vma->vm_end. */ 386 387 newflags = vm_flags; 388 newflags |= (vma->vm_flags & ~(VM_READ | VM_WRITE | VM_EXEC)); 389 390 /* newflags >> 4 shift VM_MAY% in place of VM_% */ 391 if ((newflags & ~(newflags >> 4)) & (VM_READ | VM_WRITE | VM_EXEC)) { 392 error = -EACCES; 393 goto out; 394 } 395 396 error = security_file_mprotect(vma, reqprot, prot); 397 if (error) 398 goto out; 399 400 tmp = vma->vm_end; 401 if (tmp > end) 402 tmp = end; 403 error = mprotect_fixup(vma, &prev, nstart, tmp, newflags); 404 if (error) 405 goto out; 406 nstart = tmp; 407 408 if (nstart < prev->vm_end) 409 nstart = prev->vm_end; 410 if (nstart >= end) 411 goto out; 412 413 vma = prev->vm_next; 414 if (!vma || vma->vm_start != nstart) { 415 error = -ENOMEM; 416 goto out; 417 } 418 } 419 out: 420 up_write(¤t->mm->mmap_sem); 421 return error; 422 } 423