1 /* 2 * mm/rmap.c - physical to virtual reverse mappings 3 * 4 * Copyright 2001, Rik van Riel <riel@conectiva.com.br> 5 * Released under the General Public License (GPL). 6 * 7 * Simple, low overhead reverse mapping scheme. 8 * Please try to keep this thing as modular as possible. 9 * 10 * Provides methods for unmapping each kind of mapped page: 11 * the anon methods track anonymous pages, and 12 * the file methods track pages belonging to an inode. 13 * 14 * Original design by Rik van Riel <riel@conectiva.com.br> 2001 15 * File methods by Dave McCracken <dmccr@us.ibm.com> 2003, 2004 16 * Anonymous methods by Andrea Arcangeli <andrea@suse.de> 2004 17 * Contributions by Hugh Dickins 2003, 2004 18 */ 19 20 /* 21 * Lock ordering in mm: 22 * 23 * inode->i_mutex (while writing or truncating, not reading or faulting) 24 * mm->mmap_sem 25 * page->flags PG_locked (lock_page) 26 * hugetlbfs_i_mmap_rwsem_key (in huge_pmd_share) 27 * mapping->i_mmap_rwsem 28 * anon_vma->rwsem 29 * mm->page_table_lock or pte_lock 30 * zone->lru_lock (in mark_page_accessed, isolate_lru_page) 31 * swap_lock (in swap_duplicate, swap_info_get) 32 * mmlist_lock (in mmput, drain_mmlist and others) 33 * mapping->private_lock (in __set_page_dirty_buffers) 34 * mem_cgroup_{begin,end}_page_stat (memcg->move_lock) 35 * mapping->tree_lock (widely used) 36 * inode->i_lock (in set_page_dirty's __mark_inode_dirty) 37 * bdi.wb->list_lock (in set_page_dirty's __mark_inode_dirty) 38 * sb_lock (within inode_lock in fs/fs-writeback.c) 39 * mapping->tree_lock (widely used, in set_page_dirty, 40 * in arch-dependent flush_dcache_mmap_lock, 41 * within bdi.wb->list_lock in __sync_single_inode) 42 * 43 * anon_vma->rwsem,mapping->i_mutex (memory_failure, collect_procs_anon) 44 * ->tasklist_lock 45 * pte map lock 46 */ 47 48 #include <linux/mm.h> 49 #include <linux/pagemap.h> 50 #include <linux/swap.h> 51 #include <linux/swapops.h> 52 #include <linux/slab.h> 53 #include <linux/init.h> 54 #include <linux/ksm.h> 55 #include <linux/rmap.h> 56 #include <linux/rcupdate.h> 57 #include <linux/export.h> 58 #include <linux/memcontrol.h> 59 #include <linux/mmu_notifier.h> 60 #include <linux/migrate.h> 61 #include <linux/hugetlb.h> 62 #include <linux/backing-dev.h> 63 #include <linux/page_idle.h> 64 65 #include <asm/tlbflush.h> 66 67 #include <trace/events/tlb.h> 68 69 #include "internal.h" 70 71 static struct kmem_cache *anon_vma_cachep; 72 static struct kmem_cache *anon_vma_chain_cachep; 73 74 static inline struct anon_vma *anon_vma_alloc(void) 75 { 76 struct anon_vma *anon_vma; 77 78 anon_vma = kmem_cache_alloc(anon_vma_cachep, GFP_KERNEL); 79 if (anon_vma) { 80 atomic_set(&anon_vma->refcount, 1); 81 anon_vma->degree = 1; /* Reference for first vma */ 82 anon_vma->parent = anon_vma; 83 /* 84 * Initialise the anon_vma root to point to itself. If called 85 * from fork, the root will be reset to the parents anon_vma. 86 */ 87 anon_vma->root = anon_vma; 88 } 89 90 return anon_vma; 91 } 92 93 static inline void anon_vma_free(struct anon_vma *anon_vma) 94 { 95 VM_BUG_ON(atomic_read(&anon_vma->refcount)); 96 97 /* 98 * Synchronize against page_lock_anon_vma_read() such that 99 * we can safely hold the lock without the anon_vma getting 100 * freed. 101 * 102 * Relies on the full mb implied by the atomic_dec_and_test() from 103 * put_anon_vma() against the acquire barrier implied by 104 * down_read_trylock() from page_lock_anon_vma_read(). This orders: 105 * 106 * page_lock_anon_vma_read() VS put_anon_vma() 107 * down_read_trylock() atomic_dec_and_test() 108 * LOCK MB 109 * atomic_read() rwsem_is_locked() 110 * 111 * LOCK should suffice since the actual taking of the lock must 112 * happen _before_ what follows. 113 */ 114 might_sleep(); 115 if (rwsem_is_locked(&anon_vma->root->rwsem)) { 116 anon_vma_lock_write(anon_vma); 117 anon_vma_unlock_write(anon_vma); 118 } 119 120 kmem_cache_free(anon_vma_cachep, anon_vma); 121 } 122 123 static inline struct anon_vma_chain *anon_vma_chain_alloc(gfp_t gfp) 124 { 125 return kmem_cache_alloc(anon_vma_chain_cachep, gfp); 126 } 127 128 static void anon_vma_chain_free(struct anon_vma_chain *anon_vma_chain) 129 { 130 kmem_cache_free(anon_vma_chain_cachep, anon_vma_chain); 131 } 132 133 static void anon_vma_chain_link(struct vm_area_struct *vma, 134 struct anon_vma_chain *avc, 135 struct anon_vma *anon_vma) 136 { 137 avc->vma = vma; 138 avc->anon_vma = anon_vma; 139 list_add(&avc->same_vma, &vma->anon_vma_chain); 140 anon_vma_interval_tree_insert(avc, &anon_vma->rb_root); 141 } 142 143 /** 144 * anon_vma_prepare - attach an anon_vma to a memory region 145 * @vma: the memory region in question 146 * 147 * This makes sure the memory mapping described by 'vma' has 148 * an 'anon_vma' attached to it, so that we can associate the 149 * anonymous pages mapped into it with that anon_vma. 150 * 151 * The common case will be that we already have one, but if 152 * not we either need to find an adjacent mapping that we 153 * can re-use the anon_vma from (very common when the only 154 * reason for splitting a vma has been mprotect()), or we 155 * allocate a new one. 156 * 157 * Anon-vma allocations are very subtle, because we may have 158 * optimistically looked up an anon_vma in page_lock_anon_vma_read() 159 * and that may actually touch the spinlock even in the newly 160 * allocated vma (it depends on RCU to make sure that the 161 * anon_vma isn't actually destroyed). 162 * 163 * As a result, we need to do proper anon_vma locking even 164 * for the new allocation. At the same time, we do not want 165 * to do any locking for the common case of already having 166 * an anon_vma. 167 * 168 * This must be called with the mmap_sem held for reading. 169 */ 170 int anon_vma_prepare(struct vm_area_struct *vma) 171 { 172 struct anon_vma *anon_vma = vma->anon_vma; 173 struct anon_vma_chain *avc; 174 175 might_sleep(); 176 if (unlikely(!anon_vma)) { 177 struct mm_struct *mm = vma->vm_mm; 178 struct anon_vma *allocated; 179 180 avc = anon_vma_chain_alloc(GFP_KERNEL); 181 if (!avc) 182 goto out_enomem; 183 184 anon_vma = find_mergeable_anon_vma(vma); 185 allocated = NULL; 186 if (!anon_vma) { 187 anon_vma = anon_vma_alloc(); 188 if (unlikely(!anon_vma)) 189 goto out_enomem_free_avc; 190 allocated = anon_vma; 191 } 192 193 anon_vma_lock_write(anon_vma); 194 /* page_table_lock to protect against threads */ 195 spin_lock(&mm->page_table_lock); 196 if (likely(!vma->anon_vma)) { 197 vma->anon_vma = anon_vma; 198 anon_vma_chain_link(vma, avc, anon_vma); 199 /* vma reference or self-parent link for new root */ 200 anon_vma->degree++; 201 allocated = NULL; 202 avc = NULL; 203 } 204 spin_unlock(&mm->page_table_lock); 205 anon_vma_unlock_write(anon_vma); 206 207 if (unlikely(allocated)) 208 put_anon_vma(allocated); 209 if (unlikely(avc)) 210 anon_vma_chain_free(avc); 211 } 212 return 0; 213 214 out_enomem_free_avc: 215 anon_vma_chain_free(avc); 216 out_enomem: 217 return -ENOMEM; 218 } 219 220 /* 221 * This is a useful helper function for locking the anon_vma root as 222 * we traverse the vma->anon_vma_chain, looping over anon_vma's that 223 * have the same vma. 224 * 225 * Such anon_vma's should have the same root, so you'd expect to see 226 * just a single mutex_lock for the whole traversal. 227 */ 228 static inline struct anon_vma *lock_anon_vma_root(struct anon_vma *root, struct anon_vma *anon_vma) 229 { 230 struct anon_vma *new_root = anon_vma->root; 231 if (new_root != root) { 232 if (WARN_ON_ONCE(root)) 233 up_write(&root->rwsem); 234 root = new_root; 235 down_write(&root->rwsem); 236 } 237 return root; 238 } 239 240 static inline void unlock_anon_vma_root(struct anon_vma *root) 241 { 242 if (root) 243 up_write(&root->rwsem); 244 } 245 246 /* 247 * Attach the anon_vmas from src to dst. 248 * Returns 0 on success, -ENOMEM on failure. 249 * 250 * If dst->anon_vma is NULL this function tries to find and reuse existing 251 * anon_vma which has no vmas and only one child anon_vma. This prevents 252 * degradation of anon_vma hierarchy to endless linear chain in case of 253 * constantly forking task. On the other hand, an anon_vma with more than one 254 * child isn't reused even if there was no alive vma, thus rmap walker has a 255 * good chance of avoiding scanning the whole hierarchy when it searches where 256 * page is mapped. 257 */ 258 int anon_vma_clone(struct vm_area_struct *dst, struct vm_area_struct *src) 259 { 260 struct anon_vma_chain *avc, *pavc; 261 struct anon_vma *root = NULL; 262 263 list_for_each_entry_reverse(pavc, &src->anon_vma_chain, same_vma) { 264 struct anon_vma *anon_vma; 265 266 avc = anon_vma_chain_alloc(GFP_NOWAIT | __GFP_NOWARN); 267 if (unlikely(!avc)) { 268 unlock_anon_vma_root(root); 269 root = NULL; 270 avc = anon_vma_chain_alloc(GFP_KERNEL); 271 if (!avc) 272 goto enomem_failure; 273 } 274 anon_vma = pavc->anon_vma; 275 root = lock_anon_vma_root(root, anon_vma); 276 anon_vma_chain_link(dst, avc, anon_vma); 277 278 /* 279 * Reuse existing anon_vma if its degree lower than two, 280 * that means it has no vma and only one anon_vma child. 281 * 282 * Do not chose parent anon_vma, otherwise first child 283 * will always reuse it. Root anon_vma is never reused: 284 * it has self-parent reference and at least one child. 285 */ 286 if (!dst->anon_vma && anon_vma != src->anon_vma && 287 anon_vma->degree < 2) 288 dst->anon_vma = anon_vma; 289 } 290 if (dst->anon_vma) 291 dst->anon_vma->degree++; 292 unlock_anon_vma_root(root); 293 return 0; 294 295 enomem_failure: 296 /* 297 * dst->anon_vma is dropped here otherwise its degree can be incorrectly 298 * decremented in unlink_anon_vmas(). 299 * We can safely do this because callers of anon_vma_clone() don't care 300 * about dst->anon_vma if anon_vma_clone() failed. 301 */ 302 dst->anon_vma = NULL; 303 unlink_anon_vmas(dst); 304 return -ENOMEM; 305 } 306 307 /* 308 * Attach vma to its own anon_vma, as well as to the anon_vmas that 309 * the corresponding VMA in the parent process is attached to. 310 * Returns 0 on success, non-zero on failure. 311 */ 312 int anon_vma_fork(struct vm_area_struct *vma, struct vm_area_struct *pvma) 313 { 314 struct anon_vma_chain *avc; 315 struct anon_vma *anon_vma; 316 int error; 317 318 /* Don't bother if the parent process has no anon_vma here. */ 319 if (!pvma->anon_vma) 320 return 0; 321 322 /* Drop inherited anon_vma, we'll reuse existing or allocate new. */ 323 vma->anon_vma = NULL; 324 325 /* 326 * First, attach the new VMA to the parent VMA's anon_vmas, 327 * so rmap can find non-COWed pages in child processes. 328 */ 329 error = anon_vma_clone(vma, pvma); 330 if (error) 331 return error; 332 333 /* An existing anon_vma has been reused, all done then. */ 334 if (vma->anon_vma) 335 return 0; 336 337 /* Then add our own anon_vma. */ 338 anon_vma = anon_vma_alloc(); 339 if (!anon_vma) 340 goto out_error; 341 avc = anon_vma_chain_alloc(GFP_KERNEL); 342 if (!avc) 343 goto out_error_free_anon_vma; 344 345 /* 346 * The root anon_vma's spinlock is the lock actually used when we 347 * lock any of the anon_vmas in this anon_vma tree. 348 */ 349 anon_vma->root = pvma->anon_vma->root; 350 anon_vma->parent = pvma->anon_vma; 351 /* 352 * With refcounts, an anon_vma can stay around longer than the 353 * process it belongs to. The root anon_vma needs to be pinned until 354 * this anon_vma is freed, because the lock lives in the root. 355 */ 356 get_anon_vma(anon_vma->root); 357 /* Mark this anon_vma as the one where our new (COWed) pages go. */ 358 vma->anon_vma = anon_vma; 359 anon_vma_lock_write(anon_vma); 360 anon_vma_chain_link(vma, avc, anon_vma); 361 anon_vma->parent->degree++; 362 anon_vma_unlock_write(anon_vma); 363 364 return 0; 365 366 out_error_free_anon_vma: 367 put_anon_vma(anon_vma); 368 out_error: 369 unlink_anon_vmas(vma); 370 return -ENOMEM; 371 } 372 373 void unlink_anon_vmas(struct vm_area_struct *vma) 374 { 375 struct anon_vma_chain *avc, *next; 376 struct anon_vma *root = NULL; 377 378 /* 379 * Unlink each anon_vma chained to the VMA. This list is ordered 380 * from newest to oldest, ensuring the root anon_vma gets freed last. 381 */ 382 list_for_each_entry_safe(avc, next, &vma->anon_vma_chain, same_vma) { 383 struct anon_vma *anon_vma = avc->anon_vma; 384 385 root = lock_anon_vma_root(root, anon_vma); 386 anon_vma_interval_tree_remove(avc, &anon_vma->rb_root); 387 388 /* 389 * Leave empty anon_vmas on the list - we'll need 390 * to free them outside the lock. 391 */ 392 if (RB_EMPTY_ROOT(&anon_vma->rb_root)) { 393 anon_vma->parent->degree--; 394 continue; 395 } 396 397 list_del(&avc->same_vma); 398 anon_vma_chain_free(avc); 399 } 400 if (vma->anon_vma) 401 vma->anon_vma->degree--; 402 unlock_anon_vma_root(root); 403 404 /* 405 * Iterate the list once more, it now only contains empty and unlinked 406 * anon_vmas, destroy them. Could not do before due to __put_anon_vma() 407 * needing to write-acquire the anon_vma->root->rwsem. 408 */ 409 list_for_each_entry_safe(avc, next, &vma->anon_vma_chain, same_vma) { 410 struct anon_vma *anon_vma = avc->anon_vma; 411 412 BUG_ON(anon_vma->degree); 413 put_anon_vma(anon_vma); 414 415 list_del(&avc->same_vma); 416 anon_vma_chain_free(avc); 417 } 418 } 419 420 static void anon_vma_ctor(void *data) 421 { 422 struct anon_vma *anon_vma = data; 423 424 init_rwsem(&anon_vma->rwsem); 425 atomic_set(&anon_vma->refcount, 0); 426 anon_vma->rb_root = RB_ROOT; 427 } 428 429 void __init anon_vma_init(void) 430 { 431 anon_vma_cachep = kmem_cache_create("anon_vma", sizeof(struct anon_vma), 432 0, SLAB_DESTROY_BY_RCU|SLAB_PANIC|SLAB_ACCOUNT, 433 anon_vma_ctor); 434 anon_vma_chain_cachep = KMEM_CACHE(anon_vma_chain, 435 SLAB_PANIC|SLAB_ACCOUNT); 436 } 437 438 /* 439 * Getting a lock on a stable anon_vma from a page off the LRU is tricky! 440 * 441 * Since there is no serialization what so ever against page_remove_rmap() 442 * the best this function can do is return a locked anon_vma that might 443 * have been relevant to this page. 444 * 445 * The page might have been remapped to a different anon_vma or the anon_vma 446 * returned may already be freed (and even reused). 447 * 448 * In case it was remapped to a different anon_vma, the new anon_vma will be a 449 * child of the old anon_vma, and the anon_vma lifetime rules will therefore 450 * ensure that any anon_vma obtained from the page will still be valid for as 451 * long as we observe page_mapped() [ hence all those page_mapped() tests ]. 452 * 453 * All users of this function must be very careful when walking the anon_vma 454 * chain and verify that the page in question is indeed mapped in it 455 * [ something equivalent to page_mapped_in_vma() ]. 456 * 457 * Since anon_vma's slab is DESTROY_BY_RCU and we know from page_remove_rmap() 458 * that the anon_vma pointer from page->mapping is valid if there is a 459 * mapcount, we can dereference the anon_vma after observing those. 460 */ 461 struct anon_vma *page_get_anon_vma(struct page *page) 462 { 463 struct anon_vma *anon_vma = NULL; 464 unsigned long anon_mapping; 465 466 rcu_read_lock(); 467 anon_mapping = (unsigned long)READ_ONCE(page->mapping); 468 if ((anon_mapping & PAGE_MAPPING_FLAGS) != PAGE_MAPPING_ANON) 469 goto out; 470 if (!page_mapped(page)) 471 goto out; 472 473 anon_vma = (struct anon_vma *) (anon_mapping - PAGE_MAPPING_ANON); 474 if (!atomic_inc_not_zero(&anon_vma->refcount)) { 475 anon_vma = NULL; 476 goto out; 477 } 478 479 /* 480 * If this page is still mapped, then its anon_vma cannot have been 481 * freed. But if it has been unmapped, we have no security against the 482 * anon_vma structure being freed and reused (for another anon_vma: 483 * SLAB_DESTROY_BY_RCU guarantees that - so the atomic_inc_not_zero() 484 * above cannot corrupt). 485 */ 486 if (!page_mapped(page)) { 487 rcu_read_unlock(); 488 put_anon_vma(anon_vma); 489 return NULL; 490 } 491 out: 492 rcu_read_unlock(); 493 494 return anon_vma; 495 } 496 497 /* 498 * Similar to page_get_anon_vma() except it locks the anon_vma. 499 * 500 * Its a little more complex as it tries to keep the fast path to a single 501 * atomic op -- the trylock. If we fail the trylock, we fall back to getting a 502 * reference like with page_get_anon_vma() and then block on the mutex. 503 */ 504 struct anon_vma *page_lock_anon_vma_read(struct page *page) 505 { 506 struct anon_vma *anon_vma = NULL; 507 struct anon_vma *root_anon_vma; 508 unsigned long anon_mapping; 509 510 rcu_read_lock(); 511 anon_mapping = (unsigned long)READ_ONCE(page->mapping); 512 if ((anon_mapping & PAGE_MAPPING_FLAGS) != PAGE_MAPPING_ANON) 513 goto out; 514 if (!page_mapped(page)) 515 goto out; 516 517 anon_vma = (struct anon_vma *) (anon_mapping - PAGE_MAPPING_ANON); 518 root_anon_vma = READ_ONCE(anon_vma->root); 519 if (down_read_trylock(&root_anon_vma->rwsem)) { 520 /* 521 * If the page is still mapped, then this anon_vma is still 522 * its anon_vma, and holding the mutex ensures that it will 523 * not go away, see anon_vma_free(). 524 */ 525 if (!page_mapped(page)) { 526 up_read(&root_anon_vma->rwsem); 527 anon_vma = NULL; 528 } 529 goto out; 530 } 531 532 /* trylock failed, we got to sleep */ 533 if (!atomic_inc_not_zero(&anon_vma->refcount)) { 534 anon_vma = NULL; 535 goto out; 536 } 537 538 if (!page_mapped(page)) { 539 rcu_read_unlock(); 540 put_anon_vma(anon_vma); 541 return NULL; 542 } 543 544 /* we pinned the anon_vma, its safe to sleep */ 545 rcu_read_unlock(); 546 anon_vma_lock_read(anon_vma); 547 548 if (atomic_dec_and_test(&anon_vma->refcount)) { 549 /* 550 * Oops, we held the last refcount, release the lock 551 * and bail -- can't simply use put_anon_vma() because 552 * we'll deadlock on the anon_vma_lock_write() recursion. 553 */ 554 anon_vma_unlock_read(anon_vma); 555 __put_anon_vma(anon_vma); 556 anon_vma = NULL; 557 } 558 559 return anon_vma; 560 561 out: 562 rcu_read_unlock(); 563 return anon_vma; 564 } 565 566 void page_unlock_anon_vma_read(struct anon_vma *anon_vma) 567 { 568 anon_vma_unlock_read(anon_vma); 569 } 570 571 #ifdef CONFIG_ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH 572 static void percpu_flush_tlb_batch_pages(void *data) 573 { 574 /* 575 * All TLB entries are flushed on the assumption that it is 576 * cheaper to flush all TLBs and let them be refilled than 577 * flushing individual PFNs. Note that we do not track mm's 578 * to flush as that might simply be multiple full TLB flushes 579 * for no gain. 580 */ 581 count_vm_tlb_event(NR_TLB_REMOTE_FLUSH_RECEIVED); 582 flush_tlb_local(); 583 } 584 585 /* 586 * Flush TLB entries for recently unmapped pages from remote CPUs. It is 587 * important if a PTE was dirty when it was unmapped that it's flushed 588 * before any IO is initiated on the page to prevent lost writes. Similarly, 589 * it must be flushed before freeing to prevent data leakage. 590 */ 591 void try_to_unmap_flush(void) 592 { 593 struct tlbflush_unmap_batch *tlb_ubc = ¤t->tlb_ubc; 594 int cpu; 595 596 if (!tlb_ubc->flush_required) 597 return; 598 599 cpu = get_cpu(); 600 601 trace_tlb_flush(TLB_REMOTE_SHOOTDOWN, -1UL); 602 603 if (cpumask_test_cpu(cpu, &tlb_ubc->cpumask)) 604 percpu_flush_tlb_batch_pages(&tlb_ubc->cpumask); 605 606 if (cpumask_any_but(&tlb_ubc->cpumask, cpu) < nr_cpu_ids) { 607 smp_call_function_many(&tlb_ubc->cpumask, 608 percpu_flush_tlb_batch_pages, (void *)tlb_ubc, true); 609 } 610 cpumask_clear(&tlb_ubc->cpumask); 611 tlb_ubc->flush_required = false; 612 tlb_ubc->writable = false; 613 put_cpu(); 614 } 615 616 /* Flush iff there are potentially writable TLB entries that can race with IO */ 617 void try_to_unmap_flush_dirty(void) 618 { 619 struct tlbflush_unmap_batch *tlb_ubc = ¤t->tlb_ubc; 620 621 if (tlb_ubc->writable) 622 try_to_unmap_flush(); 623 } 624 625 static void set_tlb_ubc_flush_pending(struct mm_struct *mm, 626 struct page *page, bool writable) 627 { 628 struct tlbflush_unmap_batch *tlb_ubc = ¤t->tlb_ubc; 629 630 cpumask_or(&tlb_ubc->cpumask, &tlb_ubc->cpumask, mm_cpumask(mm)); 631 tlb_ubc->flush_required = true; 632 633 /* 634 * If the PTE was dirty then it's best to assume it's writable. The 635 * caller must use try_to_unmap_flush_dirty() or try_to_unmap_flush() 636 * before the page is queued for IO. 637 */ 638 if (writable) 639 tlb_ubc->writable = true; 640 } 641 642 /* 643 * Returns true if the TLB flush should be deferred to the end of a batch of 644 * unmap operations to reduce IPIs. 645 */ 646 static bool should_defer_flush(struct mm_struct *mm, enum ttu_flags flags) 647 { 648 bool should_defer = false; 649 650 if (!(flags & TTU_BATCH_FLUSH)) 651 return false; 652 653 /* If remote CPUs need to be flushed then defer batch the flush */ 654 if (cpumask_any_but(mm_cpumask(mm), get_cpu()) < nr_cpu_ids) 655 should_defer = true; 656 put_cpu(); 657 658 return should_defer; 659 } 660 #else 661 static void set_tlb_ubc_flush_pending(struct mm_struct *mm, 662 struct page *page, bool writable) 663 { 664 } 665 666 static bool should_defer_flush(struct mm_struct *mm, enum ttu_flags flags) 667 { 668 return false; 669 } 670 #endif /* CONFIG_ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH */ 671 672 /* 673 * At what user virtual address is page expected in vma? 674 * Caller should check the page is actually part of the vma. 675 */ 676 unsigned long page_address_in_vma(struct page *page, struct vm_area_struct *vma) 677 { 678 unsigned long address; 679 if (PageAnon(page)) { 680 struct anon_vma *page__anon_vma = page_anon_vma(page); 681 /* 682 * Note: swapoff's unuse_vma() is more efficient with this 683 * check, and needs it to match anon_vma when KSM is active. 684 */ 685 if (!vma->anon_vma || !page__anon_vma || 686 vma->anon_vma->root != page__anon_vma->root) 687 return -EFAULT; 688 } else if (page->mapping) { 689 if (!vma->vm_file || vma->vm_file->f_mapping != page->mapping) 690 return -EFAULT; 691 } else 692 return -EFAULT; 693 address = __vma_address(page, vma); 694 if (unlikely(address < vma->vm_start || address >= vma->vm_end)) 695 return -EFAULT; 696 return address; 697 } 698 699 pmd_t *mm_find_pmd(struct mm_struct *mm, unsigned long address) 700 { 701 pgd_t *pgd; 702 pud_t *pud; 703 pmd_t *pmd = NULL; 704 pmd_t pmde; 705 706 pgd = pgd_offset(mm, address); 707 if (!pgd_present(*pgd)) 708 goto out; 709 710 pud = pud_offset(pgd, address); 711 if (!pud_present(*pud)) 712 goto out; 713 714 pmd = pmd_offset(pud, address); 715 /* 716 * Some THP functions use the sequence pmdp_huge_clear_flush(), set_pmd_at() 717 * without holding anon_vma lock for write. So when looking for a 718 * genuine pmde (in which to find pte), test present and !THP together. 719 */ 720 pmde = *pmd; 721 barrier(); 722 if (!pmd_present(pmde) || pmd_trans_huge(pmde)) 723 pmd = NULL; 724 out: 725 return pmd; 726 } 727 728 /* 729 * Check that @page is mapped at @address into @mm. 730 * 731 * If @sync is false, page_check_address may perform a racy check to avoid 732 * the page table lock when the pte is not present (helpful when reclaiming 733 * highly shared pages). 734 * 735 * On success returns with pte mapped and locked. 736 */ 737 pte_t *__page_check_address(struct page *page, struct mm_struct *mm, 738 unsigned long address, spinlock_t **ptlp, int sync) 739 { 740 pmd_t *pmd; 741 pte_t *pte; 742 spinlock_t *ptl; 743 744 if (unlikely(PageHuge(page))) { 745 /* when pud is not present, pte will be NULL */ 746 pte = huge_pte_offset(mm, address); 747 if (!pte) 748 return NULL; 749 750 ptl = huge_pte_lockptr(page_hstate(page), mm, pte); 751 goto check; 752 } 753 754 pmd = mm_find_pmd(mm, address); 755 if (!pmd) 756 return NULL; 757 758 pte = pte_offset_map(pmd, address); 759 /* Make a quick check before getting the lock */ 760 if (!sync && !pte_present(*pte)) { 761 pte_unmap(pte); 762 return NULL; 763 } 764 765 ptl = pte_lockptr(mm, pmd); 766 check: 767 spin_lock(ptl); 768 if (pte_present(*pte) && page_to_pfn(page) == pte_pfn(*pte)) { 769 *ptlp = ptl; 770 return pte; 771 } 772 pte_unmap_unlock(pte, ptl); 773 return NULL; 774 } 775 776 /** 777 * page_mapped_in_vma - check whether a page is really mapped in a VMA 778 * @page: the page to test 779 * @vma: the VMA to test 780 * 781 * Returns 1 if the page is mapped into the page tables of the VMA, 0 782 * if the page is not mapped into the page tables of this VMA. Only 783 * valid for normal file or anonymous VMAs. 784 */ 785 int page_mapped_in_vma(struct page *page, struct vm_area_struct *vma) 786 { 787 unsigned long address; 788 pte_t *pte; 789 spinlock_t *ptl; 790 791 address = __vma_address(page, vma); 792 if (unlikely(address < vma->vm_start || address >= vma->vm_end)) 793 return 0; 794 pte = page_check_address(page, vma->vm_mm, address, &ptl, 1); 795 if (!pte) /* the page is not in this mm */ 796 return 0; 797 pte_unmap_unlock(pte, ptl); 798 799 return 1; 800 } 801 802 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 803 /* 804 * Check that @page is mapped at @address into @mm. In contrast to 805 * page_check_address(), this function can handle transparent huge pages. 806 * 807 * On success returns true with pte mapped and locked. For PMD-mapped 808 * transparent huge pages *@ptep is set to NULL. 809 */ 810 bool page_check_address_transhuge(struct page *page, struct mm_struct *mm, 811 unsigned long address, pmd_t **pmdp, 812 pte_t **ptep, spinlock_t **ptlp) 813 { 814 pgd_t *pgd; 815 pud_t *pud; 816 pmd_t *pmd; 817 pte_t *pte; 818 spinlock_t *ptl; 819 820 if (unlikely(PageHuge(page))) { 821 /* when pud is not present, pte will be NULL */ 822 pte = huge_pte_offset(mm, address); 823 if (!pte) 824 return false; 825 826 ptl = huge_pte_lockptr(page_hstate(page), mm, pte); 827 pmd = NULL; 828 goto check_pte; 829 } 830 831 pgd = pgd_offset(mm, address); 832 if (!pgd_present(*pgd)) 833 return false; 834 pud = pud_offset(pgd, address); 835 if (!pud_present(*pud)) 836 return false; 837 pmd = pmd_offset(pud, address); 838 839 if (pmd_trans_huge(*pmd)) { 840 ptl = pmd_lock(mm, pmd); 841 if (!pmd_present(*pmd)) 842 goto unlock_pmd; 843 if (unlikely(!pmd_trans_huge(*pmd))) { 844 spin_unlock(ptl); 845 goto map_pte; 846 } 847 848 if (pmd_page(*pmd) != page) 849 goto unlock_pmd; 850 851 pte = NULL; 852 goto found; 853 unlock_pmd: 854 spin_unlock(ptl); 855 return false; 856 } else { 857 pmd_t pmde = *pmd; 858 859 barrier(); 860 if (!pmd_present(pmde) || pmd_trans_huge(pmde)) 861 return false; 862 } 863 map_pte: 864 pte = pte_offset_map(pmd, address); 865 if (!pte_present(*pte)) { 866 pte_unmap(pte); 867 return false; 868 } 869 870 ptl = pte_lockptr(mm, pmd); 871 check_pte: 872 spin_lock(ptl); 873 874 if (!pte_present(*pte)) { 875 pte_unmap_unlock(pte, ptl); 876 return false; 877 } 878 879 /* THP can be referenced by any subpage */ 880 if (pte_pfn(*pte) - page_to_pfn(page) >= hpage_nr_pages(page)) { 881 pte_unmap_unlock(pte, ptl); 882 return false; 883 } 884 found: 885 *ptep = pte; 886 *pmdp = pmd; 887 *ptlp = ptl; 888 return true; 889 } 890 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 891 892 struct page_referenced_arg { 893 int mapcount; 894 int referenced; 895 unsigned long vm_flags; 896 struct mem_cgroup *memcg; 897 }; 898 /* 899 * arg: page_referenced_arg will be passed 900 */ 901 static int page_referenced_one(struct page *page, struct vm_area_struct *vma, 902 unsigned long address, void *arg) 903 { 904 struct mm_struct *mm = vma->vm_mm; 905 struct page_referenced_arg *pra = arg; 906 pmd_t *pmd; 907 pte_t *pte; 908 spinlock_t *ptl; 909 int referenced = 0; 910 911 if (!page_check_address_transhuge(page, mm, address, &pmd, &pte, &ptl)) 912 return SWAP_AGAIN; 913 914 if (vma->vm_flags & VM_LOCKED) { 915 if (pte) 916 pte_unmap(pte); 917 spin_unlock(ptl); 918 pra->vm_flags |= VM_LOCKED; 919 return SWAP_FAIL; /* To break the loop */ 920 } 921 922 if (pte) { 923 if (ptep_clear_flush_young_notify(vma, address, pte)) { 924 /* 925 * Don't treat a reference through a sequentially read 926 * mapping as such. If the page has been used in 927 * another mapping, we will catch it; if this other 928 * mapping is already gone, the unmap path will have 929 * set PG_referenced or activated the page. 930 */ 931 if (likely(!(vma->vm_flags & VM_SEQ_READ))) 932 referenced++; 933 } 934 pte_unmap(pte); 935 } else if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) { 936 if (pmdp_clear_flush_young_notify(vma, address, pmd)) 937 referenced++; 938 } else { 939 /* unexpected pmd-mapped page? */ 940 WARN_ON_ONCE(1); 941 } 942 spin_unlock(ptl); 943 944 if (referenced) 945 clear_page_idle(page); 946 if (test_and_clear_page_young(page)) 947 referenced++; 948 949 if (referenced) { 950 pra->referenced++; 951 pra->vm_flags |= vma->vm_flags; 952 } 953 954 pra->mapcount--; 955 if (!pra->mapcount) 956 return SWAP_SUCCESS; /* To break the loop */ 957 958 return SWAP_AGAIN; 959 } 960 961 static bool invalid_page_referenced_vma(struct vm_area_struct *vma, void *arg) 962 { 963 struct page_referenced_arg *pra = arg; 964 struct mem_cgroup *memcg = pra->memcg; 965 966 if (!mm_match_cgroup(vma->vm_mm, memcg)) 967 return true; 968 969 return false; 970 } 971 972 /** 973 * page_referenced - test if the page was referenced 974 * @page: the page to test 975 * @is_locked: caller holds lock on the page 976 * @memcg: target memory cgroup 977 * @vm_flags: collect encountered vma->vm_flags who actually referenced the page 978 * 979 * Quick test_and_clear_referenced for all mappings to a page, 980 * returns the number of ptes which referenced the page. 981 */ 982 int page_referenced(struct page *page, 983 int is_locked, 984 struct mem_cgroup *memcg, 985 unsigned long *vm_flags) 986 { 987 int ret; 988 int we_locked = 0; 989 struct page_referenced_arg pra = { 990 .mapcount = total_mapcount(page), 991 .memcg = memcg, 992 }; 993 struct rmap_walk_control rwc = { 994 .rmap_one = page_referenced_one, 995 .arg = (void *)&pra, 996 .anon_lock = page_lock_anon_vma_read, 997 }; 998 999 *vm_flags = 0; 1000 if (!page_mapped(page)) 1001 return 0; 1002 1003 if (!page_rmapping(page)) 1004 return 0; 1005 1006 if (!is_locked && (!PageAnon(page) || PageKsm(page))) { 1007 we_locked = trylock_page(page); 1008 if (!we_locked) 1009 return 1; 1010 } 1011 1012 /* 1013 * If we are reclaiming on behalf of a cgroup, skip 1014 * counting on behalf of references from different 1015 * cgroups 1016 */ 1017 if (memcg) { 1018 rwc.invalid_vma = invalid_page_referenced_vma; 1019 } 1020 1021 ret = rmap_walk(page, &rwc); 1022 *vm_flags = pra.vm_flags; 1023 1024 if (we_locked) 1025 unlock_page(page); 1026 1027 return pra.referenced; 1028 } 1029 1030 static int page_mkclean_one(struct page *page, struct vm_area_struct *vma, 1031 unsigned long address, void *arg) 1032 { 1033 struct mm_struct *mm = vma->vm_mm; 1034 pte_t *pte; 1035 spinlock_t *ptl; 1036 int ret = 0; 1037 int *cleaned = arg; 1038 1039 pte = page_check_address(page, mm, address, &ptl, 1); 1040 if (!pte) 1041 goto out; 1042 1043 if (pte_dirty(*pte) || pte_write(*pte)) { 1044 pte_t entry; 1045 1046 flush_cache_page(vma, address, pte_pfn(*pte)); 1047 entry = ptep_clear_flush(vma, address, pte); 1048 entry = pte_wrprotect(entry); 1049 entry = pte_mkclean(entry); 1050 set_pte_at(mm, address, pte, entry); 1051 ret = 1; 1052 } 1053 1054 pte_unmap_unlock(pte, ptl); 1055 1056 if (ret) { 1057 mmu_notifier_invalidate_page(mm, address); 1058 (*cleaned)++; 1059 } 1060 out: 1061 return SWAP_AGAIN; 1062 } 1063 1064 static bool invalid_mkclean_vma(struct vm_area_struct *vma, void *arg) 1065 { 1066 if (vma->vm_flags & VM_SHARED) 1067 return false; 1068 1069 return true; 1070 } 1071 1072 int page_mkclean(struct page *page) 1073 { 1074 int cleaned = 0; 1075 struct address_space *mapping; 1076 struct rmap_walk_control rwc = { 1077 .arg = (void *)&cleaned, 1078 .rmap_one = page_mkclean_one, 1079 .invalid_vma = invalid_mkclean_vma, 1080 }; 1081 1082 BUG_ON(!PageLocked(page)); 1083 1084 if (!page_mapped(page)) 1085 return 0; 1086 1087 mapping = page_mapping(page); 1088 if (!mapping) 1089 return 0; 1090 1091 rmap_walk(page, &rwc); 1092 1093 return cleaned; 1094 } 1095 EXPORT_SYMBOL_GPL(page_mkclean); 1096 1097 /** 1098 * page_move_anon_rmap - move a page to our anon_vma 1099 * @page: the page to move to our anon_vma 1100 * @vma: the vma the page belongs to 1101 * @address: the user virtual address mapped 1102 * 1103 * When a page belongs exclusively to one process after a COW event, 1104 * that page can be moved into the anon_vma that belongs to just that 1105 * process, so the rmap code will not search the parent or sibling 1106 * processes. 1107 */ 1108 void page_move_anon_rmap(struct page *page, 1109 struct vm_area_struct *vma, unsigned long address) 1110 { 1111 struct anon_vma *anon_vma = vma->anon_vma; 1112 1113 VM_BUG_ON_PAGE(!PageLocked(page), page); 1114 VM_BUG_ON_VMA(!anon_vma, vma); 1115 VM_BUG_ON_PAGE(page->index != linear_page_index(vma, address), page); 1116 1117 anon_vma = (void *) anon_vma + PAGE_MAPPING_ANON; 1118 /* 1119 * Ensure that anon_vma and the PAGE_MAPPING_ANON bit are written 1120 * simultaneously, so a concurrent reader (eg page_referenced()'s 1121 * PageAnon()) will not see one without the other. 1122 */ 1123 WRITE_ONCE(page->mapping, (struct address_space *) anon_vma); 1124 } 1125 1126 /** 1127 * __page_set_anon_rmap - set up new anonymous rmap 1128 * @page: Page to add to rmap 1129 * @vma: VM area to add page to. 1130 * @address: User virtual address of the mapping 1131 * @exclusive: the page is exclusively owned by the current process 1132 */ 1133 static void __page_set_anon_rmap(struct page *page, 1134 struct vm_area_struct *vma, unsigned long address, int exclusive) 1135 { 1136 struct anon_vma *anon_vma = vma->anon_vma; 1137 1138 BUG_ON(!anon_vma); 1139 1140 if (PageAnon(page)) 1141 return; 1142 1143 /* 1144 * If the page isn't exclusively mapped into this vma, 1145 * we must use the _oldest_ possible anon_vma for the 1146 * page mapping! 1147 */ 1148 if (!exclusive) 1149 anon_vma = anon_vma->root; 1150 1151 anon_vma = (void *) anon_vma + PAGE_MAPPING_ANON; 1152 page->mapping = (struct address_space *) anon_vma; 1153 page->index = linear_page_index(vma, address); 1154 } 1155 1156 /** 1157 * __page_check_anon_rmap - sanity check anonymous rmap addition 1158 * @page: the page to add the mapping to 1159 * @vma: the vm area in which the mapping is added 1160 * @address: the user virtual address mapped 1161 */ 1162 static void __page_check_anon_rmap(struct page *page, 1163 struct vm_area_struct *vma, unsigned long address) 1164 { 1165 #ifdef CONFIG_DEBUG_VM 1166 /* 1167 * The page's anon-rmap details (mapping and index) are guaranteed to 1168 * be set up correctly at this point. 1169 * 1170 * We have exclusion against page_add_anon_rmap because the caller 1171 * always holds the page locked, except if called from page_dup_rmap, 1172 * in which case the page is already known to be setup. 1173 * 1174 * We have exclusion against page_add_new_anon_rmap because those pages 1175 * are initially only visible via the pagetables, and the pte is locked 1176 * over the call to page_add_new_anon_rmap. 1177 */ 1178 BUG_ON(page_anon_vma(page)->root != vma->anon_vma->root); 1179 BUG_ON(page_to_pgoff(page) != linear_page_index(vma, address)); 1180 #endif 1181 } 1182 1183 /** 1184 * page_add_anon_rmap - add pte mapping to an anonymous page 1185 * @page: the page to add the mapping to 1186 * @vma: the vm area in which the mapping is added 1187 * @address: the user virtual address mapped 1188 * @compound: charge the page as compound or small page 1189 * 1190 * The caller needs to hold the pte lock, and the page must be locked in 1191 * the anon_vma case: to serialize mapping,index checking after setting, 1192 * and to ensure that PageAnon is not being upgraded racily to PageKsm 1193 * (but PageKsm is never downgraded to PageAnon). 1194 */ 1195 void page_add_anon_rmap(struct page *page, 1196 struct vm_area_struct *vma, unsigned long address, bool compound) 1197 { 1198 do_page_add_anon_rmap(page, vma, address, compound ? RMAP_COMPOUND : 0); 1199 } 1200 1201 /* 1202 * Special version of the above for do_swap_page, which often runs 1203 * into pages that are exclusively owned by the current process. 1204 * Everybody else should continue to use page_add_anon_rmap above. 1205 */ 1206 void do_page_add_anon_rmap(struct page *page, 1207 struct vm_area_struct *vma, unsigned long address, int flags) 1208 { 1209 bool compound = flags & RMAP_COMPOUND; 1210 bool first; 1211 1212 if (compound) { 1213 atomic_t *mapcount; 1214 VM_BUG_ON_PAGE(!PageLocked(page), page); 1215 VM_BUG_ON_PAGE(!PageTransHuge(page), page); 1216 mapcount = compound_mapcount_ptr(page); 1217 first = atomic_inc_and_test(mapcount); 1218 } else { 1219 first = atomic_inc_and_test(&page->_mapcount); 1220 } 1221 1222 if (first) { 1223 int nr = compound ? hpage_nr_pages(page) : 1; 1224 /* 1225 * We use the irq-unsafe __{inc|mod}_zone_page_stat because 1226 * these counters are not modified in interrupt context, and 1227 * pte lock(a spinlock) is held, which implies preemption 1228 * disabled. 1229 */ 1230 if (compound) { 1231 __inc_zone_page_state(page, 1232 NR_ANON_TRANSPARENT_HUGEPAGES); 1233 } 1234 __mod_zone_page_state(page_zone(page), NR_ANON_PAGES, nr); 1235 } 1236 if (unlikely(PageKsm(page))) 1237 return; 1238 1239 VM_BUG_ON_PAGE(!PageLocked(page), page); 1240 1241 /* address might be in next vma when migration races vma_adjust */ 1242 if (first) 1243 __page_set_anon_rmap(page, vma, address, 1244 flags & RMAP_EXCLUSIVE); 1245 else 1246 __page_check_anon_rmap(page, vma, address); 1247 } 1248 1249 /** 1250 * page_add_new_anon_rmap - add pte mapping to a new anonymous page 1251 * @page: the page to add the mapping to 1252 * @vma: the vm area in which the mapping is added 1253 * @address: the user virtual address mapped 1254 * @compound: charge the page as compound or small page 1255 * 1256 * Same as page_add_anon_rmap but must only be called on *new* pages. 1257 * This means the inc-and-test can be bypassed. 1258 * Page does not have to be locked. 1259 */ 1260 void page_add_new_anon_rmap(struct page *page, 1261 struct vm_area_struct *vma, unsigned long address, bool compound) 1262 { 1263 int nr = compound ? hpage_nr_pages(page) : 1; 1264 1265 VM_BUG_ON_VMA(address < vma->vm_start || address >= vma->vm_end, vma); 1266 SetPageSwapBacked(page); 1267 if (compound) { 1268 VM_BUG_ON_PAGE(!PageTransHuge(page), page); 1269 /* increment count (starts at -1) */ 1270 atomic_set(compound_mapcount_ptr(page), 0); 1271 __inc_zone_page_state(page, NR_ANON_TRANSPARENT_HUGEPAGES); 1272 } else { 1273 /* Anon THP always mapped first with PMD */ 1274 VM_BUG_ON_PAGE(PageTransCompound(page), page); 1275 /* increment count (starts at -1) */ 1276 atomic_set(&page->_mapcount, 0); 1277 } 1278 __mod_zone_page_state(page_zone(page), NR_ANON_PAGES, nr); 1279 __page_set_anon_rmap(page, vma, address, 1); 1280 } 1281 1282 /** 1283 * page_add_file_rmap - add pte mapping to a file page 1284 * @page: the page to add the mapping to 1285 * 1286 * The caller needs to hold the pte lock. 1287 */ 1288 void page_add_file_rmap(struct page *page) 1289 { 1290 lock_page_memcg(page); 1291 if (atomic_inc_and_test(&page->_mapcount)) { 1292 __inc_zone_page_state(page, NR_FILE_MAPPED); 1293 mem_cgroup_inc_page_stat(page, MEM_CGROUP_STAT_FILE_MAPPED); 1294 } 1295 unlock_page_memcg(page); 1296 } 1297 1298 static void page_remove_file_rmap(struct page *page) 1299 { 1300 lock_page_memcg(page); 1301 1302 /* Hugepages are not counted in NR_FILE_MAPPED for now. */ 1303 if (unlikely(PageHuge(page))) { 1304 /* hugetlb pages are always mapped with pmds */ 1305 atomic_dec(compound_mapcount_ptr(page)); 1306 goto out; 1307 } 1308 1309 /* page still mapped by someone else? */ 1310 if (!atomic_add_negative(-1, &page->_mapcount)) 1311 goto out; 1312 1313 /* 1314 * We use the irq-unsafe __{inc|mod}_zone_page_stat because 1315 * these counters are not modified in interrupt context, and 1316 * pte lock(a spinlock) is held, which implies preemption disabled. 1317 */ 1318 __dec_zone_page_state(page, NR_FILE_MAPPED); 1319 mem_cgroup_dec_page_stat(page, MEM_CGROUP_STAT_FILE_MAPPED); 1320 1321 if (unlikely(PageMlocked(page))) 1322 clear_page_mlock(page); 1323 out: 1324 unlock_page_memcg(page); 1325 } 1326 1327 static void page_remove_anon_compound_rmap(struct page *page) 1328 { 1329 int i, nr; 1330 1331 if (!atomic_add_negative(-1, compound_mapcount_ptr(page))) 1332 return; 1333 1334 /* Hugepages are not counted in NR_ANON_PAGES for now. */ 1335 if (unlikely(PageHuge(page))) 1336 return; 1337 1338 if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) 1339 return; 1340 1341 __dec_zone_page_state(page, NR_ANON_TRANSPARENT_HUGEPAGES); 1342 1343 if (TestClearPageDoubleMap(page)) { 1344 /* 1345 * Subpages can be mapped with PTEs too. Check how many of 1346 * themi are still mapped. 1347 */ 1348 for (i = 0, nr = 0; i < HPAGE_PMD_NR; i++) { 1349 if (atomic_add_negative(-1, &page[i]._mapcount)) 1350 nr++; 1351 } 1352 } else { 1353 nr = HPAGE_PMD_NR; 1354 } 1355 1356 if (unlikely(PageMlocked(page))) 1357 clear_page_mlock(page); 1358 1359 if (nr) { 1360 __mod_zone_page_state(page_zone(page), NR_ANON_PAGES, -nr); 1361 deferred_split_huge_page(page); 1362 } 1363 } 1364 1365 /** 1366 * page_remove_rmap - take down pte mapping from a page 1367 * @page: page to remove mapping from 1368 * @compound: uncharge the page as compound or small page 1369 * 1370 * The caller needs to hold the pte lock. 1371 */ 1372 void page_remove_rmap(struct page *page, bool compound) 1373 { 1374 if (!PageAnon(page)) { 1375 VM_BUG_ON_PAGE(compound && !PageHuge(page), page); 1376 page_remove_file_rmap(page); 1377 return; 1378 } 1379 1380 if (compound) 1381 return page_remove_anon_compound_rmap(page); 1382 1383 /* page still mapped by someone else? */ 1384 if (!atomic_add_negative(-1, &page->_mapcount)) 1385 return; 1386 1387 /* 1388 * We use the irq-unsafe __{inc|mod}_zone_page_stat because 1389 * these counters are not modified in interrupt context, and 1390 * pte lock(a spinlock) is held, which implies preemption disabled. 1391 */ 1392 __dec_zone_page_state(page, NR_ANON_PAGES); 1393 1394 if (unlikely(PageMlocked(page))) 1395 clear_page_mlock(page); 1396 1397 if (PageTransCompound(page)) 1398 deferred_split_huge_page(compound_head(page)); 1399 1400 /* 1401 * It would be tidy to reset the PageAnon mapping here, 1402 * but that might overwrite a racing page_add_anon_rmap 1403 * which increments mapcount after us but sets mapping 1404 * before us: so leave the reset to free_hot_cold_page, 1405 * and remember that it's only reliable while mapped. 1406 * Leaving it set also helps swapoff to reinstate ptes 1407 * faster for those pages still in swapcache. 1408 */ 1409 } 1410 1411 struct rmap_private { 1412 enum ttu_flags flags; 1413 int lazyfreed; 1414 }; 1415 1416 /* 1417 * @arg: enum ttu_flags will be passed to this argument 1418 */ 1419 static int try_to_unmap_one(struct page *page, struct vm_area_struct *vma, 1420 unsigned long address, void *arg) 1421 { 1422 struct mm_struct *mm = vma->vm_mm; 1423 pte_t *pte; 1424 pte_t pteval; 1425 spinlock_t *ptl; 1426 int ret = SWAP_AGAIN; 1427 struct rmap_private *rp = arg; 1428 enum ttu_flags flags = rp->flags; 1429 1430 /* munlock has nothing to gain from examining un-locked vmas */ 1431 if ((flags & TTU_MUNLOCK) && !(vma->vm_flags & VM_LOCKED)) 1432 goto out; 1433 1434 if (flags & TTU_SPLIT_HUGE_PMD) { 1435 split_huge_pmd_address(vma, address, 1436 flags & TTU_MIGRATION, page); 1437 /* check if we have anything to do after split */ 1438 if (page_mapcount(page) == 0) 1439 goto out; 1440 } 1441 1442 pte = page_check_address(page, mm, address, &ptl, 0); 1443 if (!pte) 1444 goto out; 1445 1446 /* 1447 * If the page is mlock()d, we cannot swap it out. 1448 * If it's recently referenced (perhaps page_referenced 1449 * skipped over this mm) then we should reactivate it. 1450 */ 1451 if (!(flags & TTU_IGNORE_MLOCK)) { 1452 if (vma->vm_flags & VM_LOCKED) { 1453 /* Holding pte lock, we do *not* need mmap_sem here */ 1454 mlock_vma_page(page); 1455 ret = SWAP_MLOCK; 1456 goto out_unmap; 1457 } 1458 if (flags & TTU_MUNLOCK) 1459 goto out_unmap; 1460 } 1461 if (!(flags & TTU_IGNORE_ACCESS)) { 1462 if (ptep_clear_flush_young_notify(vma, address, pte)) { 1463 ret = SWAP_FAIL; 1464 goto out_unmap; 1465 } 1466 } 1467 1468 /* Nuke the page table entry. */ 1469 flush_cache_page(vma, address, page_to_pfn(page)); 1470 if (should_defer_flush(mm, flags)) { 1471 /* 1472 * We clear the PTE but do not flush so potentially a remote 1473 * CPU could still be writing to the page. If the entry was 1474 * previously clean then the architecture must guarantee that 1475 * a clear->dirty transition on a cached TLB entry is written 1476 * through and traps if the PTE is unmapped. 1477 */ 1478 pteval = ptep_get_and_clear(mm, address, pte); 1479 1480 set_tlb_ubc_flush_pending(mm, page, pte_dirty(pteval)); 1481 } else { 1482 pteval = ptep_clear_flush(vma, address, pte); 1483 } 1484 1485 /* Move the dirty bit to the physical page now the pte is gone. */ 1486 if (pte_dirty(pteval)) 1487 set_page_dirty(page); 1488 1489 /* Update high watermark before we lower rss */ 1490 update_hiwater_rss(mm); 1491 1492 if (PageHWPoison(page) && !(flags & TTU_IGNORE_HWPOISON)) { 1493 if (PageHuge(page)) { 1494 hugetlb_count_sub(1 << compound_order(page), mm); 1495 } else { 1496 dec_mm_counter(mm, mm_counter(page)); 1497 } 1498 set_pte_at(mm, address, pte, 1499 swp_entry_to_pte(make_hwpoison_entry(page))); 1500 } else if (pte_unused(pteval)) { 1501 /* 1502 * The guest indicated that the page content is of no 1503 * interest anymore. Simply discard the pte, vmscan 1504 * will take care of the rest. 1505 */ 1506 dec_mm_counter(mm, mm_counter(page)); 1507 } else if (IS_ENABLED(CONFIG_MIGRATION) && (flags & TTU_MIGRATION)) { 1508 swp_entry_t entry; 1509 pte_t swp_pte; 1510 /* 1511 * Store the pfn of the page in a special migration 1512 * pte. do_swap_page() will wait until the migration 1513 * pte is removed and then restart fault handling. 1514 */ 1515 entry = make_migration_entry(page, pte_write(pteval)); 1516 swp_pte = swp_entry_to_pte(entry); 1517 if (pte_soft_dirty(pteval)) 1518 swp_pte = pte_swp_mksoft_dirty(swp_pte); 1519 set_pte_at(mm, address, pte, swp_pte); 1520 } else if (PageAnon(page)) { 1521 swp_entry_t entry = { .val = page_private(page) }; 1522 pte_t swp_pte; 1523 /* 1524 * Store the swap location in the pte. 1525 * See handle_pte_fault() ... 1526 */ 1527 VM_BUG_ON_PAGE(!PageSwapCache(page), page); 1528 1529 if (!PageDirty(page) && (flags & TTU_LZFREE)) { 1530 /* It's a freeable page by MADV_FREE */ 1531 dec_mm_counter(mm, MM_ANONPAGES); 1532 rp->lazyfreed++; 1533 goto discard; 1534 } 1535 1536 if (swap_duplicate(entry) < 0) { 1537 set_pte_at(mm, address, pte, pteval); 1538 ret = SWAP_FAIL; 1539 goto out_unmap; 1540 } 1541 if (list_empty(&mm->mmlist)) { 1542 spin_lock(&mmlist_lock); 1543 if (list_empty(&mm->mmlist)) 1544 list_add(&mm->mmlist, &init_mm.mmlist); 1545 spin_unlock(&mmlist_lock); 1546 } 1547 dec_mm_counter(mm, MM_ANONPAGES); 1548 inc_mm_counter(mm, MM_SWAPENTS); 1549 swp_pte = swp_entry_to_pte(entry); 1550 if (pte_soft_dirty(pteval)) 1551 swp_pte = pte_swp_mksoft_dirty(swp_pte); 1552 set_pte_at(mm, address, pte, swp_pte); 1553 } else 1554 dec_mm_counter(mm, mm_counter_file(page)); 1555 1556 discard: 1557 page_remove_rmap(page, PageHuge(page)); 1558 page_cache_release(page); 1559 1560 out_unmap: 1561 pte_unmap_unlock(pte, ptl); 1562 if (ret != SWAP_FAIL && ret != SWAP_MLOCK && !(flags & TTU_MUNLOCK)) 1563 mmu_notifier_invalidate_page(mm, address); 1564 out: 1565 return ret; 1566 } 1567 1568 bool is_vma_temporary_stack(struct vm_area_struct *vma) 1569 { 1570 int maybe_stack = vma->vm_flags & (VM_GROWSDOWN | VM_GROWSUP); 1571 1572 if (!maybe_stack) 1573 return false; 1574 1575 if ((vma->vm_flags & VM_STACK_INCOMPLETE_SETUP) == 1576 VM_STACK_INCOMPLETE_SETUP) 1577 return true; 1578 1579 return false; 1580 } 1581 1582 static bool invalid_migration_vma(struct vm_area_struct *vma, void *arg) 1583 { 1584 return is_vma_temporary_stack(vma); 1585 } 1586 1587 static int page_mapcount_is_zero(struct page *page) 1588 { 1589 return !page_mapcount(page); 1590 } 1591 1592 /** 1593 * try_to_unmap - try to remove all page table mappings to a page 1594 * @page: the page to get unmapped 1595 * @flags: action and flags 1596 * 1597 * Tries to remove all the page table entries which are mapping this 1598 * page, used in the pageout path. Caller must hold the page lock. 1599 * Return values are: 1600 * 1601 * SWAP_SUCCESS - we succeeded in removing all mappings 1602 * SWAP_AGAIN - we missed a mapping, try again later 1603 * SWAP_FAIL - the page is unswappable 1604 * SWAP_MLOCK - page is mlocked. 1605 */ 1606 int try_to_unmap(struct page *page, enum ttu_flags flags) 1607 { 1608 int ret; 1609 struct rmap_private rp = { 1610 .flags = flags, 1611 .lazyfreed = 0, 1612 }; 1613 1614 struct rmap_walk_control rwc = { 1615 .rmap_one = try_to_unmap_one, 1616 .arg = &rp, 1617 .done = page_mapcount_is_zero, 1618 .anon_lock = page_lock_anon_vma_read, 1619 }; 1620 1621 /* 1622 * During exec, a temporary VMA is setup and later moved. 1623 * The VMA is moved under the anon_vma lock but not the 1624 * page tables leading to a race where migration cannot 1625 * find the migration ptes. Rather than increasing the 1626 * locking requirements of exec(), migration skips 1627 * temporary VMAs until after exec() completes. 1628 */ 1629 if ((flags & TTU_MIGRATION) && !PageKsm(page) && PageAnon(page)) 1630 rwc.invalid_vma = invalid_migration_vma; 1631 1632 if (flags & TTU_RMAP_LOCKED) 1633 ret = rmap_walk_locked(page, &rwc); 1634 else 1635 ret = rmap_walk(page, &rwc); 1636 1637 if (ret != SWAP_MLOCK && !page_mapcount(page)) { 1638 ret = SWAP_SUCCESS; 1639 if (rp.lazyfreed && !PageDirty(page)) 1640 ret = SWAP_LZFREE; 1641 } 1642 return ret; 1643 } 1644 1645 static int page_not_mapped(struct page *page) 1646 { 1647 return !page_mapped(page); 1648 }; 1649 1650 /** 1651 * try_to_munlock - try to munlock a page 1652 * @page: the page to be munlocked 1653 * 1654 * Called from munlock code. Checks all of the VMAs mapping the page 1655 * to make sure nobody else has this page mlocked. The page will be 1656 * returned with PG_mlocked cleared if no other vmas have it mlocked. 1657 * 1658 * Return values are: 1659 * 1660 * SWAP_AGAIN - no vma is holding page mlocked, or, 1661 * SWAP_AGAIN - page mapped in mlocked vma -- couldn't acquire mmap sem 1662 * SWAP_FAIL - page cannot be located at present 1663 * SWAP_MLOCK - page is now mlocked. 1664 */ 1665 int try_to_munlock(struct page *page) 1666 { 1667 int ret; 1668 struct rmap_private rp = { 1669 .flags = TTU_MUNLOCK, 1670 .lazyfreed = 0, 1671 }; 1672 1673 struct rmap_walk_control rwc = { 1674 .rmap_one = try_to_unmap_one, 1675 .arg = &rp, 1676 .done = page_not_mapped, 1677 .anon_lock = page_lock_anon_vma_read, 1678 1679 }; 1680 1681 VM_BUG_ON_PAGE(!PageLocked(page) || PageLRU(page), page); 1682 1683 ret = rmap_walk(page, &rwc); 1684 return ret; 1685 } 1686 1687 void __put_anon_vma(struct anon_vma *anon_vma) 1688 { 1689 struct anon_vma *root = anon_vma->root; 1690 1691 anon_vma_free(anon_vma); 1692 if (root != anon_vma && atomic_dec_and_test(&root->refcount)) 1693 anon_vma_free(root); 1694 } 1695 1696 static struct anon_vma *rmap_walk_anon_lock(struct page *page, 1697 struct rmap_walk_control *rwc) 1698 { 1699 struct anon_vma *anon_vma; 1700 1701 if (rwc->anon_lock) 1702 return rwc->anon_lock(page); 1703 1704 /* 1705 * Note: remove_migration_ptes() cannot use page_lock_anon_vma_read() 1706 * because that depends on page_mapped(); but not all its usages 1707 * are holding mmap_sem. Users without mmap_sem are required to 1708 * take a reference count to prevent the anon_vma disappearing 1709 */ 1710 anon_vma = page_anon_vma(page); 1711 if (!anon_vma) 1712 return NULL; 1713 1714 anon_vma_lock_read(anon_vma); 1715 return anon_vma; 1716 } 1717 1718 /* 1719 * rmap_walk_anon - do something to anonymous page using the object-based 1720 * rmap method 1721 * @page: the page to be handled 1722 * @rwc: control variable according to each walk type 1723 * 1724 * Find all the mappings of a page using the mapping pointer and the vma chains 1725 * contained in the anon_vma struct it points to. 1726 * 1727 * When called from try_to_munlock(), the mmap_sem of the mm containing the vma 1728 * where the page was found will be held for write. So, we won't recheck 1729 * vm_flags for that VMA. That should be OK, because that vma shouldn't be 1730 * LOCKED. 1731 */ 1732 static int rmap_walk_anon(struct page *page, struct rmap_walk_control *rwc, 1733 bool locked) 1734 { 1735 struct anon_vma *anon_vma; 1736 pgoff_t pgoff; 1737 struct anon_vma_chain *avc; 1738 int ret = SWAP_AGAIN; 1739 1740 if (locked) { 1741 anon_vma = page_anon_vma(page); 1742 /* anon_vma disappear under us? */ 1743 VM_BUG_ON_PAGE(!anon_vma, page); 1744 } else { 1745 anon_vma = rmap_walk_anon_lock(page, rwc); 1746 } 1747 if (!anon_vma) 1748 return ret; 1749 1750 pgoff = page_to_pgoff(page); 1751 anon_vma_interval_tree_foreach(avc, &anon_vma->rb_root, pgoff, pgoff) { 1752 struct vm_area_struct *vma = avc->vma; 1753 unsigned long address = vma_address(page, vma); 1754 1755 cond_resched(); 1756 1757 if (rwc->invalid_vma && rwc->invalid_vma(vma, rwc->arg)) 1758 continue; 1759 1760 ret = rwc->rmap_one(page, vma, address, rwc->arg); 1761 if (ret != SWAP_AGAIN) 1762 break; 1763 if (rwc->done && rwc->done(page)) 1764 break; 1765 } 1766 1767 if (!locked) 1768 anon_vma_unlock_read(anon_vma); 1769 return ret; 1770 } 1771 1772 /* 1773 * rmap_walk_file - do something to file page using the object-based rmap method 1774 * @page: the page to be handled 1775 * @rwc: control variable according to each walk type 1776 * 1777 * Find all the mappings of a page using the mapping pointer and the vma chains 1778 * contained in the address_space struct it points to. 1779 * 1780 * When called from try_to_munlock(), the mmap_sem of the mm containing the vma 1781 * where the page was found will be held for write. So, we won't recheck 1782 * vm_flags for that VMA. That should be OK, because that vma shouldn't be 1783 * LOCKED. 1784 */ 1785 static int rmap_walk_file(struct page *page, struct rmap_walk_control *rwc, 1786 bool locked) 1787 { 1788 struct address_space *mapping = page_mapping(page); 1789 pgoff_t pgoff; 1790 struct vm_area_struct *vma; 1791 int ret = SWAP_AGAIN; 1792 1793 /* 1794 * The page lock not only makes sure that page->mapping cannot 1795 * suddenly be NULLified by truncation, it makes sure that the 1796 * structure at mapping cannot be freed and reused yet, 1797 * so we can safely take mapping->i_mmap_rwsem. 1798 */ 1799 VM_BUG_ON_PAGE(!PageLocked(page), page); 1800 1801 if (!mapping) 1802 return ret; 1803 1804 pgoff = page_to_pgoff(page); 1805 if (!locked) 1806 i_mmap_lock_read(mapping); 1807 vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff, pgoff) { 1808 unsigned long address = vma_address(page, vma); 1809 1810 cond_resched(); 1811 1812 if (rwc->invalid_vma && rwc->invalid_vma(vma, rwc->arg)) 1813 continue; 1814 1815 ret = rwc->rmap_one(page, vma, address, rwc->arg); 1816 if (ret != SWAP_AGAIN) 1817 goto done; 1818 if (rwc->done && rwc->done(page)) 1819 goto done; 1820 } 1821 1822 done: 1823 if (!locked) 1824 i_mmap_unlock_read(mapping); 1825 return ret; 1826 } 1827 1828 int rmap_walk(struct page *page, struct rmap_walk_control *rwc) 1829 { 1830 if (unlikely(PageKsm(page))) 1831 return rmap_walk_ksm(page, rwc); 1832 else if (PageAnon(page)) 1833 return rmap_walk_anon(page, rwc, false); 1834 else 1835 return rmap_walk_file(page, rwc, false); 1836 } 1837 1838 /* Like rmap_walk, but caller holds relevant rmap lock */ 1839 int rmap_walk_locked(struct page *page, struct rmap_walk_control *rwc) 1840 { 1841 /* no ksm support for now */ 1842 VM_BUG_ON_PAGE(PageKsm(page), page); 1843 if (PageAnon(page)) 1844 return rmap_walk_anon(page, rwc, true); 1845 else 1846 return rmap_walk_file(page, rwc, true); 1847 } 1848 1849 #ifdef CONFIG_HUGETLB_PAGE 1850 /* 1851 * The following three functions are for anonymous (private mapped) hugepages. 1852 * Unlike common anonymous pages, anonymous hugepages have no accounting code 1853 * and no lru code, because we handle hugepages differently from common pages. 1854 */ 1855 static void __hugepage_set_anon_rmap(struct page *page, 1856 struct vm_area_struct *vma, unsigned long address, int exclusive) 1857 { 1858 struct anon_vma *anon_vma = vma->anon_vma; 1859 1860 BUG_ON(!anon_vma); 1861 1862 if (PageAnon(page)) 1863 return; 1864 if (!exclusive) 1865 anon_vma = anon_vma->root; 1866 1867 anon_vma = (void *) anon_vma + PAGE_MAPPING_ANON; 1868 page->mapping = (struct address_space *) anon_vma; 1869 page->index = linear_page_index(vma, address); 1870 } 1871 1872 void hugepage_add_anon_rmap(struct page *page, 1873 struct vm_area_struct *vma, unsigned long address) 1874 { 1875 struct anon_vma *anon_vma = vma->anon_vma; 1876 int first; 1877 1878 BUG_ON(!PageLocked(page)); 1879 BUG_ON(!anon_vma); 1880 /* address might be in next vma when migration races vma_adjust */ 1881 first = atomic_inc_and_test(compound_mapcount_ptr(page)); 1882 if (first) 1883 __hugepage_set_anon_rmap(page, vma, address, 0); 1884 } 1885 1886 void hugepage_add_new_anon_rmap(struct page *page, 1887 struct vm_area_struct *vma, unsigned long address) 1888 { 1889 BUG_ON(address < vma->vm_start || address >= vma->vm_end); 1890 atomic_set(compound_mapcount_ptr(page), 0); 1891 __hugepage_set_anon_rmap(page, vma, address, 1); 1892 } 1893 #endif /* CONFIG_HUGETLB_PAGE */ 1894