1 /* 2 * mm/rmap.c - physical to virtual reverse mappings 3 * 4 * Copyright 2001, Rik van Riel <riel@conectiva.com.br> 5 * Released under the General Public License (GPL). 6 * 7 * Simple, low overhead reverse mapping scheme. 8 * Please try to keep this thing as modular as possible. 9 * 10 * Provides methods for unmapping each kind of mapped page: 11 * the anon methods track anonymous pages, and 12 * the file methods track pages belonging to an inode. 13 * 14 * Original design by Rik van Riel <riel@conectiva.com.br> 2001 15 * File methods by Dave McCracken <dmccr@us.ibm.com> 2003, 2004 16 * Anonymous methods by Andrea Arcangeli <andrea@suse.de> 2004 17 * Contributions by Hugh Dickins 2003, 2004 18 */ 19 20 /* 21 * Lock ordering in mm: 22 * 23 * inode->i_mutex (while writing or truncating, not reading or faulting) 24 * mm->mmap_sem 25 * page->flags PG_locked (lock_page) 26 * hugetlbfs_i_mmap_rwsem_key (in huge_pmd_share) 27 * mapping->i_mmap_rwsem 28 * anon_vma->rwsem 29 * mm->page_table_lock or pte_lock 30 * zone_lru_lock (in mark_page_accessed, isolate_lru_page) 31 * swap_lock (in swap_duplicate, swap_info_get) 32 * mmlist_lock (in mmput, drain_mmlist and others) 33 * mapping->private_lock (in __set_page_dirty_buffers) 34 * mem_cgroup_{begin,end}_page_stat (memcg->move_lock) 35 * mapping->tree_lock (widely used) 36 * inode->i_lock (in set_page_dirty's __mark_inode_dirty) 37 * bdi.wb->list_lock (in set_page_dirty's __mark_inode_dirty) 38 * sb_lock (within inode_lock in fs/fs-writeback.c) 39 * mapping->tree_lock (widely used, in set_page_dirty, 40 * in arch-dependent flush_dcache_mmap_lock, 41 * within bdi.wb->list_lock in __sync_single_inode) 42 * 43 * anon_vma->rwsem,mapping->i_mutex (memory_failure, collect_procs_anon) 44 * ->tasklist_lock 45 * pte map lock 46 */ 47 48 #include <linux/mm.h> 49 #include <linux/sched/mm.h> 50 #include <linux/sched/task.h> 51 #include <linux/pagemap.h> 52 #include <linux/swap.h> 53 #include <linux/swapops.h> 54 #include <linux/slab.h> 55 #include <linux/init.h> 56 #include <linux/ksm.h> 57 #include <linux/rmap.h> 58 #include <linux/rcupdate.h> 59 #include <linux/export.h> 60 #include <linux/memcontrol.h> 61 #include <linux/mmu_notifier.h> 62 #include <linux/migrate.h> 63 #include <linux/hugetlb.h> 64 #include <linux/backing-dev.h> 65 #include <linux/page_idle.h> 66 67 #include <asm/tlbflush.h> 68 69 #include <trace/events/tlb.h> 70 71 #include "internal.h" 72 73 static struct kmem_cache *anon_vma_cachep; 74 static struct kmem_cache *anon_vma_chain_cachep; 75 76 static inline struct anon_vma *anon_vma_alloc(void) 77 { 78 struct anon_vma *anon_vma; 79 80 anon_vma = kmem_cache_alloc(anon_vma_cachep, GFP_KERNEL); 81 if (anon_vma) { 82 atomic_set(&anon_vma->refcount, 1); 83 anon_vma->degree = 1; /* Reference for first vma */ 84 anon_vma->parent = anon_vma; 85 /* 86 * Initialise the anon_vma root to point to itself. If called 87 * from fork, the root will be reset to the parents anon_vma. 88 */ 89 anon_vma->root = anon_vma; 90 } 91 92 return anon_vma; 93 } 94 95 static inline void anon_vma_free(struct anon_vma *anon_vma) 96 { 97 VM_BUG_ON(atomic_read(&anon_vma->refcount)); 98 99 /* 100 * Synchronize against page_lock_anon_vma_read() such that 101 * we can safely hold the lock without the anon_vma getting 102 * freed. 103 * 104 * Relies on the full mb implied by the atomic_dec_and_test() from 105 * put_anon_vma() against the acquire barrier implied by 106 * down_read_trylock() from page_lock_anon_vma_read(). This orders: 107 * 108 * page_lock_anon_vma_read() VS put_anon_vma() 109 * down_read_trylock() atomic_dec_and_test() 110 * LOCK MB 111 * atomic_read() rwsem_is_locked() 112 * 113 * LOCK should suffice since the actual taking of the lock must 114 * happen _before_ what follows. 115 */ 116 might_sleep(); 117 if (rwsem_is_locked(&anon_vma->root->rwsem)) { 118 anon_vma_lock_write(anon_vma); 119 anon_vma_unlock_write(anon_vma); 120 } 121 122 kmem_cache_free(anon_vma_cachep, anon_vma); 123 } 124 125 static inline struct anon_vma_chain *anon_vma_chain_alloc(gfp_t gfp) 126 { 127 return kmem_cache_alloc(anon_vma_chain_cachep, gfp); 128 } 129 130 static void anon_vma_chain_free(struct anon_vma_chain *anon_vma_chain) 131 { 132 kmem_cache_free(anon_vma_chain_cachep, anon_vma_chain); 133 } 134 135 static void anon_vma_chain_link(struct vm_area_struct *vma, 136 struct anon_vma_chain *avc, 137 struct anon_vma *anon_vma) 138 { 139 avc->vma = vma; 140 avc->anon_vma = anon_vma; 141 list_add(&avc->same_vma, &vma->anon_vma_chain); 142 anon_vma_interval_tree_insert(avc, &anon_vma->rb_root); 143 } 144 145 /** 146 * __anon_vma_prepare - attach an anon_vma to a memory region 147 * @vma: the memory region in question 148 * 149 * This makes sure the memory mapping described by 'vma' has 150 * an 'anon_vma' attached to it, so that we can associate the 151 * anonymous pages mapped into it with that anon_vma. 152 * 153 * The common case will be that we already have one, which 154 * is handled inline by anon_vma_prepare(). But if 155 * not we either need to find an adjacent mapping that we 156 * can re-use the anon_vma from (very common when the only 157 * reason for splitting a vma has been mprotect()), or we 158 * allocate a new one. 159 * 160 * Anon-vma allocations are very subtle, because we may have 161 * optimistically looked up an anon_vma in page_lock_anon_vma_read() 162 * and that may actually touch the spinlock even in the newly 163 * allocated vma (it depends on RCU to make sure that the 164 * anon_vma isn't actually destroyed). 165 * 166 * As a result, we need to do proper anon_vma locking even 167 * for the new allocation. At the same time, we do not want 168 * to do any locking for the common case of already having 169 * an anon_vma. 170 * 171 * This must be called with the mmap_sem held for reading. 172 */ 173 int __anon_vma_prepare(struct vm_area_struct *vma) 174 { 175 struct mm_struct *mm = vma->vm_mm; 176 struct anon_vma *anon_vma, *allocated; 177 struct anon_vma_chain *avc; 178 179 might_sleep(); 180 181 avc = anon_vma_chain_alloc(GFP_KERNEL); 182 if (!avc) 183 goto out_enomem; 184 185 anon_vma = find_mergeable_anon_vma(vma); 186 allocated = NULL; 187 if (!anon_vma) { 188 anon_vma = anon_vma_alloc(); 189 if (unlikely(!anon_vma)) 190 goto out_enomem_free_avc; 191 allocated = anon_vma; 192 } 193 194 anon_vma_lock_write(anon_vma); 195 /* page_table_lock to protect against threads */ 196 spin_lock(&mm->page_table_lock); 197 if (likely(!vma->anon_vma)) { 198 vma->anon_vma = anon_vma; 199 anon_vma_chain_link(vma, avc, anon_vma); 200 /* vma reference or self-parent link for new root */ 201 anon_vma->degree++; 202 allocated = NULL; 203 avc = NULL; 204 } 205 spin_unlock(&mm->page_table_lock); 206 anon_vma_unlock_write(anon_vma); 207 208 if (unlikely(allocated)) 209 put_anon_vma(allocated); 210 if (unlikely(avc)) 211 anon_vma_chain_free(avc); 212 213 return 0; 214 215 out_enomem_free_avc: 216 anon_vma_chain_free(avc); 217 out_enomem: 218 return -ENOMEM; 219 } 220 221 /* 222 * This is a useful helper function for locking the anon_vma root as 223 * we traverse the vma->anon_vma_chain, looping over anon_vma's that 224 * have the same vma. 225 * 226 * Such anon_vma's should have the same root, so you'd expect to see 227 * just a single mutex_lock for the whole traversal. 228 */ 229 static inline struct anon_vma *lock_anon_vma_root(struct anon_vma *root, struct anon_vma *anon_vma) 230 { 231 struct anon_vma *new_root = anon_vma->root; 232 if (new_root != root) { 233 if (WARN_ON_ONCE(root)) 234 up_write(&root->rwsem); 235 root = new_root; 236 down_write(&root->rwsem); 237 } 238 return root; 239 } 240 241 static inline void unlock_anon_vma_root(struct anon_vma *root) 242 { 243 if (root) 244 up_write(&root->rwsem); 245 } 246 247 /* 248 * Attach the anon_vmas from src to dst. 249 * Returns 0 on success, -ENOMEM on failure. 250 * 251 * If dst->anon_vma is NULL this function tries to find and reuse existing 252 * anon_vma which has no vmas and only one child anon_vma. This prevents 253 * degradation of anon_vma hierarchy to endless linear chain in case of 254 * constantly forking task. On the other hand, an anon_vma with more than one 255 * child isn't reused even if there was no alive vma, thus rmap walker has a 256 * good chance of avoiding scanning the whole hierarchy when it searches where 257 * page is mapped. 258 */ 259 int anon_vma_clone(struct vm_area_struct *dst, struct vm_area_struct *src) 260 { 261 struct anon_vma_chain *avc, *pavc; 262 struct anon_vma *root = NULL; 263 264 list_for_each_entry_reverse(pavc, &src->anon_vma_chain, same_vma) { 265 struct anon_vma *anon_vma; 266 267 avc = anon_vma_chain_alloc(GFP_NOWAIT | __GFP_NOWARN); 268 if (unlikely(!avc)) { 269 unlock_anon_vma_root(root); 270 root = NULL; 271 avc = anon_vma_chain_alloc(GFP_KERNEL); 272 if (!avc) 273 goto enomem_failure; 274 } 275 anon_vma = pavc->anon_vma; 276 root = lock_anon_vma_root(root, anon_vma); 277 anon_vma_chain_link(dst, avc, anon_vma); 278 279 /* 280 * Reuse existing anon_vma if its degree lower than two, 281 * that means it has no vma and only one anon_vma child. 282 * 283 * Do not chose parent anon_vma, otherwise first child 284 * will always reuse it. Root anon_vma is never reused: 285 * it has self-parent reference and at least one child. 286 */ 287 if (!dst->anon_vma && anon_vma != src->anon_vma && 288 anon_vma->degree < 2) 289 dst->anon_vma = anon_vma; 290 } 291 if (dst->anon_vma) 292 dst->anon_vma->degree++; 293 unlock_anon_vma_root(root); 294 return 0; 295 296 enomem_failure: 297 /* 298 * dst->anon_vma is dropped here otherwise its degree can be incorrectly 299 * decremented in unlink_anon_vmas(). 300 * We can safely do this because callers of anon_vma_clone() don't care 301 * about dst->anon_vma if anon_vma_clone() failed. 302 */ 303 dst->anon_vma = NULL; 304 unlink_anon_vmas(dst); 305 return -ENOMEM; 306 } 307 308 /* 309 * Attach vma to its own anon_vma, as well as to the anon_vmas that 310 * the corresponding VMA in the parent process is attached to. 311 * Returns 0 on success, non-zero on failure. 312 */ 313 int anon_vma_fork(struct vm_area_struct *vma, struct vm_area_struct *pvma) 314 { 315 struct anon_vma_chain *avc; 316 struct anon_vma *anon_vma; 317 int error; 318 319 /* Don't bother if the parent process has no anon_vma here. */ 320 if (!pvma->anon_vma) 321 return 0; 322 323 /* Drop inherited anon_vma, we'll reuse existing or allocate new. */ 324 vma->anon_vma = NULL; 325 326 /* 327 * First, attach the new VMA to the parent VMA's anon_vmas, 328 * so rmap can find non-COWed pages in child processes. 329 */ 330 error = anon_vma_clone(vma, pvma); 331 if (error) 332 return error; 333 334 /* An existing anon_vma has been reused, all done then. */ 335 if (vma->anon_vma) 336 return 0; 337 338 /* Then add our own anon_vma. */ 339 anon_vma = anon_vma_alloc(); 340 if (!anon_vma) 341 goto out_error; 342 avc = anon_vma_chain_alloc(GFP_KERNEL); 343 if (!avc) 344 goto out_error_free_anon_vma; 345 346 /* 347 * The root anon_vma's spinlock is the lock actually used when we 348 * lock any of the anon_vmas in this anon_vma tree. 349 */ 350 anon_vma->root = pvma->anon_vma->root; 351 anon_vma->parent = pvma->anon_vma; 352 /* 353 * With refcounts, an anon_vma can stay around longer than the 354 * process it belongs to. The root anon_vma needs to be pinned until 355 * this anon_vma is freed, because the lock lives in the root. 356 */ 357 get_anon_vma(anon_vma->root); 358 /* Mark this anon_vma as the one where our new (COWed) pages go. */ 359 vma->anon_vma = anon_vma; 360 anon_vma_lock_write(anon_vma); 361 anon_vma_chain_link(vma, avc, anon_vma); 362 anon_vma->parent->degree++; 363 anon_vma_unlock_write(anon_vma); 364 365 return 0; 366 367 out_error_free_anon_vma: 368 put_anon_vma(anon_vma); 369 out_error: 370 unlink_anon_vmas(vma); 371 return -ENOMEM; 372 } 373 374 void unlink_anon_vmas(struct vm_area_struct *vma) 375 { 376 struct anon_vma_chain *avc, *next; 377 struct anon_vma *root = NULL; 378 379 /* 380 * Unlink each anon_vma chained to the VMA. This list is ordered 381 * from newest to oldest, ensuring the root anon_vma gets freed last. 382 */ 383 list_for_each_entry_safe(avc, next, &vma->anon_vma_chain, same_vma) { 384 struct anon_vma *anon_vma = avc->anon_vma; 385 386 root = lock_anon_vma_root(root, anon_vma); 387 anon_vma_interval_tree_remove(avc, &anon_vma->rb_root); 388 389 /* 390 * Leave empty anon_vmas on the list - we'll need 391 * to free them outside the lock. 392 */ 393 if (RB_EMPTY_ROOT(&anon_vma->rb_root)) { 394 anon_vma->parent->degree--; 395 continue; 396 } 397 398 list_del(&avc->same_vma); 399 anon_vma_chain_free(avc); 400 } 401 if (vma->anon_vma) 402 vma->anon_vma->degree--; 403 unlock_anon_vma_root(root); 404 405 /* 406 * Iterate the list once more, it now only contains empty and unlinked 407 * anon_vmas, destroy them. Could not do before due to __put_anon_vma() 408 * needing to write-acquire the anon_vma->root->rwsem. 409 */ 410 list_for_each_entry_safe(avc, next, &vma->anon_vma_chain, same_vma) { 411 struct anon_vma *anon_vma = avc->anon_vma; 412 413 VM_WARN_ON(anon_vma->degree); 414 put_anon_vma(anon_vma); 415 416 list_del(&avc->same_vma); 417 anon_vma_chain_free(avc); 418 } 419 } 420 421 static void anon_vma_ctor(void *data) 422 { 423 struct anon_vma *anon_vma = data; 424 425 init_rwsem(&anon_vma->rwsem); 426 atomic_set(&anon_vma->refcount, 0); 427 anon_vma->rb_root = RB_ROOT; 428 } 429 430 void __init anon_vma_init(void) 431 { 432 anon_vma_cachep = kmem_cache_create("anon_vma", sizeof(struct anon_vma), 433 0, SLAB_TYPESAFE_BY_RCU|SLAB_PANIC|SLAB_ACCOUNT, 434 anon_vma_ctor); 435 anon_vma_chain_cachep = KMEM_CACHE(anon_vma_chain, 436 SLAB_PANIC|SLAB_ACCOUNT); 437 } 438 439 /* 440 * Getting a lock on a stable anon_vma from a page off the LRU is tricky! 441 * 442 * Since there is no serialization what so ever against page_remove_rmap() 443 * the best this function can do is return a locked anon_vma that might 444 * have been relevant to this page. 445 * 446 * The page might have been remapped to a different anon_vma or the anon_vma 447 * returned may already be freed (and even reused). 448 * 449 * In case it was remapped to a different anon_vma, the new anon_vma will be a 450 * child of the old anon_vma, and the anon_vma lifetime rules will therefore 451 * ensure that any anon_vma obtained from the page will still be valid for as 452 * long as we observe page_mapped() [ hence all those page_mapped() tests ]. 453 * 454 * All users of this function must be very careful when walking the anon_vma 455 * chain and verify that the page in question is indeed mapped in it 456 * [ something equivalent to page_mapped_in_vma() ]. 457 * 458 * Since anon_vma's slab is DESTROY_BY_RCU and we know from page_remove_rmap() 459 * that the anon_vma pointer from page->mapping is valid if there is a 460 * mapcount, we can dereference the anon_vma after observing those. 461 */ 462 struct anon_vma *page_get_anon_vma(struct page *page) 463 { 464 struct anon_vma *anon_vma = NULL; 465 unsigned long anon_mapping; 466 467 rcu_read_lock(); 468 anon_mapping = (unsigned long)READ_ONCE(page->mapping); 469 if ((anon_mapping & PAGE_MAPPING_FLAGS) != PAGE_MAPPING_ANON) 470 goto out; 471 if (!page_mapped(page)) 472 goto out; 473 474 anon_vma = (struct anon_vma *) (anon_mapping - PAGE_MAPPING_ANON); 475 if (!atomic_inc_not_zero(&anon_vma->refcount)) { 476 anon_vma = NULL; 477 goto out; 478 } 479 480 /* 481 * If this page is still mapped, then its anon_vma cannot have been 482 * freed. But if it has been unmapped, we have no security against the 483 * anon_vma structure being freed and reused (for another anon_vma: 484 * SLAB_TYPESAFE_BY_RCU guarantees that - so the atomic_inc_not_zero() 485 * above cannot corrupt). 486 */ 487 if (!page_mapped(page)) { 488 rcu_read_unlock(); 489 put_anon_vma(anon_vma); 490 return NULL; 491 } 492 out: 493 rcu_read_unlock(); 494 495 return anon_vma; 496 } 497 498 /* 499 * Similar to page_get_anon_vma() except it locks the anon_vma. 500 * 501 * Its a little more complex as it tries to keep the fast path to a single 502 * atomic op -- the trylock. If we fail the trylock, we fall back to getting a 503 * reference like with page_get_anon_vma() and then block on the mutex. 504 */ 505 struct anon_vma *page_lock_anon_vma_read(struct page *page) 506 { 507 struct anon_vma *anon_vma = NULL; 508 struct anon_vma *root_anon_vma; 509 unsigned long anon_mapping; 510 511 rcu_read_lock(); 512 anon_mapping = (unsigned long)READ_ONCE(page->mapping); 513 if ((anon_mapping & PAGE_MAPPING_FLAGS) != PAGE_MAPPING_ANON) 514 goto out; 515 if (!page_mapped(page)) 516 goto out; 517 518 anon_vma = (struct anon_vma *) (anon_mapping - PAGE_MAPPING_ANON); 519 root_anon_vma = READ_ONCE(anon_vma->root); 520 if (down_read_trylock(&root_anon_vma->rwsem)) { 521 /* 522 * If the page is still mapped, then this anon_vma is still 523 * its anon_vma, and holding the mutex ensures that it will 524 * not go away, see anon_vma_free(). 525 */ 526 if (!page_mapped(page)) { 527 up_read(&root_anon_vma->rwsem); 528 anon_vma = NULL; 529 } 530 goto out; 531 } 532 533 /* trylock failed, we got to sleep */ 534 if (!atomic_inc_not_zero(&anon_vma->refcount)) { 535 anon_vma = NULL; 536 goto out; 537 } 538 539 if (!page_mapped(page)) { 540 rcu_read_unlock(); 541 put_anon_vma(anon_vma); 542 return NULL; 543 } 544 545 /* we pinned the anon_vma, its safe to sleep */ 546 rcu_read_unlock(); 547 anon_vma_lock_read(anon_vma); 548 549 if (atomic_dec_and_test(&anon_vma->refcount)) { 550 /* 551 * Oops, we held the last refcount, release the lock 552 * and bail -- can't simply use put_anon_vma() because 553 * we'll deadlock on the anon_vma_lock_write() recursion. 554 */ 555 anon_vma_unlock_read(anon_vma); 556 __put_anon_vma(anon_vma); 557 anon_vma = NULL; 558 } 559 560 return anon_vma; 561 562 out: 563 rcu_read_unlock(); 564 return anon_vma; 565 } 566 567 void page_unlock_anon_vma_read(struct anon_vma *anon_vma) 568 { 569 anon_vma_unlock_read(anon_vma); 570 } 571 572 #ifdef CONFIG_ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH 573 /* 574 * Flush TLB entries for recently unmapped pages from remote CPUs. It is 575 * important if a PTE was dirty when it was unmapped that it's flushed 576 * before any IO is initiated on the page to prevent lost writes. Similarly, 577 * it must be flushed before freeing to prevent data leakage. 578 */ 579 void try_to_unmap_flush(void) 580 { 581 struct tlbflush_unmap_batch *tlb_ubc = ¤t->tlb_ubc; 582 583 if (!tlb_ubc->flush_required) 584 return; 585 586 arch_tlbbatch_flush(&tlb_ubc->arch); 587 tlb_ubc->flush_required = false; 588 tlb_ubc->writable = false; 589 } 590 591 /* Flush iff there are potentially writable TLB entries that can race with IO */ 592 void try_to_unmap_flush_dirty(void) 593 { 594 struct tlbflush_unmap_batch *tlb_ubc = ¤t->tlb_ubc; 595 596 if (tlb_ubc->writable) 597 try_to_unmap_flush(); 598 } 599 600 static void set_tlb_ubc_flush_pending(struct mm_struct *mm, bool writable) 601 { 602 struct tlbflush_unmap_batch *tlb_ubc = ¤t->tlb_ubc; 603 604 arch_tlbbatch_add_mm(&tlb_ubc->arch, mm); 605 tlb_ubc->flush_required = true; 606 607 /* 608 * Ensure compiler does not re-order the setting of tlb_flush_batched 609 * before the PTE is cleared. 610 */ 611 barrier(); 612 mm->tlb_flush_batched = true; 613 614 /* 615 * If the PTE was dirty then it's best to assume it's writable. The 616 * caller must use try_to_unmap_flush_dirty() or try_to_unmap_flush() 617 * before the page is queued for IO. 618 */ 619 if (writable) 620 tlb_ubc->writable = true; 621 } 622 623 /* 624 * Returns true if the TLB flush should be deferred to the end of a batch of 625 * unmap operations to reduce IPIs. 626 */ 627 static bool should_defer_flush(struct mm_struct *mm, enum ttu_flags flags) 628 { 629 bool should_defer = false; 630 631 if (!(flags & TTU_BATCH_FLUSH)) 632 return false; 633 634 /* If remote CPUs need to be flushed then defer batch the flush */ 635 if (cpumask_any_but(mm_cpumask(mm), get_cpu()) < nr_cpu_ids) 636 should_defer = true; 637 put_cpu(); 638 639 return should_defer; 640 } 641 642 /* 643 * Reclaim unmaps pages under the PTL but do not flush the TLB prior to 644 * releasing the PTL if TLB flushes are batched. It's possible for a parallel 645 * operation such as mprotect or munmap to race between reclaim unmapping 646 * the page and flushing the page. If this race occurs, it potentially allows 647 * access to data via a stale TLB entry. Tracking all mm's that have TLB 648 * batching in flight would be expensive during reclaim so instead track 649 * whether TLB batching occurred in the past and if so then do a flush here 650 * if required. This will cost one additional flush per reclaim cycle paid 651 * by the first operation at risk such as mprotect and mumap. 652 * 653 * This must be called under the PTL so that an access to tlb_flush_batched 654 * that is potentially a "reclaim vs mprotect/munmap/etc" race will synchronise 655 * via the PTL. 656 */ 657 void flush_tlb_batched_pending(struct mm_struct *mm) 658 { 659 if (mm->tlb_flush_batched) { 660 flush_tlb_mm(mm); 661 662 /* 663 * Do not allow the compiler to re-order the clearing of 664 * tlb_flush_batched before the tlb is flushed. 665 */ 666 barrier(); 667 mm->tlb_flush_batched = false; 668 } 669 } 670 #else 671 static void set_tlb_ubc_flush_pending(struct mm_struct *mm, bool writable) 672 { 673 } 674 675 static bool should_defer_flush(struct mm_struct *mm, enum ttu_flags flags) 676 { 677 return false; 678 } 679 #endif /* CONFIG_ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH */ 680 681 /* 682 * At what user virtual address is page expected in vma? 683 * Caller should check the page is actually part of the vma. 684 */ 685 unsigned long page_address_in_vma(struct page *page, struct vm_area_struct *vma) 686 { 687 unsigned long address; 688 if (PageAnon(page)) { 689 struct anon_vma *page__anon_vma = page_anon_vma(page); 690 /* 691 * Note: swapoff's unuse_vma() is more efficient with this 692 * check, and needs it to match anon_vma when KSM is active. 693 */ 694 if (!vma->anon_vma || !page__anon_vma || 695 vma->anon_vma->root != page__anon_vma->root) 696 return -EFAULT; 697 } else if (page->mapping) { 698 if (!vma->vm_file || vma->vm_file->f_mapping != page->mapping) 699 return -EFAULT; 700 } else 701 return -EFAULT; 702 address = __vma_address(page, vma); 703 if (unlikely(address < vma->vm_start || address >= vma->vm_end)) 704 return -EFAULT; 705 return address; 706 } 707 708 pmd_t *mm_find_pmd(struct mm_struct *mm, unsigned long address) 709 { 710 pgd_t *pgd; 711 p4d_t *p4d; 712 pud_t *pud; 713 pmd_t *pmd = NULL; 714 pmd_t pmde; 715 716 pgd = pgd_offset(mm, address); 717 if (!pgd_present(*pgd)) 718 goto out; 719 720 p4d = p4d_offset(pgd, address); 721 if (!p4d_present(*p4d)) 722 goto out; 723 724 pud = pud_offset(p4d, address); 725 if (!pud_present(*pud)) 726 goto out; 727 728 pmd = pmd_offset(pud, address); 729 /* 730 * Some THP functions use the sequence pmdp_huge_clear_flush(), set_pmd_at() 731 * without holding anon_vma lock for write. So when looking for a 732 * genuine pmde (in which to find pte), test present and !THP together. 733 */ 734 pmde = *pmd; 735 barrier(); 736 if (!pmd_present(pmde) || pmd_trans_huge(pmde)) 737 pmd = NULL; 738 out: 739 return pmd; 740 } 741 742 struct page_referenced_arg { 743 int mapcount; 744 int referenced; 745 unsigned long vm_flags; 746 struct mem_cgroup *memcg; 747 }; 748 /* 749 * arg: page_referenced_arg will be passed 750 */ 751 static bool page_referenced_one(struct page *page, struct vm_area_struct *vma, 752 unsigned long address, void *arg) 753 { 754 struct page_referenced_arg *pra = arg; 755 struct page_vma_mapped_walk pvmw = { 756 .page = page, 757 .vma = vma, 758 .address = address, 759 }; 760 int referenced = 0; 761 762 while (page_vma_mapped_walk(&pvmw)) { 763 address = pvmw.address; 764 765 if (vma->vm_flags & VM_LOCKED) { 766 page_vma_mapped_walk_done(&pvmw); 767 pra->vm_flags |= VM_LOCKED; 768 return false; /* To break the loop */ 769 } 770 771 if (pvmw.pte) { 772 if (ptep_clear_flush_young_notify(vma, address, 773 pvmw.pte)) { 774 /* 775 * Don't treat a reference through 776 * a sequentially read mapping as such. 777 * If the page has been used in another mapping, 778 * we will catch it; if this other mapping is 779 * already gone, the unmap path will have set 780 * PG_referenced or activated the page. 781 */ 782 if (likely(!(vma->vm_flags & VM_SEQ_READ))) 783 referenced++; 784 } 785 } else if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) { 786 if (pmdp_clear_flush_young_notify(vma, address, 787 pvmw.pmd)) 788 referenced++; 789 } else { 790 /* unexpected pmd-mapped page? */ 791 WARN_ON_ONCE(1); 792 } 793 794 pra->mapcount--; 795 } 796 797 if (referenced) 798 clear_page_idle(page); 799 if (test_and_clear_page_young(page)) 800 referenced++; 801 802 if (referenced) { 803 pra->referenced++; 804 pra->vm_flags |= vma->vm_flags; 805 } 806 807 if (!pra->mapcount) 808 return false; /* To break the loop */ 809 810 return true; 811 } 812 813 static bool invalid_page_referenced_vma(struct vm_area_struct *vma, void *arg) 814 { 815 struct page_referenced_arg *pra = arg; 816 struct mem_cgroup *memcg = pra->memcg; 817 818 if (!mm_match_cgroup(vma->vm_mm, memcg)) 819 return true; 820 821 return false; 822 } 823 824 /** 825 * page_referenced - test if the page was referenced 826 * @page: the page to test 827 * @is_locked: caller holds lock on the page 828 * @memcg: target memory cgroup 829 * @vm_flags: collect encountered vma->vm_flags who actually referenced the page 830 * 831 * Quick test_and_clear_referenced for all mappings to a page, 832 * returns the number of ptes which referenced the page. 833 */ 834 int page_referenced(struct page *page, 835 int is_locked, 836 struct mem_cgroup *memcg, 837 unsigned long *vm_flags) 838 { 839 int we_locked = 0; 840 struct page_referenced_arg pra = { 841 .mapcount = total_mapcount(page), 842 .memcg = memcg, 843 }; 844 struct rmap_walk_control rwc = { 845 .rmap_one = page_referenced_one, 846 .arg = (void *)&pra, 847 .anon_lock = page_lock_anon_vma_read, 848 }; 849 850 *vm_flags = 0; 851 if (!page_mapped(page)) 852 return 0; 853 854 if (!page_rmapping(page)) 855 return 0; 856 857 if (!is_locked && (!PageAnon(page) || PageKsm(page))) { 858 we_locked = trylock_page(page); 859 if (!we_locked) 860 return 1; 861 } 862 863 /* 864 * If we are reclaiming on behalf of a cgroup, skip 865 * counting on behalf of references from different 866 * cgroups 867 */ 868 if (memcg) { 869 rwc.invalid_vma = invalid_page_referenced_vma; 870 } 871 872 rmap_walk(page, &rwc); 873 *vm_flags = pra.vm_flags; 874 875 if (we_locked) 876 unlock_page(page); 877 878 return pra.referenced; 879 } 880 881 static bool page_mkclean_one(struct page *page, struct vm_area_struct *vma, 882 unsigned long address, void *arg) 883 { 884 struct page_vma_mapped_walk pvmw = { 885 .page = page, 886 .vma = vma, 887 .address = address, 888 .flags = PVMW_SYNC, 889 }; 890 int *cleaned = arg; 891 892 while (page_vma_mapped_walk(&pvmw)) { 893 int ret = 0; 894 address = pvmw.address; 895 if (pvmw.pte) { 896 pte_t entry; 897 pte_t *pte = pvmw.pte; 898 899 if (!pte_dirty(*pte) && !pte_write(*pte)) 900 continue; 901 902 flush_cache_page(vma, address, pte_pfn(*pte)); 903 entry = ptep_clear_flush(vma, address, pte); 904 entry = pte_wrprotect(entry); 905 entry = pte_mkclean(entry); 906 set_pte_at(vma->vm_mm, address, pte, entry); 907 ret = 1; 908 } else { 909 #ifdef CONFIG_TRANSPARENT_HUGE_PAGECACHE 910 pmd_t *pmd = pvmw.pmd; 911 pmd_t entry; 912 913 if (!pmd_dirty(*pmd) && !pmd_write(*pmd)) 914 continue; 915 916 flush_cache_page(vma, address, page_to_pfn(page)); 917 entry = pmdp_huge_clear_flush(vma, address, pmd); 918 entry = pmd_wrprotect(entry); 919 entry = pmd_mkclean(entry); 920 set_pmd_at(vma->vm_mm, address, pmd, entry); 921 ret = 1; 922 #else 923 /* unexpected pmd-mapped page? */ 924 WARN_ON_ONCE(1); 925 #endif 926 } 927 928 if (ret) { 929 mmu_notifier_invalidate_page(vma->vm_mm, address); 930 (*cleaned)++; 931 } 932 } 933 934 return true; 935 } 936 937 static bool invalid_mkclean_vma(struct vm_area_struct *vma, void *arg) 938 { 939 if (vma->vm_flags & VM_SHARED) 940 return false; 941 942 return true; 943 } 944 945 int page_mkclean(struct page *page) 946 { 947 int cleaned = 0; 948 struct address_space *mapping; 949 struct rmap_walk_control rwc = { 950 .arg = (void *)&cleaned, 951 .rmap_one = page_mkclean_one, 952 .invalid_vma = invalid_mkclean_vma, 953 }; 954 955 BUG_ON(!PageLocked(page)); 956 957 if (!page_mapped(page)) 958 return 0; 959 960 mapping = page_mapping(page); 961 if (!mapping) 962 return 0; 963 964 rmap_walk(page, &rwc); 965 966 return cleaned; 967 } 968 EXPORT_SYMBOL_GPL(page_mkclean); 969 970 /** 971 * page_move_anon_rmap - move a page to our anon_vma 972 * @page: the page to move to our anon_vma 973 * @vma: the vma the page belongs to 974 * 975 * When a page belongs exclusively to one process after a COW event, 976 * that page can be moved into the anon_vma that belongs to just that 977 * process, so the rmap code will not search the parent or sibling 978 * processes. 979 */ 980 void page_move_anon_rmap(struct page *page, struct vm_area_struct *vma) 981 { 982 struct anon_vma *anon_vma = vma->anon_vma; 983 984 page = compound_head(page); 985 986 VM_BUG_ON_PAGE(!PageLocked(page), page); 987 VM_BUG_ON_VMA(!anon_vma, vma); 988 989 anon_vma = (void *) anon_vma + PAGE_MAPPING_ANON; 990 /* 991 * Ensure that anon_vma and the PAGE_MAPPING_ANON bit are written 992 * simultaneously, so a concurrent reader (eg page_referenced()'s 993 * PageAnon()) will not see one without the other. 994 */ 995 WRITE_ONCE(page->mapping, (struct address_space *) anon_vma); 996 } 997 998 /** 999 * __page_set_anon_rmap - set up new anonymous rmap 1000 * @page: Page to add to rmap 1001 * @vma: VM area to add page to. 1002 * @address: User virtual address of the mapping 1003 * @exclusive: the page is exclusively owned by the current process 1004 */ 1005 static void __page_set_anon_rmap(struct page *page, 1006 struct vm_area_struct *vma, unsigned long address, int exclusive) 1007 { 1008 struct anon_vma *anon_vma = vma->anon_vma; 1009 1010 BUG_ON(!anon_vma); 1011 1012 if (PageAnon(page)) 1013 return; 1014 1015 /* 1016 * If the page isn't exclusively mapped into this vma, 1017 * we must use the _oldest_ possible anon_vma for the 1018 * page mapping! 1019 */ 1020 if (!exclusive) 1021 anon_vma = anon_vma->root; 1022 1023 anon_vma = (void *) anon_vma + PAGE_MAPPING_ANON; 1024 page->mapping = (struct address_space *) anon_vma; 1025 page->index = linear_page_index(vma, address); 1026 } 1027 1028 /** 1029 * __page_check_anon_rmap - sanity check anonymous rmap addition 1030 * @page: the page to add the mapping to 1031 * @vma: the vm area in which the mapping is added 1032 * @address: the user virtual address mapped 1033 */ 1034 static void __page_check_anon_rmap(struct page *page, 1035 struct vm_area_struct *vma, unsigned long address) 1036 { 1037 #ifdef CONFIG_DEBUG_VM 1038 /* 1039 * The page's anon-rmap details (mapping and index) are guaranteed to 1040 * be set up correctly at this point. 1041 * 1042 * We have exclusion against page_add_anon_rmap because the caller 1043 * always holds the page locked, except if called from page_dup_rmap, 1044 * in which case the page is already known to be setup. 1045 * 1046 * We have exclusion against page_add_new_anon_rmap because those pages 1047 * are initially only visible via the pagetables, and the pte is locked 1048 * over the call to page_add_new_anon_rmap. 1049 */ 1050 BUG_ON(page_anon_vma(page)->root != vma->anon_vma->root); 1051 BUG_ON(page_to_pgoff(page) != linear_page_index(vma, address)); 1052 #endif 1053 } 1054 1055 /** 1056 * page_add_anon_rmap - add pte mapping to an anonymous page 1057 * @page: the page to add the mapping to 1058 * @vma: the vm area in which the mapping is added 1059 * @address: the user virtual address mapped 1060 * @compound: charge the page as compound or small page 1061 * 1062 * The caller needs to hold the pte lock, and the page must be locked in 1063 * the anon_vma case: to serialize mapping,index checking after setting, 1064 * and to ensure that PageAnon is not being upgraded racily to PageKsm 1065 * (but PageKsm is never downgraded to PageAnon). 1066 */ 1067 void page_add_anon_rmap(struct page *page, 1068 struct vm_area_struct *vma, unsigned long address, bool compound) 1069 { 1070 do_page_add_anon_rmap(page, vma, address, compound ? RMAP_COMPOUND : 0); 1071 } 1072 1073 /* 1074 * Special version of the above for do_swap_page, which often runs 1075 * into pages that are exclusively owned by the current process. 1076 * Everybody else should continue to use page_add_anon_rmap above. 1077 */ 1078 void do_page_add_anon_rmap(struct page *page, 1079 struct vm_area_struct *vma, unsigned long address, int flags) 1080 { 1081 bool compound = flags & RMAP_COMPOUND; 1082 bool first; 1083 1084 if (compound) { 1085 atomic_t *mapcount; 1086 VM_BUG_ON_PAGE(!PageLocked(page), page); 1087 VM_BUG_ON_PAGE(!PageTransHuge(page), page); 1088 mapcount = compound_mapcount_ptr(page); 1089 first = atomic_inc_and_test(mapcount); 1090 } else { 1091 first = atomic_inc_and_test(&page->_mapcount); 1092 } 1093 1094 if (first) { 1095 int nr = compound ? hpage_nr_pages(page) : 1; 1096 /* 1097 * We use the irq-unsafe __{inc|mod}_zone_page_stat because 1098 * these counters are not modified in interrupt context, and 1099 * pte lock(a spinlock) is held, which implies preemption 1100 * disabled. 1101 */ 1102 if (compound) 1103 __inc_node_page_state(page, NR_ANON_THPS); 1104 __mod_node_page_state(page_pgdat(page), NR_ANON_MAPPED, nr); 1105 } 1106 if (unlikely(PageKsm(page))) 1107 return; 1108 1109 VM_BUG_ON_PAGE(!PageLocked(page), page); 1110 1111 /* address might be in next vma when migration races vma_adjust */ 1112 if (first) 1113 __page_set_anon_rmap(page, vma, address, 1114 flags & RMAP_EXCLUSIVE); 1115 else 1116 __page_check_anon_rmap(page, vma, address); 1117 } 1118 1119 /** 1120 * page_add_new_anon_rmap - add pte mapping to a new anonymous page 1121 * @page: the page to add the mapping to 1122 * @vma: the vm area in which the mapping is added 1123 * @address: the user virtual address mapped 1124 * @compound: charge the page as compound or small page 1125 * 1126 * Same as page_add_anon_rmap but must only be called on *new* pages. 1127 * This means the inc-and-test can be bypassed. 1128 * Page does not have to be locked. 1129 */ 1130 void page_add_new_anon_rmap(struct page *page, 1131 struct vm_area_struct *vma, unsigned long address, bool compound) 1132 { 1133 int nr = compound ? hpage_nr_pages(page) : 1; 1134 1135 VM_BUG_ON_VMA(address < vma->vm_start || address >= vma->vm_end, vma); 1136 __SetPageSwapBacked(page); 1137 if (compound) { 1138 VM_BUG_ON_PAGE(!PageTransHuge(page), page); 1139 /* increment count (starts at -1) */ 1140 atomic_set(compound_mapcount_ptr(page), 0); 1141 __inc_node_page_state(page, NR_ANON_THPS); 1142 } else { 1143 /* Anon THP always mapped first with PMD */ 1144 VM_BUG_ON_PAGE(PageTransCompound(page), page); 1145 /* increment count (starts at -1) */ 1146 atomic_set(&page->_mapcount, 0); 1147 } 1148 __mod_node_page_state(page_pgdat(page), NR_ANON_MAPPED, nr); 1149 __page_set_anon_rmap(page, vma, address, 1); 1150 } 1151 1152 /** 1153 * page_add_file_rmap - add pte mapping to a file page 1154 * @page: the page to add the mapping to 1155 * 1156 * The caller needs to hold the pte lock. 1157 */ 1158 void page_add_file_rmap(struct page *page, bool compound) 1159 { 1160 int i, nr = 1; 1161 1162 VM_BUG_ON_PAGE(compound && !PageTransHuge(page), page); 1163 lock_page_memcg(page); 1164 if (compound && PageTransHuge(page)) { 1165 for (i = 0, nr = 0; i < HPAGE_PMD_NR; i++) { 1166 if (atomic_inc_and_test(&page[i]._mapcount)) 1167 nr++; 1168 } 1169 if (!atomic_inc_and_test(compound_mapcount_ptr(page))) 1170 goto out; 1171 VM_BUG_ON_PAGE(!PageSwapBacked(page), page); 1172 __inc_node_page_state(page, NR_SHMEM_PMDMAPPED); 1173 } else { 1174 if (PageTransCompound(page) && page_mapping(page)) { 1175 VM_WARN_ON_ONCE(!PageLocked(page)); 1176 1177 SetPageDoubleMap(compound_head(page)); 1178 if (PageMlocked(page)) 1179 clear_page_mlock(compound_head(page)); 1180 } 1181 if (!atomic_inc_and_test(&page->_mapcount)) 1182 goto out; 1183 } 1184 __mod_lruvec_page_state(page, NR_FILE_MAPPED, nr); 1185 out: 1186 unlock_page_memcg(page); 1187 } 1188 1189 static void page_remove_file_rmap(struct page *page, bool compound) 1190 { 1191 int i, nr = 1; 1192 1193 VM_BUG_ON_PAGE(compound && !PageHead(page), page); 1194 lock_page_memcg(page); 1195 1196 /* Hugepages are not counted in NR_FILE_MAPPED for now. */ 1197 if (unlikely(PageHuge(page))) { 1198 /* hugetlb pages are always mapped with pmds */ 1199 atomic_dec(compound_mapcount_ptr(page)); 1200 goto out; 1201 } 1202 1203 /* page still mapped by someone else? */ 1204 if (compound && PageTransHuge(page)) { 1205 for (i = 0, nr = 0; i < HPAGE_PMD_NR; i++) { 1206 if (atomic_add_negative(-1, &page[i]._mapcount)) 1207 nr++; 1208 } 1209 if (!atomic_add_negative(-1, compound_mapcount_ptr(page))) 1210 goto out; 1211 VM_BUG_ON_PAGE(!PageSwapBacked(page), page); 1212 __dec_node_page_state(page, NR_SHMEM_PMDMAPPED); 1213 } else { 1214 if (!atomic_add_negative(-1, &page->_mapcount)) 1215 goto out; 1216 } 1217 1218 /* 1219 * We use the irq-unsafe __{inc|mod}_lruvec_page_state because 1220 * these counters are not modified in interrupt context, and 1221 * pte lock(a spinlock) is held, which implies preemption disabled. 1222 */ 1223 __mod_lruvec_page_state(page, NR_FILE_MAPPED, -nr); 1224 1225 if (unlikely(PageMlocked(page))) 1226 clear_page_mlock(page); 1227 out: 1228 unlock_page_memcg(page); 1229 } 1230 1231 static void page_remove_anon_compound_rmap(struct page *page) 1232 { 1233 int i, nr; 1234 1235 if (!atomic_add_negative(-1, compound_mapcount_ptr(page))) 1236 return; 1237 1238 /* Hugepages are not counted in NR_ANON_PAGES for now. */ 1239 if (unlikely(PageHuge(page))) 1240 return; 1241 1242 if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) 1243 return; 1244 1245 __dec_node_page_state(page, NR_ANON_THPS); 1246 1247 if (TestClearPageDoubleMap(page)) { 1248 /* 1249 * Subpages can be mapped with PTEs too. Check how many of 1250 * themi are still mapped. 1251 */ 1252 for (i = 0, nr = 0; i < HPAGE_PMD_NR; i++) { 1253 if (atomic_add_negative(-1, &page[i]._mapcount)) 1254 nr++; 1255 } 1256 } else { 1257 nr = HPAGE_PMD_NR; 1258 } 1259 1260 if (unlikely(PageMlocked(page))) 1261 clear_page_mlock(page); 1262 1263 if (nr) { 1264 __mod_node_page_state(page_pgdat(page), NR_ANON_MAPPED, -nr); 1265 deferred_split_huge_page(page); 1266 } 1267 } 1268 1269 /** 1270 * page_remove_rmap - take down pte mapping from a page 1271 * @page: page to remove mapping from 1272 * @compound: uncharge the page as compound or small page 1273 * 1274 * The caller needs to hold the pte lock. 1275 */ 1276 void page_remove_rmap(struct page *page, bool compound) 1277 { 1278 if (!PageAnon(page)) 1279 return page_remove_file_rmap(page, compound); 1280 1281 if (compound) 1282 return page_remove_anon_compound_rmap(page); 1283 1284 /* page still mapped by someone else? */ 1285 if (!atomic_add_negative(-1, &page->_mapcount)) 1286 return; 1287 1288 /* 1289 * We use the irq-unsafe __{inc|mod}_zone_page_stat because 1290 * these counters are not modified in interrupt context, and 1291 * pte lock(a spinlock) is held, which implies preemption disabled. 1292 */ 1293 __dec_node_page_state(page, NR_ANON_MAPPED); 1294 1295 if (unlikely(PageMlocked(page))) 1296 clear_page_mlock(page); 1297 1298 if (PageTransCompound(page)) 1299 deferred_split_huge_page(compound_head(page)); 1300 1301 /* 1302 * It would be tidy to reset the PageAnon mapping here, 1303 * but that might overwrite a racing page_add_anon_rmap 1304 * which increments mapcount after us but sets mapping 1305 * before us: so leave the reset to free_hot_cold_page, 1306 * and remember that it's only reliable while mapped. 1307 * Leaving it set also helps swapoff to reinstate ptes 1308 * faster for those pages still in swapcache. 1309 */ 1310 } 1311 1312 /* 1313 * @arg: enum ttu_flags will be passed to this argument 1314 */ 1315 static bool try_to_unmap_one(struct page *page, struct vm_area_struct *vma, 1316 unsigned long address, void *arg) 1317 { 1318 struct mm_struct *mm = vma->vm_mm; 1319 struct page_vma_mapped_walk pvmw = { 1320 .page = page, 1321 .vma = vma, 1322 .address = address, 1323 }; 1324 pte_t pteval; 1325 struct page *subpage; 1326 bool ret = true; 1327 enum ttu_flags flags = (enum ttu_flags)arg; 1328 1329 /* munlock has nothing to gain from examining un-locked vmas */ 1330 if ((flags & TTU_MUNLOCK) && !(vma->vm_flags & VM_LOCKED)) 1331 return true; 1332 1333 if (flags & TTU_SPLIT_HUGE_PMD) { 1334 split_huge_pmd_address(vma, address, 1335 flags & TTU_MIGRATION, page); 1336 } 1337 1338 while (page_vma_mapped_walk(&pvmw)) { 1339 /* 1340 * If the page is mlock()d, we cannot swap it out. 1341 * If it's recently referenced (perhaps page_referenced 1342 * skipped over this mm) then we should reactivate it. 1343 */ 1344 if (!(flags & TTU_IGNORE_MLOCK)) { 1345 if (vma->vm_flags & VM_LOCKED) { 1346 /* PTE-mapped THP are never mlocked */ 1347 if (!PageTransCompound(page)) { 1348 /* 1349 * Holding pte lock, we do *not* need 1350 * mmap_sem here 1351 */ 1352 mlock_vma_page(page); 1353 } 1354 ret = false; 1355 page_vma_mapped_walk_done(&pvmw); 1356 break; 1357 } 1358 if (flags & TTU_MUNLOCK) 1359 continue; 1360 } 1361 1362 /* Unexpected PMD-mapped THP? */ 1363 VM_BUG_ON_PAGE(!pvmw.pte, page); 1364 1365 subpage = page - page_to_pfn(page) + pte_pfn(*pvmw.pte); 1366 address = pvmw.address; 1367 1368 1369 if (!(flags & TTU_IGNORE_ACCESS)) { 1370 if (ptep_clear_flush_young_notify(vma, address, 1371 pvmw.pte)) { 1372 ret = false; 1373 page_vma_mapped_walk_done(&pvmw); 1374 break; 1375 } 1376 } 1377 1378 /* Nuke the page table entry. */ 1379 flush_cache_page(vma, address, pte_pfn(*pvmw.pte)); 1380 if (should_defer_flush(mm, flags)) { 1381 /* 1382 * We clear the PTE but do not flush so potentially 1383 * a remote CPU could still be writing to the page. 1384 * If the entry was previously clean then the 1385 * architecture must guarantee that a clear->dirty 1386 * transition on a cached TLB entry is written through 1387 * and traps if the PTE is unmapped. 1388 */ 1389 pteval = ptep_get_and_clear(mm, address, pvmw.pte); 1390 1391 set_tlb_ubc_flush_pending(mm, pte_dirty(pteval)); 1392 } else { 1393 pteval = ptep_clear_flush(vma, address, pvmw.pte); 1394 } 1395 1396 /* Move the dirty bit to the page. Now the pte is gone. */ 1397 if (pte_dirty(pteval)) 1398 set_page_dirty(page); 1399 1400 /* Update high watermark before we lower rss */ 1401 update_hiwater_rss(mm); 1402 1403 if (PageHWPoison(page) && !(flags & TTU_IGNORE_HWPOISON)) { 1404 pteval = swp_entry_to_pte(make_hwpoison_entry(subpage)); 1405 if (PageHuge(page)) { 1406 int nr = 1 << compound_order(page); 1407 hugetlb_count_sub(nr, mm); 1408 set_huge_swap_pte_at(mm, address, 1409 pvmw.pte, pteval, 1410 vma_mmu_pagesize(vma)); 1411 } else { 1412 dec_mm_counter(mm, mm_counter(page)); 1413 set_pte_at(mm, address, pvmw.pte, pteval); 1414 } 1415 1416 } else if (pte_unused(pteval)) { 1417 /* 1418 * The guest indicated that the page content is of no 1419 * interest anymore. Simply discard the pte, vmscan 1420 * will take care of the rest. 1421 */ 1422 dec_mm_counter(mm, mm_counter(page)); 1423 } else if (IS_ENABLED(CONFIG_MIGRATION) && 1424 (flags & TTU_MIGRATION)) { 1425 swp_entry_t entry; 1426 pte_t swp_pte; 1427 /* 1428 * Store the pfn of the page in a special migration 1429 * pte. do_swap_page() will wait until the migration 1430 * pte is removed and then restart fault handling. 1431 */ 1432 entry = make_migration_entry(subpage, 1433 pte_write(pteval)); 1434 swp_pte = swp_entry_to_pte(entry); 1435 if (pte_soft_dirty(pteval)) 1436 swp_pte = pte_swp_mksoft_dirty(swp_pte); 1437 set_pte_at(mm, address, pvmw.pte, swp_pte); 1438 } else if (PageAnon(page)) { 1439 swp_entry_t entry = { .val = page_private(subpage) }; 1440 pte_t swp_pte; 1441 /* 1442 * Store the swap location in the pte. 1443 * See handle_pte_fault() ... 1444 */ 1445 if (unlikely(PageSwapBacked(page) != PageSwapCache(page))) { 1446 WARN_ON_ONCE(1); 1447 ret = false; 1448 page_vma_mapped_walk_done(&pvmw); 1449 break; 1450 } 1451 1452 /* MADV_FREE page check */ 1453 if (!PageSwapBacked(page)) { 1454 if (!PageDirty(page)) { 1455 dec_mm_counter(mm, MM_ANONPAGES); 1456 goto discard; 1457 } 1458 1459 /* 1460 * If the page was redirtied, it cannot be 1461 * discarded. Remap the page to page table. 1462 */ 1463 set_pte_at(mm, address, pvmw.pte, pteval); 1464 SetPageSwapBacked(page); 1465 ret = false; 1466 page_vma_mapped_walk_done(&pvmw); 1467 break; 1468 } 1469 1470 if (swap_duplicate(entry) < 0) { 1471 set_pte_at(mm, address, pvmw.pte, pteval); 1472 ret = false; 1473 page_vma_mapped_walk_done(&pvmw); 1474 break; 1475 } 1476 if (list_empty(&mm->mmlist)) { 1477 spin_lock(&mmlist_lock); 1478 if (list_empty(&mm->mmlist)) 1479 list_add(&mm->mmlist, &init_mm.mmlist); 1480 spin_unlock(&mmlist_lock); 1481 } 1482 dec_mm_counter(mm, MM_ANONPAGES); 1483 inc_mm_counter(mm, MM_SWAPENTS); 1484 swp_pte = swp_entry_to_pte(entry); 1485 if (pte_soft_dirty(pteval)) 1486 swp_pte = pte_swp_mksoft_dirty(swp_pte); 1487 set_pte_at(mm, address, pvmw.pte, swp_pte); 1488 } else 1489 dec_mm_counter(mm, mm_counter_file(page)); 1490 discard: 1491 page_remove_rmap(subpage, PageHuge(page)); 1492 put_page(page); 1493 mmu_notifier_invalidate_page(mm, address); 1494 } 1495 return ret; 1496 } 1497 1498 bool is_vma_temporary_stack(struct vm_area_struct *vma) 1499 { 1500 int maybe_stack = vma->vm_flags & (VM_GROWSDOWN | VM_GROWSUP); 1501 1502 if (!maybe_stack) 1503 return false; 1504 1505 if ((vma->vm_flags & VM_STACK_INCOMPLETE_SETUP) == 1506 VM_STACK_INCOMPLETE_SETUP) 1507 return true; 1508 1509 return false; 1510 } 1511 1512 static bool invalid_migration_vma(struct vm_area_struct *vma, void *arg) 1513 { 1514 return is_vma_temporary_stack(vma); 1515 } 1516 1517 static int page_mapcount_is_zero(struct page *page) 1518 { 1519 return !total_mapcount(page); 1520 } 1521 1522 /** 1523 * try_to_unmap - try to remove all page table mappings to a page 1524 * @page: the page to get unmapped 1525 * @flags: action and flags 1526 * 1527 * Tries to remove all the page table entries which are mapping this 1528 * page, used in the pageout path. Caller must hold the page lock. 1529 * 1530 * If unmap is successful, return true. Otherwise, false. 1531 */ 1532 bool try_to_unmap(struct page *page, enum ttu_flags flags) 1533 { 1534 struct rmap_walk_control rwc = { 1535 .rmap_one = try_to_unmap_one, 1536 .arg = (void *)flags, 1537 .done = page_mapcount_is_zero, 1538 .anon_lock = page_lock_anon_vma_read, 1539 }; 1540 1541 /* 1542 * During exec, a temporary VMA is setup and later moved. 1543 * The VMA is moved under the anon_vma lock but not the 1544 * page tables leading to a race where migration cannot 1545 * find the migration ptes. Rather than increasing the 1546 * locking requirements of exec(), migration skips 1547 * temporary VMAs until after exec() completes. 1548 */ 1549 if ((flags & TTU_MIGRATION) && !PageKsm(page) && PageAnon(page)) 1550 rwc.invalid_vma = invalid_migration_vma; 1551 1552 if (flags & TTU_RMAP_LOCKED) 1553 rmap_walk_locked(page, &rwc); 1554 else 1555 rmap_walk(page, &rwc); 1556 1557 return !page_mapcount(page) ? true : false; 1558 } 1559 1560 static int page_not_mapped(struct page *page) 1561 { 1562 return !page_mapped(page); 1563 }; 1564 1565 /** 1566 * try_to_munlock - try to munlock a page 1567 * @page: the page to be munlocked 1568 * 1569 * Called from munlock code. Checks all of the VMAs mapping the page 1570 * to make sure nobody else has this page mlocked. The page will be 1571 * returned with PG_mlocked cleared if no other vmas have it mlocked. 1572 */ 1573 1574 void try_to_munlock(struct page *page) 1575 { 1576 struct rmap_walk_control rwc = { 1577 .rmap_one = try_to_unmap_one, 1578 .arg = (void *)TTU_MUNLOCK, 1579 .done = page_not_mapped, 1580 .anon_lock = page_lock_anon_vma_read, 1581 1582 }; 1583 1584 VM_BUG_ON_PAGE(!PageLocked(page) || PageLRU(page), page); 1585 VM_BUG_ON_PAGE(PageCompound(page) && PageDoubleMap(page), page); 1586 1587 rmap_walk(page, &rwc); 1588 } 1589 1590 void __put_anon_vma(struct anon_vma *anon_vma) 1591 { 1592 struct anon_vma *root = anon_vma->root; 1593 1594 anon_vma_free(anon_vma); 1595 if (root != anon_vma && atomic_dec_and_test(&root->refcount)) 1596 anon_vma_free(root); 1597 } 1598 1599 static struct anon_vma *rmap_walk_anon_lock(struct page *page, 1600 struct rmap_walk_control *rwc) 1601 { 1602 struct anon_vma *anon_vma; 1603 1604 if (rwc->anon_lock) 1605 return rwc->anon_lock(page); 1606 1607 /* 1608 * Note: remove_migration_ptes() cannot use page_lock_anon_vma_read() 1609 * because that depends on page_mapped(); but not all its usages 1610 * are holding mmap_sem. Users without mmap_sem are required to 1611 * take a reference count to prevent the anon_vma disappearing 1612 */ 1613 anon_vma = page_anon_vma(page); 1614 if (!anon_vma) 1615 return NULL; 1616 1617 anon_vma_lock_read(anon_vma); 1618 return anon_vma; 1619 } 1620 1621 /* 1622 * rmap_walk_anon - do something to anonymous page using the object-based 1623 * rmap method 1624 * @page: the page to be handled 1625 * @rwc: control variable according to each walk type 1626 * 1627 * Find all the mappings of a page using the mapping pointer and the vma chains 1628 * contained in the anon_vma struct it points to. 1629 * 1630 * When called from try_to_munlock(), the mmap_sem of the mm containing the vma 1631 * where the page was found will be held for write. So, we won't recheck 1632 * vm_flags for that VMA. That should be OK, because that vma shouldn't be 1633 * LOCKED. 1634 */ 1635 static void rmap_walk_anon(struct page *page, struct rmap_walk_control *rwc, 1636 bool locked) 1637 { 1638 struct anon_vma *anon_vma; 1639 pgoff_t pgoff_start, pgoff_end; 1640 struct anon_vma_chain *avc; 1641 1642 if (locked) { 1643 anon_vma = page_anon_vma(page); 1644 /* anon_vma disappear under us? */ 1645 VM_BUG_ON_PAGE(!anon_vma, page); 1646 } else { 1647 anon_vma = rmap_walk_anon_lock(page, rwc); 1648 } 1649 if (!anon_vma) 1650 return; 1651 1652 pgoff_start = page_to_pgoff(page); 1653 pgoff_end = pgoff_start + hpage_nr_pages(page) - 1; 1654 anon_vma_interval_tree_foreach(avc, &anon_vma->rb_root, 1655 pgoff_start, pgoff_end) { 1656 struct vm_area_struct *vma = avc->vma; 1657 unsigned long address = vma_address(page, vma); 1658 1659 cond_resched(); 1660 1661 if (rwc->invalid_vma && rwc->invalid_vma(vma, rwc->arg)) 1662 continue; 1663 1664 if (!rwc->rmap_one(page, vma, address, rwc->arg)) 1665 break; 1666 if (rwc->done && rwc->done(page)) 1667 break; 1668 } 1669 1670 if (!locked) 1671 anon_vma_unlock_read(anon_vma); 1672 } 1673 1674 /* 1675 * rmap_walk_file - do something to file page using the object-based rmap method 1676 * @page: the page to be handled 1677 * @rwc: control variable according to each walk type 1678 * 1679 * Find all the mappings of a page using the mapping pointer and the vma chains 1680 * contained in the address_space struct it points to. 1681 * 1682 * When called from try_to_munlock(), the mmap_sem of the mm containing the vma 1683 * where the page was found will be held for write. So, we won't recheck 1684 * vm_flags for that VMA. That should be OK, because that vma shouldn't be 1685 * LOCKED. 1686 */ 1687 static void rmap_walk_file(struct page *page, struct rmap_walk_control *rwc, 1688 bool locked) 1689 { 1690 struct address_space *mapping = page_mapping(page); 1691 pgoff_t pgoff_start, pgoff_end; 1692 struct vm_area_struct *vma; 1693 1694 /* 1695 * The page lock not only makes sure that page->mapping cannot 1696 * suddenly be NULLified by truncation, it makes sure that the 1697 * structure at mapping cannot be freed and reused yet, 1698 * so we can safely take mapping->i_mmap_rwsem. 1699 */ 1700 VM_BUG_ON_PAGE(!PageLocked(page), page); 1701 1702 if (!mapping) 1703 return; 1704 1705 pgoff_start = page_to_pgoff(page); 1706 pgoff_end = pgoff_start + hpage_nr_pages(page) - 1; 1707 if (!locked) 1708 i_mmap_lock_read(mapping); 1709 vma_interval_tree_foreach(vma, &mapping->i_mmap, 1710 pgoff_start, pgoff_end) { 1711 unsigned long address = vma_address(page, vma); 1712 1713 cond_resched(); 1714 1715 if (rwc->invalid_vma && rwc->invalid_vma(vma, rwc->arg)) 1716 continue; 1717 1718 if (!rwc->rmap_one(page, vma, address, rwc->arg)) 1719 goto done; 1720 if (rwc->done && rwc->done(page)) 1721 goto done; 1722 } 1723 1724 done: 1725 if (!locked) 1726 i_mmap_unlock_read(mapping); 1727 } 1728 1729 void rmap_walk(struct page *page, struct rmap_walk_control *rwc) 1730 { 1731 if (unlikely(PageKsm(page))) 1732 rmap_walk_ksm(page, rwc); 1733 else if (PageAnon(page)) 1734 rmap_walk_anon(page, rwc, false); 1735 else 1736 rmap_walk_file(page, rwc, false); 1737 } 1738 1739 /* Like rmap_walk, but caller holds relevant rmap lock */ 1740 void rmap_walk_locked(struct page *page, struct rmap_walk_control *rwc) 1741 { 1742 /* no ksm support for now */ 1743 VM_BUG_ON_PAGE(PageKsm(page), page); 1744 if (PageAnon(page)) 1745 rmap_walk_anon(page, rwc, true); 1746 else 1747 rmap_walk_file(page, rwc, true); 1748 } 1749 1750 #ifdef CONFIG_HUGETLB_PAGE 1751 /* 1752 * The following three functions are for anonymous (private mapped) hugepages. 1753 * Unlike common anonymous pages, anonymous hugepages have no accounting code 1754 * and no lru code, because we handle hugepages differently from common pages. 1755 */ 1756 static void __hugepage_set_anon_rmap(struct page *page, 1757 struct vm_area_struct *vma, unsigned long address, int exclusive) 1758 { 1759 struct anon_vma *anon_vma = vma->anon_vma; 1760 1761 BUG_ON(!anon_vma); 1762 1763 if (PageAnon(page)) 1764 return; 1765 if (!exclusive) 1766 anon_vma = anon_vma->root; 1767 1768 anon_vma = (void *) anon_vma + PAGE_MAPPING_ANON; 1769 page->mapping = (struct address_space *) anon_vma; 1770 page->index = linear_page_index(vma, address); 1771 } 1772 1773 void hugepage_add_anon_rmap(struct page *page, 1774 struct vm_area_struct *vma, unsigned long address) 1775 { 1776 struct anon_vma *anon_vma = vma->anon_vma; 1777 int first; 1778 1779 BUG_ON(!PageLocked(page)); 1780 BUG_ON(!anon_vma); 1781 /* address might be in next vma when migration races vma_adjust */ 1782 first = atomic_inc_and_test(compound_mapcount_ptr(page)); 1783 if (first) 1784 __hugepage_set_anon_rmap(page, vma, address, 0); 1785 } 1786 1787 void hugepage_add_new_anon_rmap(struct page *page, 1788 struct vm_area_struct *vma, unsigned long address) 1789 { 1790 BUG_ON(address < vma->vm_start || address >= vma->vm_end); 1791 atomic_set(compound_mapcount_ptr(page), 0); 1792 __hugepage_set_anon_rmap(page, vma, address, 1); 1793 } 1794 #endif /* CONFIG_HUGETLB_PAGE */ 1795