1 /* 2 * mm/rmap.c - physical to virtual reverse mappings 3 * 4 * Copyright 2001, Rik van Riel <riel@conectiva.com.br> 5 * Released under the General Public License (GPL). 6 * 7 * Simple, low overhead reverse mapping scheme. 8 * Please try to keep this thing as modular as possible. 9 * 10 * Provides methods for unmapping each kind of mapped page: 11 * the anon methods track anonymous pages, and 12 * the file methods track pages belonging to an inode. 13 * 14 * Original design by Rik van Riel <riel@conectiva.com.br> 2001 15 * File methods by Dave McCracken <dmccr@us.ibm.com> 2003, 2004 16 * Anonymous methods by Andrea Arcangeli <andrea@suse.de> 2004 17 * Contributions by Hugh Dickins 2003, 2004 18 */ 19 20 /* 21 * Lock ordering in mm: 22 * 23 * inode->i_mutex (while writing or truncating, not reading or faulting) 24 * mm->mmap_sem 25 * page->flags PG_locked (lock_page) 26 * hugetlbfs_i_mmap_rwsem_key (in huge_pmd_share) 27 * mapping->i_mmap_rwsem 28 * anon_vma->rwsem 29 * mm->page_table_lock or pte_lock 30 * pgdat->lru_lock (in mark_page_accessed, isolate_lru_page) 31 * swap_lock (in swap_duplicate, swap_info_get) 32 * mmlist_lock (in mmput, drain_mmlist and others) 33 * mapping->private_lock (in __set_page_dirty_buffers) 34 * mem_cgroup_{begin,end}_page_stat (memcg->move_lock) 35 * i_pages lock (widely used) 36 * inode->i_lock (in set_page_dirty's __mark_inode_dirty) 37 * bdi.wb->list_lock (in set_page_dirty's __mark_inode_dirty) 38 * sb_lock (within inode_lock in fs/fs-writeback.c) 39 * i_pages lock (widely used, in set_page_dirty, 40 * in arch-dependent flush_dcache_mmap_lock, 41 * within bdi.wb->list_lock in __sync_single_inode) 42 * 43 * anon_vma->rwsem,mapping->i_mutex (memory_failure, collect_procs_anon) 44 * ->tasklist_lock 45 * pte map lock 46 */ 47 48 #include <linux/mm.h> 49 #include <linux/sched/mm.h> 50 #include <linux/sched/task.h> 51 #include <linux/pagemap.h> 52 #include <linux/swap.h> 53 #include <linux/swapops.h> 54 #include <linux/slab.h> 55 #include <linux/init.h> 56 #include <linux/ksm.h> 57 #include <linux/rmap.h> 58 #include <linux/rcupdate.h> 59 #include <linux/export.h> 60 #include <linux/memcontrol.h> 61 #include <linux/mmu_notifier.h> 62 #include <linux/migrate.h> 63 #include <linux/hugetlb.h> 64 #include <linux/backing-dev.h> 65 #include <linux/page_idle.h> 66 #include <linux/memremap.h> 67 #include <linux/userfaultfd_k.h> 68 69 #include <asm/tlbflush.h> 70 71 #include <trace/events/tlb.h> 72 73 #include "internal.h" 74 75 static struct kmem_cache *anon_vma_cachep; 76 static struct kmem_cache *anon_vma_chain_cachep; 77 78 static inline struct anon_vma *anon_vma_alloc(void) 79 { 80 struct anon_vma *anon_vma; 81 82 anon_vma = kmem_cache_alloc(anon_vma_cachep, GFP_KERNEL); 83 if (anon_vma) { 84 atomic_set(&anon_vma->refcount, 1); 85 anon_vma->degree = 1; /* Reference for first vma */ 86 anon_vma->parent = anon_vma; 87 /* 88 * Initialise the anon_vma root to point to itself. If called 89 * from fork, the root will be reset to the parents anon_vma. 90 */ 91 anon_vma->root = anon_vma; 92 } 93 94 return anon_vma; 95 } 96 97 static inline void anon_vma_free(struct anon_vma *anon_vma) 98 { 99 VM_BUG_ON(atomic_read(&anon_vma->refcount)); 100 101 /* 102 * Synchronize against page_lock_anon_vma_read() such that 103 * we can safely hold the lock without the anon_vma getting 104 * freed. 105 * 106 * Relies on the full mb implied by the atomic_dec_and_test() from 107 * put_anon_vma() against the acquire barrier implied by 108 * down_read_trylock() from page_lock_anon_vma_read(). This orders: 109 * 110 * page_lock_anon_vma_read() VS put_anon_vma() 111 * down_read_trylock() atomic_dec_and_test() 112 * LOCK MB 113 * atomic_read() rwsem_is_locked() 114 * 115 * LOCK should suffice since the actual taking of the lock must 116 * happen _before_ what follows. 117 */ 118 might_sleep(); 119 if (rwsem_is_locked(&anon_vma->root->rwsem)) { 120 anon_vma_lock_write(anon_vma); 121 anon_vma_unlock_write(anon_vma); 122 } 123 124 kmem_cache_free(anon_vma_cachep, anon_vma); 125 } 126 127 static inline struct anon_vma_chain *anon_vma_chain_alloc(gfp_t gfp) 128 { 129 return kmem_cache_alloc(anon_vma_chain_cachep, gfp); 130 } 131 132 static void anon_vma_chain_free(struct anon_vma_chain *anon_vma_chain) 133 { 134 kmem_cache_free(anon_vma_chain_cachep, anon_vma_chain); 135 } 136 137 static void anon_vma_chain_link(struct vm_area_struct *vma, 138 struct anon_vma_chain *avc, 139 struct anon_vma *anon_vma) 140 { 141 avc->vma = vma; 142 avc->anon_vma = anon_vma; 143 list_add(&avc->same_vma, &vma->anon_vma_chain); 144 anon_vma_interval_tree_insert(avc, &anon_vma->rb_root); 145 } 146 147 /** 148 * __anon_vma_prepare - attach an anon_vma to a memory region 149 * @vma: the memory region in question 150 * 151 * This makes sure the memory mapping described by 'vma' has 152 * an 'anon_vma' attached to it, so that we can associate the 153 * anonymous pages mapped into it with that anon_vma. 154 * 155 * The common case will be that we already have one, which 156 * is handled inline by anon_vma_prepare(). But if 157 * not we either need to find an adjacent mapping that we 158 * can re-use the anon_vma from (very common when the only 159 * reason for splitting a vma has been mprotect()), or we 160 * allocate a new one. 161 * 162 * Anon-vma allocations are very subtle, because we may have 163 * optimistically looked up an anon_vma in page_lock_anon_vma_read() 164 * and that may actually touch the spinlock even in the newly 165 * allocated vma (it depends on RCU to make sure that the 166 * anon_vma isn't actually destroyed). 167 * 168 * As a result, we need to do proper anon_vma locking even 169 * for the new allocation. At the same time, we do not want 170 * to do any locking for the common case of already having 171 * an anon_vma. 172 * 173 * This must be called with the mmap_sem held for reading. 174 */ 175 int __anon_vma_prepare(struct vm_area_struct *vma) 176 { 177 struct mm_struct *mm = vma->vm_mm; 178 struct anon_vma *anon_vma, *allocated; 179 struct anon_vma_chain *avc; 180 181 might_sleep(); 182 183 avc = anon_vma_chain_alloc(GFP_KERNEL); 184 if (!avc) 185 goto out_enomem; 186 187 anon_vma = find_mergeable_anon_vma(vma); 188 allocated = NULL; 189 if (!anon_vma) { 190 anon_vma = anon_vma_alloc(); 191 if (unlikely(!anon_vma)) 192 goto out_enomem_free_avc; 193 allocated = anon_vma; 194 } 195 196 anon_vma_lock_write(anon_vma); 197 /* page_table_lock to protect against threads */ 198 spin_lock(&mm->page_table_lock); 199 if (likely(!vma->anon_vma)) { 200 vma->anon_vma = anon_vma; 201 anon_vma_chain_link(vma, avc, anon_vma); 202 /* vma reference or self-parent link for new root */ 203 anon_vma->degree++; 204 allocated = NULL; 205 avc = NULL; 206 } 207 spin_unlock(&mm->page_table_lock); 208 anon_vma_unlock_write(anon_vma); 209 210 if (unlikely(allocated)) 211 put_anon_vma(allocated); 212 if (unlikely(avc)) 213 anon_vma_chain_free(avc); 214 215 return 0; 216 217 out_enomem_free_avc: 218 anon_vma_chain_free(avc); 219 out_enomem: 220 return -ENOMEM; 221 } 222 223 /* 224 * This is a useful helper function for locking the anon_vma root as 225 * we traverse the vma->anon_vma_chain, looping over anon_vma's that 226 * have the same vma. 227 * 228 * Such anon_vma's should have the same root, so you'd expect to see 229 * just a single mutex_lock for the whole traversal. 230 */ 231 static inline struct anon_vma *lock_anon_vma_root(struct anon_vma *root, struct anon_vma *anon_vma) 232 { 233 struct anon_vma *new_root = anon_vma->root; 234 if (new_root != root) { 235 if (WARN_ON_ONCE(root)) 236 up_write(&root->rwsem); 237 root = new_root; 238 down_write(&root->rwsem); 239 } 240 return root; 241 } 242 243 static inline void unlock_anon_vma_root(struct anon_vma *root) 244 { 245 if (root) 246 up_write(&root->rwsem); 247 } 248 249 /* 250 * Attach the anon_vmas from src to dst. 251 * Returns 0 on success, -ENOMEM on failure. 252 * 253 * If dst->anon_vma is NULL this function tries to find and reuse existing 254 * anon_vma which has no vmas and only one child anon_vma. This prevents 255 * degradation of anon_vma hierarchy to endless linear chain in case of 256 * constantly forking task. On the other hand, an anon_vma with more than one 257 * child isn't reused even if there was no alive vma, thus rmap walker has a 258 * good chance of avoiding scanning the whole hierarchy when it searches where 259 * page is mapped. 260 */ 261 int anon_vma_clone(struct vm_area_struct *dst, struct vm_area_struct *src) 262 { 263 struct anon_vma_chain *avc, *pavc; 264 struct anon_vma *root = NULL; 265 266 list_for_each_entry_reverse(pavc, &src->anon_vma_chain, same_vma) { 267 struct anon_vma *anon_vma; 268 269 avc = anon_vma_chain_alloc(GFP_NOWAIT | __GFP_NOWARN); 270 if (unlikely(!avc)) { 271 unlock_anon_vma_root(root); 272 root = NULL; 273 avc = anon_vma_chain_alloc(GFP_KERNEL); 274 if (!avc) 275 goto enomem_failure; 276 } 277 anon_vma = pavc->anon_vma; 278 root = lock_anon_vma_root(root, anon_vma); 279 anon_vma_chain_link(dst, avc, anon_vma); 280 281 /* 282 * Reuse existing anon_vma if its degree lower than two, 283 * that means it has no vma and only one anon_vma child. 284 * 285 * Do not chose parent anon_vma, otherwise first child 286 * will always reuse it. Root anon_vma is never reused: 287 * it has self-parent reference and at least one child. 288 */ 289 if (!dst->anon_vma && anon_vma != src->anon_vma && 290 anon_vma->degree < 2) 291 dst->anon_vma = anon_vma; 292 } 293 if (dst->anon_vma) 294 dst->anon_vma->degree++; 295 unlock_anon_vma_root(root); 296 return 0; 297 298 enomem_failure: 299 /* 300 * dst->anon_vma is dropped here otherwise its degree can be incorrectly 301 * decremented in unlink_anon_vmas(). 302 * We can safely do this because callers of anon_vma_clone() don't care 303 * about dst->anon_vma if anon_vma_clone() failed. 304 */ 305 dst->anon_vma = NULL; 306 unlink_anon_vmas(dst); 307 return -ENOMEM; 308 } 309 310 /* 311 * Attach vma to its own anon_vma, as well as to the anon_vmas that 312 * the corresponding VMA in the parent process is attached to. 313 * Returns 0 on success, non-zero on failure. 314 */ 315 int anon_vma_fork(struct vm_area_struct *vma, struct vm_area_struct *pvma) 316 { 317 struct anon_vma_chain *avc; 318 struct anon_vma *anon_vma; 319 int error; 320 321 /* Don't bother if the parent process has no anon_vma here. */ 322 if (!pvma->anon_vma) 323 return 0; 324 325 /* Drop inherited anon_vma, we'll reuse existing or allocate new. */ 326 vma->anon_vma = NULL; 327 328 /* 329 * First, attach the new VMA to the parent VMA's anon_vmas, 330 * so rmap can find non-COWed pages in child processes. 331 */ 332 error = anon_vma_clone(vma, pvma); 333 if (error) 334 return error; 335 336 /* An existing anon_vma has been reused, all done then. */ 337 if (vma->anon_vma) 338 return 0; 339 340 /* Then add our own anon_vma. */ 341 anon_vma = anon_vma_alloc(); 342 if (!anon_vma) 343 goto out_error; 344 avc = anon_vma_chain_alloc(GFP_KERNEL); 345 if (!avc) 346 goto out_error_free_anon_vma; 347 348 /* 349 * The root anon_vma's spinlock is the lock actually used when we 350 * lock any of the anon_vmas in this anon_vma tree. 351 */ 352 anon_vma->root = pvma->anon_vma->root; 353 anon_vma->parent = pvma->anon_vma; 354 /* 355 * With refcounts, an anon_vma can stay around longer than the 356 * process it belongs to. The root anon_vma needs to be pinned until 357 * this anon_vma is freed, because the lock lives in the root. 358 */ 359 get_anon_vma(anon_vma->root); 360 /* Mark this anon_vma as the one where our new (COWed) pages go. */ 361 vma->anon_vma = anon_vma; 362 anon_vma_lock_write(anon_vma); 363 anon_vma_chain_link(vma, avc, anon_vma); 364 anon_vma->parent->degree++; 365 anon_vma_unlock_write(anon_vma); 366 367 return 0; 368 369 out_error_free_anon_vma: 370 put_anon_vma(anon_vma); 371 out_error: 372 unlink_anon_vmas(vma); 373 return -ENOMEM; 374 } 375 376 void unlink_anon_vmas(struct vm_area_struct *vma) 377 { 378 struct anon_vma_chain *avc, *next; 379 struct anon_vma *root = NULL; 380 381 /* 382 * Unlink each anon_vma chained to the VMA. This list is ordered 383 * from newest to oldest, ensuring the root anon_vma gets freed last. 384 */ 385 list_for_each_entry_safe(avc, next, &vma->anon_vma_chain, same_vma) { 386 struct anon_vma *anon_vma = avc->anon_vma; 387 388 root = lock_anon_vma_root(root, anon_vma); 389 anon_vma_interval_tree_remove(avc, &anon_vma->rb_root); 390 391 /* 392 * Leave empty anon_vmas on the list - we'll need 393 * to free them outside the lock. 394 */ 395 if (RB_EMPTY_ROOT(&anon_vma->rb_root.rb_root)) { 396 anon_vma->parent->degree--; 397 continue; 398 } 399 400 list_del(&avc->same_vma); 401 anon_vma_chain_free(avc); 402 } 403 if (vma->anon_vma) 404 vma->anon_vma->degree--; 405 unlock_anon_vma_root(root); 406 407 /* 408 * Iterate the list once more, it now only contains empty and unlinked 409 * anon_vmas, destroy them. Could not do before due to __put_anon_vma() 410 * needing to write-acquire the anon_vma->root->rwsem. 411 */ 412 list_for_each_entry_safe(avc, next, &vma->anon_vma_chain, same_vma) { 413 struct anon_vma *anon_vma = avc->anon_vma; 414 415 VM_WARN_ON(anon_vma->degree); 416 put_anon_vma(anon_vma); 417 418 list_del(&avc->same_vma); 419 anon_vma_chain_free(avc); 420 } 421 } 422 423 static void anon_vma_ctor(void *data) 424 { 425 struct anon_vma *anon_vma = data; 426 427 init_rwsem(&anon_vma->rwsem); 428 atomic_set(&anon_vma->refcount, 0); 429 anon_vma->rb_root = RB_ROOT_CACHED; 430 } 431 432 void __init anon_vma_init(void) 433 { 434 anon_vma_cachep = kmem_cache_create("anon_vma", sizeof(struct anon_vma), 435 0, SLAB_TYPESAFE_BY_RCU|SLAB_PANIC|SLAB_ACCOUNT, 436 anon_vma_ctor); 437 anon_vma_chain_cachep = KMEM_CACHE(anon_vma_chain, 438 SLAB_PANIC|SLAB_ACCOUNT); 439 } 440 441 /* 442 * Getting a lock on a stable anon_vma from a page off the LRU is tricky! 443 * 444 * Since there is no serialization what so ever against page_remove_rmap() 445 * the best this function can do is return a locked anon_vma that might 446 * have been relevant to this page. 447 * 448 * The page might have been remapped to a different anon_vma or the anon_vma 449 * returned may already be freed (and even reused). 450 * 451 * In case it was remapped to a different anon_vma, the new anon_vma will be a 452 * child of the old anon_vma, and the anon_vma lifetime rules will therefore 453 * ensure that any anon_vma obtained from the page will still be valid for as 454 * long as we observe page_mapped() [ hence all those page_mapped() tests ]. 455 * 456 * All users of this function must be very careful when walking the anon_vma 457 * chain and verify that the page in question is indeed mapped in it 458 * [ something equivalent to page_mapped_in_vma() ]. 459 * 460 * Since anon_vma's slab is DESTROY_BY_RCU and we know from page_remove_rmap() 461 * that the anon_vma pointer from page->mapping is valid if there is a 462 * mapcount, we can dereference the anon_vma after observing those. 463 */ 464 struct anon_vma *page_get_anon_vma(struct page *page) 465 { 466 struct anon_vma *anon_vma = NULL; 467 unsigned long anon_mapping; 468 469 rcu_read_lock(); 470 anon_mapping = (unsigned long)READ_ONCE(page->mapping); 471 if ((anon_mapping & PAGE_MAPPING_FLAGS) != PAGE_MAPPING_ANON) 472 goto out; 473 if (!page_mapped(page)) 474 goto out; 475 476 anon_vma = (struct anon_vma *) (anon_mapping - PAGE_MAPPING_ANON); 477 if (!atomic_inc_not_zero(&anon_vma->refcount)) { 478 anon_vma = NULL; 479 goto out; 480 } 481 482 /* 483 * If this page is still mapped, then its anon_vma cannot have been 484 * freed. But if it has been unmapped, we have no security against the 485 * anon_vma structure being freed and reused (for another anon_vma: 486 * SLAB_TYPESAFE_BY_RCU guarantees that - so the atomic_inc_not_zero() 487 * above cannot corrupt). 488 */ 489 if (!page_mapped(page)) { 490 rcu_read_unlock(); 491 put_anon_vma(anon_vma); 492 return NULL; 493 } 494 out: 495 rcu_read_unlock(); 496 497 return anon_vma; 498 } 499 500 /* 501 * Similar to page_get_anon_vma() except it locks the anon_vma. 502 * 503 * Its a little more complex as it tries to keep the fast path to a single 504 * atomic op -- the trylock. If we fail the trylock, we fall back to getting a 505 * reference like with page_get_anon_vma() and then block on the mutex. 506 */ 507 struct anon_vma *page_lock_anon_vma_read(struct page *page) 508 { 509 struct anon_vma *anon_vma = NULL; 510 struct anon_vma *root_anon_vma; 511 unsigned long anon_mapping; 512 513 rcu_read_lock(); 514 anon_mapping = (unsigned long)READ_ONCE(page->mapping); 515 if ((anon_mapping & PAGE_MAPPING_FLAGS) != PAGE_MAPPING_ANON) 516 goto out; 517 if (!page_mapped(page)) 518 goto out; 519 520 anon_vma = (struct anon_vma *) (anon_mapping - PAGE_MAPPING_ANON); 521 root_anon_vma = READ_ONCE(anon_vma->root); 522 if (down_read_trylock(&root_anon_vma->rwsem)) { 523 /* 524 * If the page is still mapped, then this anon_vma is still 525 * its anon_vma, and holding the mutex ensures that it will 526 * not go away, see anon_vma_free(). 527 */ 528 if (!page_mapped(page)) { 529 up_read(&root_anon_vma->rwsem); 530 anon_vma = NULL; 531 } 532 goto out; 533 } 534 535 /* trylock failed, we got to sleep */ 536 if (!atomic_inc_not_zero(&anon_vma->refcount)) { 537 anon_vma = NULL; 538 goto out; 539 } 540 541 if (!page_mapped(page)) { 542 rcu_read_unlock(); 543 put_anon_vma(anon_vma); 544 return NULL; 545 } 546 547 /* we pinned the anon_vma, its safe to sleep */ 548 rcu_read_unlock(); 549 anon_vma_lock_read(anon_vma); 550 551 if (atomic_dec_and_test(&anon_vma->refcount)) { 552 /* 553 * Oops, we held the last refcount, release the lock 554 * and bail -- can't simply use put_anon_vma() because 555 * we'll deadlock on the anon_vma_lock_write() recursion. 556 */ 557 anon_vma_unlock_read(anon_vma); 558 __put_anon_vma(anon_vma); 559 anon_vma = NULL; 560 } 561 562 return anon_vma; 563 564 out: 565 rcu_read_unlock(); 566 return anon_vma; 567 } 568 569 void page_unlock_anon_vma_read(struct anon_vma *anon_vma) 570 { 571 anon_vma_unlock_read(anon_vma); 572 } 573 574 #ifdef CONFIG_ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH 575 /* 576 * Flush TLB entries for recently unmapped pages from remote CPUs. It is 577 * important if a PTE was dirty when it was unmapped that it's flushed 578 * before any IO is initiated on the page to prevent lost writes. Similarly, 579 * it must be flushed before freeing to prevent data leakage. 580 */ 581 void try_to_unmap_flush(void) 582 { 583 struct tlbflush_unmap_batch *tlb_ubc = ¤t->tlb_ubc; 584 585 if (!tlb_ubc->flush_required) 586 return; 587 588 arch_tlbbatch_flush(&tlb_ubc->arch); 589 tlb_ubc->flush_required = false; 590 tlb_ubc->writable = false; 591 } 592 593 /* Flush iff there are potentially writable TLB entries that can race with IO */ 594 void try_to_unmap_flush_dirty(void) 595 { 596 struct tlbflush_unmap_batch *tlb_ubc = ¤t->tlb_ubc; 597 598 if (tlb_ubc->writable) 599 try_to_unmap_flush(); 600 } 601 602 static void set_tlb_ubc_flush_pending(struct mm_struct *mm, bool writable) 603 { 604 struct tlbflush_unmap_batch *tlb_ubc = ¤t->tlb_ubc; 605 606 arch_tlbbatch_add_mm(&tlb_ubc->arch, mm); 607 tlb_ubc->flush_required = true; 608 609 /* 610 * Ensure compiler does not re-order the setting of tlb_flush_batched 611 * before the PTE is cleared. 612 */ 613 barrier(); 614 mm->tlb_flush_batched = true; 615 616 /* 617 * If the PTE was dirty then it's best to assume it's writable. The 618 * caller must use try_to_unmap_flush_dirty() or try_to_unmap_flush() 619 * before the page is queued for IO. 620 */ 621 if (writable) 622 tlb_ubc->writable = true; 623 } 624 625 /* 626 * Returns true if the TLB flush should be deferred to the end of a batch of 627 * unmap operations to reduce IPIs. 628 */ 629 static bool should_defer_flush(struct mm_struct *mm, enum ttu_flags flags) 630 { 631 bool should_defer = false; 632 633 if (!(flags & TTU_BATCH_FLUSH)) 634 return false; 635 636 /* If remote CPUs need to be flushed then defer batch the flush */ 637 if (cpumask_any_but(mm_cpumask(mm), get_cpu()) < nr_cpu_ids) 638 should_defer = true; 639 put_cpu(); 640 641 return should_defer; 642 } 643 644 /* 645 * Reclaim unmaps pages under the PTL but do not flush the TLB prior to 646 * releasing the PTL if TLB flushes are batched. It's possible for a parallel 647 * operation such as mprotect or munmap to race between reclaim unmapping 648 * the page and flushing the page. If this race occurs, it potentially allows 649 * access to data via a stale TLB entry. Tracking all mm's that have TLB 650 * batching in flight would be expensive during reclaim so instead track 651 * whether TLB batching occurred in the past and if so then do a flush here 652 * if required. This will cost one additional flush per reclaim cycle paid 653 * by the first operation at risk such as mprotect and mumap. 654 * 655 * This must be called under the PTL so that an access to tlb_flush_batched 656 * that is potentially a "reclaim vs mprotect/munmap/etc" race will synchronise 657 * via the PTL. 658 */ 659 void flush_tlb_batched_pending(struct mm_struct *mm) 660 { 661 if (mm->tlb_flush_batched) { 662 flush_tlb_mm(mm); 663 664 /* 665 * Do not allow the compiler to re-order the clearing of 666 * tlb_flush_batched before the tlb is flushed. 667 */ 668 barrier(); 669 mm->tlb_flush_batched = false; 670 } 671 } 672 #else 673 static void set_tlb_ubc_flush_pending(struct mm_struct *mm, bool writable) 674 { 675 } 676 677 static bool should_defer_flush(struct mm_struct *mm, enum ttu_flags flags) 678 { 679 return false; 680 } 681 #endif /* CONFIG_ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH */ 682 683 /* 684 * At what user virtual address is page expected in vma? 685 * Caller should check the page is actually part of the vma. 686 */ 687 unsigned long page_address_in_vma(struct page *page, struct vm_area_struct *vma) 688 { 689 unsigned long address; 690 if (PageAnon(page)) { 691 struct anon_vma *page__anon_vma = page_anon_vma(page); 692 /* 693 * Note: swapoff's unuse_vma() is more efficient with this 694 * check, and needs it to match anon_vma when KSM is active. 695 */ 696 if (!vma->anon_vma || !page__anon_vma || 697 vma->anon_vma->root != page__anon_vma->root) 698 return -EFAULT; 699 } else if (page->mapping) { 700 if (!vma->vm_file || vma->vm_file->f_mapping != page->mapping) 701 return -EFAULT; 702 } else 703 return -EFAULT; 704 address = __vma_address(page, vma); 705 if (unlikely(address < vma->vm_start || address >= vma->vm_end)) 706 return -EFAULT; 707 return address; 708 } 709 710 pmd_t *mm_find_pmd(struct mm_struct *mm, unsigned long address) 711 { 712 pgd_t *pgd; 713 p4d_t *p4d; 714 pud_t *pud; 715 pmd_t *pmd = NULL; 716 pmd_t pmde; 717 718 pgd = pgd_offset(mm, address); 719 if (!pgd_present(*pgd)) 720 goto out; 721 722 p4d = p4d_offset(pgd, address); 723 if (!p4d_present(*p4d)) 724 goto out; 725 726 pud = pud_offset(p4d, address); 727 if (!pud_present(*pud)) 728 goto out; 729 730 pmd = pmd_offset(pud, address); 731 /* 732 * Some THP functions use the sequence pmdp_huge_clear_flush(), set_pmd_at() 733 * without holding anon_vma lock for write. So when looking for a 734 * genuine pmde (in which to find pte), test present and !THP together. 735 */ 736 pmde = *pmd; 737 barrier(); 738 if (!pmd_present(pmde) || pmd_trans_huge(pmde)) 739 pmd = NULL; 740 out: 741 return pmd; 742 } 743 744 struct page_referenced_arg { 745 int mapcount; 746 int referenced; 747 unsigned long vm_flags; 748 struct mem_cgroup *memcg; 749 }; 750 /* 751 * arg: page_referenced_arg will be passed 752 */ 753 static bool page_referenced_one(struct page *page, struct vm_area_struct *vma, 754 unsigned long address, void *arg) 755 { 756 struct page_referenced_arg *pra = arg; 757 struct page_vma_mapped_walk pvmw = { 758 .page = page, 759 .vma = vma, 760 .address = address, 761 }; 762 int referenced = 0; 763 764 while (page_vma_mapped_walk(&pvmw)) { 765 address = pvmw.address; 766 767 if (vma->vm_flags & VM_LOCKED) { 768 page_vma_mapped_walk_done(&pvmw); 769 pra->vm_flags |= VM_LOCKED; 770 return false; /* To break the loop */ 771 } 772 773 if (pvmw.pte) { 774 if (ptep_clear_flush_young_notify(vma, address, 775 pvmw.pte)) { 776 /* 777 * Don't treat a reference through 778 * a sequentially read mapping as such. 779 * If the page has been used in another mapping, 780 * we will catch it; if this other mapping is 781 * already gone, the unmap path will have set 782 * PG_referenced or activated the page. 783 */ 784 if (likely(!(vma->vm_flags & VM_SEQ_READ))) 785 referenced++; 786 } 787 } else if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) { 788 if (pmdp_clear_flush_young_notify(vma, address, 789 pvmw.pmd)) 790 referenced++; 791 } else { 792 /* unexpected pmd-mapped page? */ 793 WARN_ON_ONCE(1); 794 } 795 796 pra->mapcount--; 797 } 798 799 if (referenced) 800 clear_page_idle(page); 801 if (test_and_clear_page_young(page)) 802 referenced++; 803 804 if (referenced) { 805 pra->referenced++; 806 pra->vm_flags |= vma->vm_flags; 807 } 808 809 if (!pra->mapcount) 810 return false; /* To break the loop */ 811 812 return true; 813 } 814 815 static bool invalid_page_referenced_vma(struct vm_area_struct *vma, void *arg) 816 { 817 struct page_referenced_arg *pra = arg; 818 struct mem_cgroup *memcg = pra->memcg; 819 820 if (!mm_match_cgroup(vma->vm_mm, memcg)) 821 return true; 822 823 return false; 824 } 825 826 /** 827 * page_referenced - test if the page was referenced 828 * @page: the page to test 829 * @is_locked: caller holds lock on the page 830 * @memcg: target memory cgroup 831 * @vm_flags: collect encountered vma->vm_flags who actually referenced the page 832 * 833 * Quick test_and_clear_referenced for all mappings to a page, 834 * returns the number of ptes which referenced the page. 835 */ 836 int page_referenced(struct page *page, 837 int is_locked, 838 struct mem_cgroup *memcg, 839 unsigned long *vm_flags) 840 { 841 int we_locked = 0; 842 struct page_referenced_arg pra = { 843 .mapcount = total_mapcount(page), 844 .memcg = memcg, 845 }; 846 struct rmap_walk_control rwc = { 847 .rmap_one = page_referenced_one, 848 .arg = (void *)&pra, 849 .anon_lock = page_lock_anon_vma_read, 850 }; 851 852 *vm_flags = 0; 853 if (!pra.mapcount) 854 return 0; 855 856 if (!page_rmapping(page)) 857 return 0; 858 859 if (!is_locked && (!PageAnon(page) || PageKsm(page))) { 860 we_locked = trylock_page(page); 861 if (!we_locked) 862 return 1; 863 } 864 865 /* 866 * If we are reclaiming on behalf of a cgroup, skip 867 * counting on behalf of references from different 868 * cgroups 869 */ 870 if (memcg) { 871 rwc.invalid_vma = invalid_page_referenced_vma; 872 } 873 874 rmap_walk(page, &rwc); 875 *vm_flags = pra.vm_flags; 876 877 if (we_locked) 878 unlock_page(page); 879 880 return pra.referenced; 881 } 882 883 static bool page_mkclean_one(struct page *page, struct vm_area_struct *vma, 884 unsigned long address, void *arg) 885 { 886 struct page_vma_mapped_walk pvmw = { 887 .page = page, 888 .vma = vma, 889 .address = address, 890 .flags = PVMW_SYNC, 891 }; 892 struct mmu_notifier_range range; 893 int *cleaned = arg; 894 895 /* 896 * We have to assume the worse case ie pmd for invalidation. Note that 897 * the page can not be free from this function. 898 */ 899 mmu_notifier_range_init(&range, MMU_NOTIFY_PROTECTION_PAGE, 900 0, vma, vma->vm_mm, address, 901 min(vma->vm_end, address + page_size(page))); 902 mmu_notifier_invalidate_range_start(&range); 903 904 while (page_vma_mapped_walk(&pvmw)) { 905 int ret = 0; 906 907 address = pvmw.address; 908 if (pvmw.pte) { 909 pte_t entry; 910 pte_t *pte = pvmw.pte; 911 912 if (!pte_dirty(*pte) && !pte_write(*pte)) 913 continue; 914 915 flush_cache_page(vma, address, pte_pfn(*pte)); 916 entry = ptep_clear_flush(vma, address, pte); 917 entry = pte_wrprotect(entry); 918 entry = pte_mkclean(entry); 919 set_pte_at(vma->vm_mm, address, pte, entry); 920 ret = 1; 921 } else { 922 #ifdef CONFIG_TRANSPARENT_HUGE_PAGECACHE 923 pmd_t *pmd = pvmw.pmd; 924 pmd_t entry; 925 926 if (!pmd_dirty(*pmd) && !pmd_write(*pmd)) 927 continue; 928 929 flush_cache_page(vma, address, page_to_pfn(page)); 930 entry = pmdp_invalidate(vma, address, pmd); 931 entry = pmd_wrprotect(entry); 932 entry = pmd_mkclean(entry); 933 set_pmd_at(vma->vm_mm, address, pmd, entry); 934 ret = 1; 935 #else 936 /* unexpected pmd-mapped page? */ 937 WARN_ON_ONCE(1); 938 #endif 939 } 940 941 /* 942 * No need to call mmu_notifier_invalidate_range() as we are 943 * downgrading page table protection not changing it to point 944 * to a new page. 945 * 946 * See Documentation/vm/mmu_notifier.rst 947 */ 948 if (ret) 949 (*cleaned)++; 950 } 951 952 mmu_notifier_invalidate_range_end(&range); 953 954 return true; 955 } 956 957 static bool invalid_mkclean_vma(struct vm_area_struct *vma, void *arg) 958 { 959 if (vma->vm_flags & VM_SHARED) 960 return false; 961 962 return true; 963 } 964 965 int page_mkclean(struct page *page) 966 { 967 int cleaned = 0; 968 struct address_space *mapping; 969 struct rmap_walk_control rwc = { 970 .arg = (void *)&cleaned, 971 .rmap_one = page_mkclean_one, 972 .invalid_vma = invalid_mkclean_vma, 973 }; 974 975 BUG_ON(!PageLocked(page)); 976 977 if (!page_mapped(page)) 978 return 0; 979 980 mapping = page_mapping(page); 981 if (!mapping) 982 return 0; 983 984 rmap_walk(page, &rwc); 985 986 return cleaned; 987 } 988 EXPORT_SYMBOL_GPL(page_mkclean); 989 990 /** 991 * page_move_anon_rmap - move a page to our anon_vma 992 * @page: the page to move to our anon_vma 993 * @vma: the vma the page belongs to 994 * 995 * When a page belongs exclusively to one process after a COW event, 996 * that page can be moved into the anon_vma that belongs to just that 997 * process, so the rmap code will not search the parent or sibling 998 * processes. 999 */ 1000 void page_move_anon_rmap(struct page *page, struct vm_area_struct *vma) 1001 { 1002 struct anon_vma *anon_vma = vma->anon_vma; 1003 1004 page = compound_head(page); 1005 1006 VM_BUG_ON_PAGE(!PageLocked(page), page); 1007 VM_BUG_ON_VMA(!anon_vma, vma); 1008 1009 anon_vma = (void *) anon_vma + PAGE_MAPPING_ANON; 1010 /* 1011 * Ensure that anon_vma and the PAGE_MAPPING_ANON bit are written 1012 * simultaneously, so a concurrent reader (eg page_referenced()'s 1013 * PageAnon()) will not see one without the other. 1014 */ 1015 WRITE_ONCE(page->mapping, (struct address_space *) anon_vma); 1016 } 1017 1018 /** 1019 * __page_set_anon_rmap - set up new anonymous rmap 1020 * @page: Page or Hugepage to add to rmap 1021 * @vma: VM area to add page to. 1022 * @address: User virtual address of the mapping 1023 * @exclusive: the page is exclusively owned by the current process 1024 */ 1025 static void __page_set_anon_rmap(struct page *page, 1026 struct vm_area_struct *vma, unsigned long address, int exclusive) 1027 { 1028 struct anon_vma *anon_vma = vma->anon_vma; 1029 1030 BUG_ON(!anon_vma); 1031 1032 if (PageAnon(page)) 1033 return; 1034 1035 /* 1036 * If the page isn't exclusively mapped into this vma, 1037 * we must use the _oldest_ possible anon_vma for the 1038 * page mapping! 1039 */ 1040 if (!exclusive) 1041 anon_vma = anon_vma->root; 1042 1043 anon_vma = (void *) anon_vma + PAGE_MAPPING_ANON; 1044 page->mapping = (struct address_space *) anon_vma; 1045 page->index = linear_page_index(vma, address); 1046 } 1047 1048 /** 1049 * __page_check_anon_rmap - sanity check anonymous rmap addition 1050 * @page: the page to add the mapping to 1051 * @vma: the vm area in which the mapping is added 1052 * @address: the user virtual address mapped 1053 */ 1054 static void __page_check_anon_rmap(struct page *page, 1055 struct vm_area_struct *vma, unsigned long address) 1056 { 1057 #ifdef CONFIG_DEBUG_VM 1058 /* 1059 * The page's anon-rmap details (mapping and index) are guaranteed to 1060 * be set up correctly at this point. 1061 * 1062 * We have exclusion against page_add_anon_rmap because the caller 1063 * always holds the page locked, except if called from page_dup_rmap, 1064 * in which case the page is already known to be setup. 1065 * 1066 * We have exclusion against page_add_new_anon_rmap because those pages 1067 * are initially only visible via the pagetables, and the pte is locked 1068 * over the call to page_add_new_anon_rmap. 1069 */ 1070 BUG_ON(page_anon_vma(page)->root != vma->anon_vma->root); 1071 BUG_ON(page_to_pgoff(page) != linear_page_index(vma, address)); 1072 #endif 1073 } 1074 1075 /** 1076 * page_add_anon_rmap - add pte mapping to an anonymous page 1077 * @page: the page to add the mapping to 1078 * @vma: the vm area in which the mapping is added 1079 * @address: the user virtual address mapped 1080 * @compound: charge the page as compound or small page 1081 * 1082 * The caller needs to hold the pte lock, and the page must be locked in 1083 * the anon_vma case: to serialize mapping,index checking after setting, 1084 * and to ensure that PageAnon is not being upgraded racily to PageKsm 1085 * (but PageKsm is never downgraded to PageAnon). 1086 */ 1087 void page_add_anon_rmap(struct page *page, 1088 struct vm_area_struct *vma, unsigned long address, bool compound) 1089 { 1090 do_page_add_anon_rmap(page, vma, address, compound ? RMAP_COMPOUND : 0); 1091 } 1092 1093 /* 1094 * Special version of the above for do_swap_page, which often runs 1095 * into pages that are exclusively owned by the current process. 1096 * Everybody else should continue to use page_add_anon_rmap above. 1097 */ 1098 void do_page_add_anon_rmap(struct page *page, 1099 struct vm_area_struct *vma, unsigned long address, int flags) 1100 { 1101 bool compound = flags & RMAP_COMPOUND; 1102 bool first; 1103 1104 if (compound) { 1105 atomic_t *mapcount; 1106 VM_BUG_ON_PAGE(!PageLocked(page), page); 1107 VM_BUG_ON_PAGE(!PageTransHuge(page), page); 1108 mapcount = compound_mapcount_ptr(page); 1109 first = atomic_inc_and_test(mapcount); 1110 } else { 1111 first = atomic_inc_and_test(&page->_mapcount); 1112 } 1113 1114 if (first) { 1115 int nr = compound ? hpage_nr_pages(page) : 1; 1116 /* 1117 * We use the irq-unsafe __{inc|mod}_zone_page_stat because 1118 * these counters are not modified in interrupt context, and 1119 * pte lock(a spinlock) is held, which implies preemption 1120 * disabled. 1121 */ 1122 if (compound) 1123 __inc_node_page_state(page, NR_ANON_THPS); 1124 __mod_node_page_state(page_pgdat(page), NR_ANON_MAPPED, nr); 1125 } 1126 if (unlikely(PageKsm(page))) 1127 return; 1128 1129 VM_BUG_ON_PAGE(!PageLocked(page), page); 1130 1131 /* address might be in next vma when migration races vma_adjust */ 1132 if (first) 1133 __page_set_anon_rmap(page, vma, address, 1134 flags & RMAP_EXCLUSIVE); 1135 else 1136 __page_check_anon_rmap(page, vma, address); 1137 } 1138 1139 /** 1140 * page_add_new_anon_rmap - add pte mapping to a new anonymous page 1141 * @page: the page to add the mapping to 1142 * @vma: the vm area in which the mapping is added 1143 * @address: the user virtual address mapped 1144 * @compound: charge the page as compound or small page 1145 * 1146 * Same as page_add_anon_rmap but must only be called on *new* pages. 1147 * This means the inc-and-test can be bypassed. 1148 * Page does not have to be locked. 1149 */ 1150 void page_add_new_anon_rmap(struct page *page, 1151 struct vm_area_struct *vma, unsigned long address, bool compound) 1152 { 1153 int nr = compound ? hpage_nr_pages(page) : 1; 1154 1155 VM_BUG_ON_VMA(address < vma->vm_start || address >= vma->vm_end, vma); 1156 __SetPageSwapBacked(page); 1157 if (compound) { 1158 VM_BUG_ON_PAGE(!PageTransHuge(page), page); 1159 /* increment count (starts at -1) */ 1160 atomic_set(compound_mapcount_ptr(page), 0); 1161 __inc_node_page_state(page, NR_ANON_THPS); 1162 } else { 1163 /* Anon THP always mapped first with PMD */ 1164 VM_BUG_ON_PAGE(PageTransCompound(page), page); 1165 /* increment count (starts at -1) */ 1166 atomic_set(&page->_mapcount, 0); 1167 } 1168 __mod_node_page_state(page_pgdat(page), NR_ANON_MAPPED, nr); 1169 __page_set_anon_rmap(page, vma, address, 1); 1170 } 1171 1172 /** 1173 * page_add_file_rmap - add pte mapping to a file page 1174 * @page: the page to add the mapping to 1175 * @compound: charge the page as compound or small page 1176 * 1177 * The caller needs to hold the pte lock. 1178 */ 1179 void page_add_file_rmap(struct page *page, bool compound) 1180 { 1181 int i, nr = 1; 1182 1183 VM_BUG_ON_PAGE(compound && !PageTransHuge(page), page); 1184 lock_page_memcg(page); 1185 if (compound && PageTransHuge(page)) { 1186 for (i = 0, nr = 0; i < HPAGE_PMD_NR; i++) { 1187 if (atomic_inc_and_test(&page[i]._mapcount)) 1188 nr++; 1189 } 1190 if (!atomic_inc_and_test(compound_mapcount_ptr(page))) 1191 goto out; 1192 VM_BUG_ON_PAGE(!PageSwapBacked(page), page); 1193 __inc_node_page_state(page, NR_SHMEM_PMDMAPPED); 1194 } else { 1195 if (PageTransCompound(page) && page_mapping(page)) { 1196 VM_WARN_ON_ONCE(!PageLocked(page)); 1197 1198 SetPageDoubleMap(compound_head(page)); 1199 if (PageMlocked(page)) 1200 clear_page_mlock(compound_head(page)); 1201 } 1202 if (!atomic_inc_and_test(&page->_mapcount)) 1203 goto out; 1204 } 1205 __mod_lruvec_page_state(page, NR_FILE_MAPPED, nr); 1206 out: 1207 unlock_page_memcg(page); 1208 } 1209 1210 static void page_remove_file_rmap(struct page *page, bool compound) 1211 { 1212 int i, nr = 1; 1213 1214 VM_BUG_ON_PAGE(compound && !PageHead(page), page); 1215 lock_page_memcg(page); 1216 1217 /* Hugepages are not counted in NR_FILE_MAPPED for now. */ 1218 if (unlikely(PageHuge(page))) { 1219 /* hugetlb pages are always mapped with pmds */ 1220 atomic_dec(compound_mapcount_ptr(page)); 1221 goto out; 1222 } 1223 1224 /* page still mapped by someone else? */ 1225 if (compound && PageTransHuge(page)) { 1226 for (i = 0, nr = 0; i < HPAGE_PMD_NR; i++) { 1227 if (atomic_add_negative(-1, &page[i]._mapcount)) 1228 nr++; 1229 } 1230 if (!atomic_add_negative(-1, compound_mapcount_ptr(page))) 1231 goto out; 1232 VM_BUG_ON_PAGE(!PageSwapBacked(page), page); 1233 __dec_node_page_state(page, NR_SHMEM_PMDMAPPED); 1234 } else { 1235 if (!atomic_add_negative(-1, &page->_mapcount)) 1236 goto out; 1237 } 1238 1239 /* 1240 * We use the irq-unsafe __{inc|mod}_lruvec_page_state because 1241 * these counters are not modified in interrupt context, and 1242 * pte lock(a spinlock) is held, which implies preemption disabled. 1243 */ 1244 __mod_lruvec_page_state(page, NR_FILE_MAPPED, -nr); 1245 1246 if (unlikely(PageMlocked(page))) 1247 clear_page_mlock(page); 1248 out: 1249 unlock_page_memcg(page); 1250 } 1251 1252 static void page_remove_anon_compound_rmap(struct page *page) 1253 { 1254 int i, nr; 1255 1256 if (!atomic_add_negative(-1, compound_mapcount_ptr(page))) 1257 return; 1258 1259 /* Hugepages are not counted in NR_ANON_PAGES for now. */ 1260 if (unlikely(PageHuge(page))) 1261 return; 1262 1263 if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) 1264 return; 1265 1266 __dec_node_page_state(page, NR_ANON_THPS); 1267 1268 if (TestClearPageDoubleMap(page)) { 1269 /* 1270 * Subpages can be mapped with PTEs too. Check how many of 1271 * themi are still mapped. 1272 */ 1273 for (i = 0, nr = 0; i < HPAGE_PMD_NR; i++) { 1274 if (atomic_add_negative(-1, &page[i]._mapcount)) 1275 nr++; 1276 } 1277 } else { 1278 nr = HPAGE_PMD_NR; 1279 } 1280 1281 if (unlikely(PageMlocked(page))) 1282 clear_page_mlock(page); 1283 1284 if (nr) { 1285 __mod_node_page_state(page_pgdat(page), NR_ANON_MAPPED, -nr); 1286 deferred_split_huge_page(page); 1287 } 1288 } 1289 1290 /** 1291 * page_remove_rmap - take down pte mapping from a page 1292 * @page: page to remove mapping from 1293 * @compound: uncharge the page as compound or small page 1294 * 1295 * The caller needs to hold the pte lock. 1296 */ 1297 void page_remove_rmap(struct page *page, bool compound) 1298 { 1299 if (!PageAnon(page)) 1300 return page_remove_file_rmap(page, compound); 1301 1302 if (compound) 1303 return page_remove_anon_compound_rmap(page); 1304 1305 /* page still mapped by someone else? */ 1306 if (!atomic_add_negative(-1, &page->_mapcount)) 1307 return; 1308 1309 /* 1310 * We use the irq-unsafe __{inc|mod}_zone_page_stat because 1311 * these counters are not modified in interrupt context, and 1312 * pte lock(a spinlock) is held, which implies preemption disabled. 1313 */ 1314 __dec_node_page_state(page, NR_ANON_MAPPED); 1315 1316 if (unlikely(PageMlocked(page))) 1317 clear_page_mlock(page); 1318 1319 if (PageTransCompound(page)) 1320 deferred_split_huge_page(compound_head(page)); 1321 1322 /* 1323 * It would be tidy to reset the PageAnon mapping here, 1324 * but that might overwrite a racing page_add_anon_rmap 1325 * which increments mapcount after us but sets mapping 1326 * before us: so leave the reset to free_unref_page, 1327 * and remember that it's only reliable while mapped. 1328 * Leaving it set also helps swapoff to reinstate ptes 1329 * faster for those pages still in swapcache. 1330 */ 1331 } 1332 1333 /* 1334 * @arg: enum ttu_flags will be passed to this argument 1335 */ 1336 static bool try_to_unmap_one(struct page *page, struct vm_area_struct *vma, 1337 unsigned long address, void *arg) 1338 { 1339 struct mm_struct *mm = vma->vm_mm; 1340 struct page_vma_mapped_walk pvmw = { 1341 .page = page, 1342 .vma = vma, 1343 .address = address, 1344 }; 1345 pte_t pteval; 1346 struct page *subpage; 1347 bool ret = true; 1348 struct mmu_notifier_range range; 1349 enum ttu_flags flags = (enum ttu_flags)arg; 1350 1351 /* munlock has nothing to gain from examining un-locked vmas */ 1352 if ((flags & TTU_MUNLOCK) && !(vma->vm_flags & VM_LOCKED)) 1353 return true; 1354 1355 if (IS_ENABLED(CONFIG_MIGRATION) && (flags & TTU_MIGRATION) && 1356 is_zone_device_page(page) && !is_device_private_page(page)) 1357 return true; 1358 1359 if (flags & TTU_SPLIT_HUGE_PMD) { 1360 split_huge_pmd_address(vma, address, 1361 flags & TTU_SPLIT_FREEZE, page); 1362 } 1363 1364 /* 1365 * For THP, we have to assume the worse case ie pmd for invalidation. 1366 * For hugetlb, it could be much worse if we need to do pud 1367 * invalidation in the case of pmd sharing. 1368 * 1369 * Note that the page can not be free in this function as call of 1370 * try_to_unmap() must hold a reference on the page. 1371 */ 1372 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, vma->vm_mm, 1373 address, 1374 min(vma->vm_end, address + page_size(page))); 1375 if (PageHuge(page)) { 1376 /* 1377 * If sharing is possible, start and end will be adjusted 1378 * accordingly. 1379 */ 1380 adjust_range_if_pmd_sharing_possible(vma, &range.start, 1381 &range.end); 1382 } 1383 mmu_notifier_invalidate_range_start(&range); 1384 1385 while (page_vma_mapped_walk(&pvmw)) { 1386 #ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION 1387 /* PMD-mapped THP migration entry */ 1388 if (!pvmw.pte && (flags & TTU_MIGRATION)) { 1389 VM_BUG_ON_PAGE(PageHuge(page) || !PageTransCompound(page), page); 1390 1391 set_pmd_migration_entry(&pvmw, page); 1392 continue; 1393 } 1394 #endif 1395 1396 /* 1397 * If the page is mlock()d, we cannot swap it out. 1398 * If it's recently referenced (perhaps page_referenced 1399 * skipped over this mm) then we should reactivate it. 1400 */ 1401 if (!(flags & TTU_IGNORE_MLOCK)) { 1402 if (vma->vm_flags & VM_LOCKED) { 1403 /* PTE-mapped THP are never mlocked */ 1404 if (!PageTransCompound(page)) { 1405 /* 1406 * Holding pte lock, we do *not* need 1407 * mmap_sem here 1408 */ 1409 mlock_vma_page(page); 1410 } 1411 ret = false; 1412 page_vma_mapped_walk_done(&pvmw); 1413 break; 1414 } 1415 if (flags & TTU_MUNLOCK) 1416 continue; 1417 } 1418 1419 /* Unexpected PMD-mapped THP? */ 1420 VM_BUG_ON_PAGE(!pvmw.pte, page); 1421 1422 subpage = page - page_to_pfn(page) + pte_pfn(*pvmw.pte); 1423 address = pvmw.address; 1424 1425 if (PageHuge(page)) { 1426 if (huge_pmd_unshare(mm, &address, pvmw.pte)) { 1427 /* 1428 * huge_pmd_unshare unmapped an entire PMD 1429 * page. There is no way of knowing exactly 1430 * which PMDs may be cached for this mm, so 1431 * we must flush them all. start/end were 1432 * already adjusted above to cover this range. 1433 */ 1434 flush_cache_range(vma, range.start, range.end); 1435 flush_tlb_range(vma, range.start, range.end); 1436 mmu_notifier_invalidate_range(mm, range.start, 1437 range.end); 1438 1439 /* 1440 * The ref count of the PMD page was dropped 1441 * which is part of the way map counting 1442 * is done for shared PMDs. Return 'true' 1443 * here. When there is no other sharing, 1444 * huge_pmd_unshare returns false and we will 1445 * unmap the actual page and drop map count 1446 * to zero. 1447 */ 1448 page_vma_mapped_walk_done(&pvmw); 1449 break; 1450 } 1451 } 1452 1453 if (IS_ENABLED(CONFIG_MIGRATION) && 1454 (flags & TTU_MIGRATION) && 1455 is_zone_device_page(page)) { 1456 swp_entry_t entry; 1457 pte_t swp_pte; 1458 1459 pteval = ptep_get_and_clear(mm, pvmw.address, pvmw.pte); 1460 1461 /* 1462 * Store the pfn of the page in a special migration 1463 * pte. do_swap_page() will wait until the migration 1464 * pte is removed and then restart fault handling. 1465 */ 1466 entry = make_migration_entry(page, 0); 1467 swp_pte = swp_entry_to_pte(entry); 1468 if (pte_soft_dirty(pteval)) 1469 swp_pte = pte_swp_mksoft_dirty(swp_pte); 1470 set_pte_at(mm, pvmw.address, pvmw.pte, swp_pte); 1471 /* 1472 * No need to invalidate here it will synchronize on 1473 * against the special swap migration pte. 1474 * 1475 * The assignment to subpage above was computed from a 1476 * swap PTE which results in an invalid pointer. 1477 * Since only PAGE_SIZE pages can currently be 1478 * migrated, just set it to page. This will need to be 1479 * changed when hugepage migrations to device private 1480 * memory are supported. 1481 */ 1482 subpage = page; 1483 goto discard; 1484 } 1485 1486 if (!(flags & TTU_IGNORE_ACCESS)) { 1487 if (ptep_clear_flush_young_notify(vma, address, 1488 pvmw.pte)) { 1489 ret = false; 1490 page_vma_mapped_walk_done(&pvmw); 1491 break; 1492 } 1493 } 1494 1495 /* Nuke the page table entry. */ 1496 flush_cache_page(vma, address, pte_pfn(*pvmw.pte)); 1497 if (should_defer_flush(mm, flags)) { 1498 /* 1499 * We clear the PTE but do not flush so potentially 1500 * a remote CPU could still be writing to the page. 1501 * If the entry was previously clean then the 1502 * architecture must guarantee that a clear->dirty 1503 * transition on a cached TLB entry is written through 1504 * and traps if the PTE is unmapped. 1505 */ 1506 pteval = ptep_get_and_clear(mm, address, pvmw.pte); 1507 1508 set_tlb_ubc_flush_pending(mm, pte_dirty(pteval)); 1509 } else { 1510 pteval = ptep_clear_flush(vma, address, pvmw.pte); 1511 } 1512 1513 /* Move the dirty bit to the page. Now the pte is gone. */ 1514 if (pte_dirty(pteval)) 1515 set_page_dirty(page); 1516 1517 /* Update high watermark before we lower rss */ 1518 update_hiwater_rss(mm); 1519 1520 if (PageHWPoison(page) && !(flags & TTU_IGNORE_HWPOISON)) { 1521 pteval = swp_entry_to_pte(make_hwpoison_entry(subpage)); 1522 if (PageHuge(page)) { 1523 int nr = 1 << compound_order(page); 1524 hugetlb_count_sub(nr, mm); 1525 set_huge_swap_pte_at(mm, address, 1526 pvmw.pte, pteval, 1527 vma_mmu_pagesize(vma)); 1528 } else { 1529 dec_mm_counter(mm, mm_counter(page)); 1530 set_pte_at(mm, address, pvmw.pte, pteval); 1531 } 1532 1533 } else if (pte_unused(pteval) && !userfaultfd_armed(vma)) { 1534 /* 1535 * The guest indicated that the page content is of no 1536 * interest anymore. Simply discard the pte, vmscan 1537 * will take care of the rest. 1538 * A future reference will then fault in a new zero 1539 * page. When userfaultfd is active, we must not drop 1540 * this page though, as its main user (postcopy 1541 * migration) will not expect userfaults on already 1542 * copied pages. 1543 */ 1544 dec_mm_counter(mm, mm_counter(page)); 1545 /* We have to invalidate as we cleared the pte */ 1546 mmu_notifier_invalidate_range(mm, address, 1547 address + PAGE_SIZE); 1548 } else if (IS_ENABLED(CONFIG_MIGRATION) && 1549 (flags & (TTU_MIGRATION|TTU_SPLIT_FREEZE))) { 1550 swp_entry_t entry; 1551 pte_t swp_pte; 1552 1553 if (arch_unmap_one(mm, vma, address, pteval) < 0) { 1554 set_pte_at(mm, address, pvmw.pte, pteval); 1555 ret = false; 1556 page_vma_mapped_walk_done(&pvmw); 1557 break; 1558 } 1559 1560 /* 1561 * Store the pfn of the page in a special migration 1562 * pte. do_swap_page() will wait until the migration 1563 * pte is removed and then restart fault handling. 1564 */ 1565 entry = make_migration_entry(subpage, 1566 pte_write(pteval)); 1567 swp_pte = swp_entry_to_pte(entry); 1568 if (pte_soft_dirty(pteval)) 1569 swp_pte = pte_swp_mksoft_dirty(swp_pte); 1570 set_pte_at(mm, address, pvmw.pte, swp_pte); 1571 /* 1572 * No need to invalidate here it will synchronize on 1573 * against the special swap migration pte. 1574 */ 1575 } else if (PageAnon(page)) { 1576 swp_entry_t entry = { .val = page_private(subpage) }; 1577 pte_t swp_pte; 1578 /* 1579 * Store the swap location in the pte. 1580 * See handle_pte_fault() ... 1581 */ 1582 if (unlikely(PageSwapBacked(page) != PageSwapCache(page))) { 1583 WARN_ON_ONCE(1); 1584 ret = false; 1585 /* We have to invalidate as we cleared the pte */ 1586 mmu_notifier_invalidate_range(mm, address, 1587 address + PAGE_SIZE); 1588 page_vma_mapped_walk_done(&pvmw); 1589 break; 1590 } 1591 1592 /* MADV_FREE page check */ 1593 if (!PageSwapBacked(page)) { 1594 if (!PageDirty(page)) { 1595 /* Invalidate as we cleared the pte */ 1596 mmu_notifier_invalidate_range(mm, 1597 address, address + PAGE_SIZE); 1598 dec_mm_counter(mm, MM_ANONPAGES); 1599 goto discard; 1600 } 1601 1602 /* 1603 * If the page was redirtied, it cannot be 1604 * discarded. Remap the page to page table. 1605 */ 1606 set_pte_at(mm, address, pvmw.pte, pteval); 1607 SetPageSwapBacked(page); 1608 ret = false; 1609 page_vma_mapped_walk_done(&pvmw); 1610 break; 1611 } 1612 1613 if (swap_duplicate(entry) < 0) { 1614 set_pte_at(mm, address, pvmw.pte, pteval); 1615 ret = false; 1616 page_vma_mapped_walk_done(&pvmw); 1617 break; 1618 } 1619 if (arch_unmap_one(mm, vma, address, pteval) < 0) { 1620 set_pte_at(mm, address, pvmw.pte, pteval); 1621 ret = false; 1622 page_vma_mapped_walk_done(&pvmw); 1623 break; 1624 } 1625 if (list_empty(&mm->mmlist)) { 1626 spin_lock(&mmlist_lock); 1627 if (list_empty(&mm->mmlist)) 1628 list_add(&mm->mmlist, &init_mm.mmlist); 1629 spin_unlock(&mmlist_lock); 1630 } 1631 dec_mm_counter(mm, MM_ANONPAGES); 1632 inc_mm_counter(mm, MM_SWAPENTS); 1633 swp_pte = swp_entry_to_pte(entry); 1634 if (pte_soft_dirty(pteval)) 1635 swp_pte = pte_swp_mksoft_dirty(swp_pte); 1636 set_pte_at(mm, address, pvmw.pte, swp_pte); 1637 /* Invalidate as we cleared the pte */ 1638 mmu_notifier_invalidate_range(mm, address, 1639 address + PAGE_SIZE); 1640 } else { 1641 /* 1642 * This is a locked file-backed page, thus it cannot 1643 * be removed from the page cache and replaced by a new 1644 * page before mmu_notifier_invalidate_range_end, so no 1645 * concurrent thread might update its page table to 1646 * point at new page while a device still is using this 1647 * page. 1648 * 1649 * See Documentation/vm/mmu_notifier.rst 1650 */ 1651 dec_mm_counter(mm, mm_counter_file(page)); 1652 } 1653 discard: 1654 /* 1655 * No need to call mmu_notifier_invalidate_range() it has be 1656 * done above for all cases requiring it to happen under page 1657 * table lock before mmu_notifier_invalidate_range_end() 1658 * 1659 * See Documentation/vm/mmu_notifier.rst 1660 */ 1661 page_remove_rmap(subpage, PageHuge(page)); 1662 put_page(page); 1663 } 1664 1665 mmu_notifier_invalidate_range_end(&range); 1666 1667 return ret; 1668 } 1669 1670 bool is_vma_temporary_stack(struct vm_area_struct *vma) 1671 { 1672 int maybe_stack = vma->vm_flags & (VM_GROWSDOWN | VM_GROWSUP); 1673 1674 if (!maybe_stack) 1675 return false; 1676 1677 if ((vma->vm_flags & VM_STACK_INCOMPLETE_SETUP) == 1678 VM_STACK_INCOMPLETE_SETUP) 1679 return true; 1680 1681 return false; 1682 } 1683 1684 static bool invalid_migration_vma(struct vm_area_struct *vma, void *arg) 1685 { 1686 return is_vma_temporary_stack(vma); 1687 } 1688 1689 static int page_mapcount_is_zero(struct page *page) 1690 { 1691 return !total_mapcount(page); 1692 } 1693 1694 /** 1695 * try_to_unmap - try to remove all page table mappings to a page 1696 * @page: the page to get unmapped 1697 * @flags: action and flags 1698 * 1699 * Tries to remove all the page table entries which are mapping this 1700 * page, used in the pageout path. Caller must hold the page lock. 1701 * 1702 * If unmap is successful, return true. Otherwise, false. 1703 */ 1704 bool try_to_unmap(struct page *page, enum ttu_flags flags) 1705 { 1706 struct rmap_walk_control rwc = { 1707 .rmap_one = try_to_unmap_one, 1708 .arg = (void *)flags, 1709 .done = page_mapcount_is_zero, 1710 .anon_lock = page_lock_anon_vma_read, 1711 }; 1712 1713 /* 1714 * During exec, a temporary VMA is setup and later moved. 1715 * The VMA is moved under the anon_vma lock but not the 1716 * page tables leading to a race where migration cannot 1717 * find the migration ptes. Rather than increasing the 1718 * locking requirements of exec(), migration skips 1719 * temporary VMAs until after exec() completes. 1720 */ 1721 if ((flags & (TTU_MIGRATION|TTU_SPLIT_FREEZE)) 1722 && !PageKsm(page) && PageAnon(page)) 1723 rwc.invalid_vma = invalid_migration_vma; 1724 1725 if (flags & TTU_RMAP_LOCKED) 1726 rmap_walk_locked(page, &rwc); 1727 else 1728 rmap_walk(page, &rwc); 1729 1730 return !page_mapcount(page) ? true : false; 1731 } 1732 1733 static int page_not_mapped(struct page *page) 1734 { 1735 return !page_mapped(page); 1736 }; 1737 1738 /** 1739 * try_to_munlock - try to munlock a page 1740 * @page: the page to be munlocked 1741 * 1742 * Called from munlock code. Checks all of the VMAs mapping the page 1743 * to make sure nobody else has this page mlocked. The page will be 1744 * returned with PG_mlocked cleared if no other vmas have it mlocked. 1745 */ 1746 1747 void try_to_munlock(struct page *page) 1748 { 1749 struct rmap_walk_control rwc = { 1750 .rmap_one = try_to_unmap_one, 1751 .arg = (void *)TTU_MUNLOCK, 1752 .done = page_not_mapped, 1753 .anon_lock = page_lock_anon_vma_read, 1754 1755 }; 1756 1757 VM_BUG_ON_PAGE(!PageLocked(page) || PageLRU(page), page); 1758 VM_BUG_ON_PAGE(PageCompound(page) && PageDoubleMap(page), page); 1759 1760 rmap_walk(page, &rwc); 1761 } 1762 1763 void __put_anon_vma(struct anon_vma *anon_vma) 1764 { 1765 struct anon_vma *root = anon_vma->root; 1766 1767 anon_vma_free(anon_vma); 1768 if (root != anon_vma && atomic_dec_and_test(&root->refcount)) 1769 anon_vma_free(root); 1770 } 1771 1772 static struct anon_vma *rmap_walk_anon_lock(struct page *page, 1773 struct rmap_walk_control *rwc) 1774 { 1775 struct anon_vma *anon_vma; 1776 1777 if (rwc->anon_lock) 1778 return rwc->anon_lock(page); 1779 1780 /* 1781 * Note: remove_migration_ptes() cannot use page_lock_anon_vma_read() 1782 * because that depends on page_mapped(); but not all its usages 1783 * are holding mmap_sem. Users without mmap_sem are required to 1784 * take a reference count to prevent the anon_vma disappearing 1785 */ 1786 anon_vma = page_anon_vma(page); 1787 if (!anon_vma) 1788 return NULL; 1789 1790 anon_vma_lock_read(anon_vma); 1791 return anon_vma; 1792 } 1793 1794 /* 1795 * rmap_walk_anon - do something to anonymous page using the object-based 1796 * rmap method 1797 * @page: the page to be handled 1798 * @rwc: control variable according to each walk type 1799 * 1800 * Find all the mappings of a page using the mapping pointer and the vma chains 1801 * contained in the anon_vma struct it points to. 1802 * 1803 * When called from try_to_munlock(), the mmap_sem of the mm containing the vma 1804 * where the page was found will be held for write. So, we won't recheck 1805 * vm_flags for that VMA. That should be OK, because that vma shouldn't be 1806 * LOCKED. 1807 */ 1808 static void rmap_walk_anon(struct page *page, struct rmap_walk_control *rwc, 1809 bool locked) 1810 { 1811 struct anon_vma *anon_vma; 1812 pgoff_t pgoff_start, pgoff_end; 1813 struct anon_vma_chain *avc; 1814 1815 if (locked) { 1816 anon_vma = page_anon_vma(page); 1817 /* anon_vma disappear under us? */ 1818 VM_BUG_ON_PAGE(!anon_vma, page); 1819 } else { 1820 anon_vma = rmap_walk_anon_lock(page, rwc); 1821 } 1822 if (!anon_vma) 1823 return; 1824 1825 pgoff_start = page_to_pgoff(page); 1826 pgoff_end = pgoff_start + hpage_nr_pages(page) - 1; 1827 anon_vma_interval_tree_foreach(avc, &anon_vma->rb_root, 1828 pgoff_start, pgoff_end) { 1829 struct vm_area_struct *vma = avc->vma; 1830 unsigned long address = vma_address(page, vma); 1831 1832 cond_resched(); 1833 1834 if (rwc->invalid_vma && rwc->invalid_vma(vma, rwc->arg)) 1835 continue; 1836 1837 if (!rwc->rmap_one(page, vma, address, rwc->arg)) 1838 break; 1839 if (rwc->done && rwc->done(page)) 1840 break; 1841 } 1842 1843 if (!locked) 1844 anon_vma_unlock_read(anon_vma); 1845 } 1846 1847 /* 1848 * rmap_walk_file - do something to file page using the object-based rmap method 1849 * @page: the page to be handled 1850 * @rwc: control variable according to each walk type 1851 * 1852 * Find all the mappings of a page using the mapping pointer and the vma chains 1853 * contained in the address_space struct it points to. 1854 * 1855 * When called from try_to_munlock(), the mmap_sem of the mm containing the vma 1856 * where the page was found will be held for write. So, we won't recheck 1857 * vm_flags for that VMA. That should be OK, because that vma shouldn't be 1858 * LOCKED. 1859 */ 1860 static void rmap_walk_file(struct page *page, struct rmap_walk_control *rwc, 1861 bool locked) 1862 { 1863 struct address_space *mapping = page_mapping(page); 1864 pgoff_t pgoff_start, pgoff_end; 1865 struct vm_area_struct *vma; 1866 1867 /* 1868 * The page lock not only makes sure that page->mapping cannot 1869 * suddenly be NULLified by truncation, it makes sure that the 1870 * structure at mapping cannot be freed and reused yet, 1871 * so we can safely take mapping->i_mmap_rwsem. 1872 */ 1873 VM_BUG_ON_PAGE(!PageLocked(page), page); 1874 1875 if (!mapping) 1876 return; 1877 1878 pgoff_start = page_to_pgoff(page); 1879 pgoff_end = pgoff_start + hpage_nr_pages(page) - 1; 1880 if (!locked) 1881 i_mmap_lock_read(mapping); 1882 vma_interval_tree_foreach(vma, &mapping->i_mmap, 1883 pgoff_start, pgoff_end) { 1884 unsigned long address = vma_address(page, vma); 1885 1886 cond_resched(); 1887 1888 if (rwc->invalid_vma && rwc->invalid_vma(vma, rwc->arg)) 1889 continue; 1890 1891 if (!rwc->rmap_one(page, vma, address, rwc->arg)) 1892 goto done; 1893 if (rwc->done && rwc->done(page)) 1894 goto done; 1895 } 1896 1897 done: 1898 if (!locked) 1899 i_mmap_unlock_read(mapping); 1900 } 1901 1902 void rmap_walk(struct page *page, struct rmap_walk_control *rwc) 1903 { 1904 if (unlikely(PageKsm(page))) 1905 rmap_walk_ksm(page, rwc); 1906 else if (PageAnon(page)) 1907 rmap_walk_anon(page, rwc, false); 1908 else 1909 rmap_walk_file(page, rwc, false); 1910 } 1911 1912 /* Like rmap_walk, but caller holds relevant rmap lock */ 1913 void rmap_walk_locked(struct page *page, struct rmap_walk_control *rwc) 1914 { 1915 /* no ksm support for now */ 1916 VM_BUG_ON_PAGE(PageKsm(page), page); 1917 if (PageAnon(page)) 1918 rmap_walk_anon(page, rwc, true); 1919 else 1920 rmap_walk_file(page, rwc, true); 1921 } 1922 1923 #ifdef CONFIG_HUGETLB_PAGE 1924 /* 1925 * The following two functions are for anonymous (private mapped) hugepages. 1926 * Unlike common anonymous pages, anonymous hugepages have no accounting code 1927 * and no lru code, because we handle hugepages differently from common pages. 1928 */ 1929 void hugepage_add_anon_rmap(struct page *page, 1930 struct vm_area_struct *vma, unsigned long address) 1931 { 1932 struct anon_vma *anon_vma = vma->anon_vma; 1933 int first; 1934 1935 BUG_ON(!PageLocked(page)); 1936 BUG_ON(!anon_vma); 1937 /* address might be in next vma when migration races vma_adjust */ 1938 first = atomic_inc_and_test(compound_mapcount_ptr(page)); 1939 if (first) 1940 __page_set_anon_rmap(page, vma, address, 0); 1941 } 1942 1943 void hugepage_add_new_anon_rmap(struct page *page, 1944 struct vm_area_struct *vma, unsigned long address) 1945 { 1946 BUG_ON(address < vma->vm_start || address >= vma->vm_end); 1947 atomic_set(compound_mapcount_ptr(page), 0); 1948 __page_set_anon_rmap(page, vma, address, 1); 1949 } 1950 #endif /* CONFIG_HUGETLB_PAGE */ 1951