1 // SPDX-License-Identifier: GPL-2.0 2 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 3 4 #include <linux/mm.h> 5 #include <linux/sched.h> 6 #include <linux/sched/mm.h> 7 #include <linux/sched/coredump.h> 8 #include <linux/mmu_notifier.h> 9 #include <linux/rmap.h> 10 #include <linux/swap.h> 11 #include <linux/mm_inline.h> 12 #include <linux/kthread.h> 13 #include <linux/khugepaged.h> 14 #include <linux/freezer.h> 15 #include <linux/mman.h> 16 #include <linux/hashtable.h> 17 #include <linux/userfaultfd_k.h> 18 #include <linux/page_idle.h> 19 #include <linux/page_table_check.h> 20 #include <linux/swapops.h> 21 #include <linux/shmem_fs.h> 22 23 #include <asm/tlb.h> 24 #include <asm/pgalloc.h> 25 #include "internal.h" 26 27 enum scan_result { 28 SCAN_FAIL, 29 SCAN_SUCCEED, 30 SCAN_PMD_NULL, 31 SCAN_EXCEED_NONE_PTE, 32 SCAN_EXCEED_SWAP_PTE, 33 SCAN_EXCEED_SHARED_PTE, 34 SCAN_PTE_NON_PRESENT, 35 SCAN_PTE_UFFD_WP, 36 SCAN_PAGE_RO, 37 SCAN_LACK_REFERENCED_PAGE, 38 SCAN_PAGE_NULL, 39 SCAN_SCAN_ABORT, 40 SCAN_PAGE_COUNT, 41 SCAN_PAGE_LRU, 42 SCAN_PAGE_LOCK, 43 SCAN_PAGE_ANON, 44 SCAN_PAGE_COMPOUND, 45 SCAN_ANY_PROCESS, 46 SCAN_VMA_NULL, 47 SCAN_VMA_CHECK, 48 SCAN_ADDRESS_RANGE, 49 SCAN_DEL_PAGE_LRU, 50 SCAN_ALLOC_HUGE_PAGE_FAIL, 51 SCAN_CGROUP_CHARGE_FAIL, 52 SCAN_TRUNCATED, 53 SCAN_PAGE_HAS_PRIVATE, 54 }; 55 56 #define CREATE_TRACE_POINTS 57 #include <trace/events/huge_memory.h> 58 59 static struct task_struct *khugepaged_thread __read_mostly; 60 static DEFINE_MUTEX(khugepaged_mutex); 61 62 /* default scan 8*512 pte (or vmas) every 30 second */ 63 static unsigned int khugepaged_pages_to_scan __read_mostly; 64 static unsigned int khugepaged_pages_collapsed; 65 static unsigned int khugepaged_full_scans; 66 static unsigned int khugepaged_scan_sleep_millisecs __read_mostly = 10000; 67 /* during fragmentation poll the hugepage allocator once every minute */ 68 static unsigned int khugepaged_alloc_sleep_millisecs __read_mostly = 60000; 69 static unsigned long khugepaged_sleep_expire; 70 static DEFINE_SPINLOCK(khugepaged_mm_lock); 71 static DECLARE_WAIT_QUEUE_HEAD(khugepaged_wait); 72 /* 73 * default collapse hugepages if there is at least one pte mapped like 74 * it would have happened if the vma was large enough during page 75 * fault. 76 */ 77 static unsigned int khugepaged_max_ptes_none __read_mostly; 78 static unsigned int khugepaged_max_ptes_swap __read_mostly; 79 static unsigned int khugepaged_max_ptes_shared __read_mostly; 80 81 #define MM_SLOTS_HASH_BITS 10 82 static __read_mostly DEFINE_HASHTABLE(mm_slots_hash, MM_SLOTS_HASH_BITS); 83 84 static struct kmem_cache *mm_slot_cache __read_mostly; 85 86 #define MAX_PTE_MAPPED_THP 8 87 88 /** 89 * struct mm_slot - hash lookup from mm to mm_slot 90 * @hash: hash collision list 91 * @mm_node: khugepaged scan list headed in khugepaged_scan.mm_head 92 * @mm: the mm that this information is valid for 93 * @nr_pte_mapped_thp: number of pte mapped THP 94 * @pte_mapped_thp: address array corresponding pte mapped THP 95 */ 96 struct mm_slot { 97 struct hlist_node hash; 98 struct list_head mm_node; 99 struct mm_struct *mm; 100 101 /* pte-mapped THP in this mm */ 102 int nr_pte_mapped_thp; 103 unsigned long pte_mapped_thp[MAX_PTE_MAPPED_THP]; 104 }; 105 106 /** 107 * struct khugepaged_scan - cursor for scanning 108 * @mm_head: the head of the mm list to scan 109 * @mm_slot: the current mm_slot we are scanning 110 * @address: the next address inside that to be scanned 111 * 112 * There is only the one khugepaged_scan instance of this cursor structure. 113 */ 114 struct khugepaged_scan { 115 struct list_head mm_head; 116 struct mm_slot *mm_slot; 117 unsigned long address; 118 }; 119 120 static struct khugepaged_scan khugepaged_scan = { 121 .mm_head = LIST_HEAD_INIT(khugepaged_scan.mm_head), 122 }; 123 124 #ifdef CONFIG_SYSFS 125 static ssize_t scan_sleep_millisecs_show(struct kobject *kobj, 126 struct kobj_attribute *attr, 127 char *buf) 128 { 129 return sysfs_emit(buf, "%u\n", khugepaged_scan_sleep_millisecs); 130 } 131 132 static ssize_t scan_sleep_millisecs_store(struct kobject *kobj, 133 struct kobj_attribute *attr, 134 const char *buf, size_t count) 135 { 136 unsigned int msecs; 137 int err; 138 139 err = kstrtouint(buf, 10, &msecs); 140 if (err) 141 return -EINVAL; 142 143 khugepaged_scan_sleep_millisecs = msecs; 144 khugepaged_sleep_expire = 0; 145 wake_up_interruptible(&khugepaged_wait); 146 147 return count; 148 } 149 static struct kobj_attribute scan_sleep_millisecs_attr = 150 __ATTR_RW(scan_sleep_millisecs); 151 152 static ssize_t alloc_sleep_millisecs_show(struct kobject *kobj, 153 struct kobj_attribute *attr, 154 char *buf) 155 { 156 return sysfs_emit(buf, "%u\n", khugepaged_alloc_sleep_millisecs); 157 } 158 159 static ssize_t alloc_sleep_millisecs_store(struct kobject *kobj, 160 struct kobj_attribute *attr, 161 const char *buf, size_t count) 162 { 163 unsigned int msecs; 164 int err; 165 166 err = kstrtouint(buf, 10, &msecs); 167 if (err) 168 return -EINVAL; 169 170 khugepaged_alloc_sleep_millisecs = msecs; 171 khugepaged_sleep_expire = 0; 172 wake_up_interruptible(&khugepaged_wait); 173 174 return count; 175 } 176 static struct kobj_attribute alloc_sleep_millisecs_attr = 177 __ATTR_RW(alloc_sleep_millisecs); 178 179 static ssize_t pages_to_scan_show(struct kobject *kobj, 180 struct kobj_attribute *attr, 181 char *buf) 182 { 183 return sysfs_emit(buf, "%u\n", khugepaged_pages_to_scan); 184 } 185 static ssize_t pages_to_scan_store(struct kobject *kobj, 186 struct kobj_attribute *attr, 187 const char *buf, size_t count) 188 { 189 unsigned int pages; 190 int err; 191 192 err = kstrtouint(buf, 10, &pages); 193 if (err || !pages) 194 return -EINVAL; 195 196 khugepaged_pages_to_scan = pages; 197 198 return count; 199 } 200 static struct kobj_attribute pages_to_scan_attr = 201 __ATTR_RW(pages_to_scan); 202 203 static ssize_t pages_collapsed_show(struct kobject *kobj, 204 struct kobj_attribute *attr, 205 char *buf) 206 { 207 return sysfs_emit(buf, "%u\n", khugepaged_pages_collapsed); 208 } 209 static struct kobj_attribute pages_collapsed_attr = 210 __ATTR_RO(pages_collapsed); 211 212 static ssize_t full_scans_show(struct kobject *kobj, 213 struct kobj_attribute *attr, 214 char *buf) 215 { 216 return sysfs_emit(buf, "%u\n", khugepaged_full_scans); 217 } 218 static struct kobj_attribute full_scans_attr = 219 __ATTR_RO(full_scans); 220 221 static ssize_t defrag_show(struct kobject *kobj, 222 struct kobj_attribute *attr, char *buf) 223 { 224 return single_hugepage_flag_show(kobj, attr, buf, 225 TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG); 226 } 227 static ssize_t defrag_store(struct kobject *kobj, 228 struct kobj_attribute *attr, 229 const char *buf, size_t count) 230 { 231 return single_hugepage_flag_store(kobj, attr, buf, count, 232 TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG); 233 } 234 static struct kobj_attribute khugepaged_defrag_attr = 235 __ATTR_RW(defrag); 236 237 /* 238 * max_ptes_none controls if khugepaged should collapse hugepages over 239 * any unmapped ptes in turn potentially increasing the memory 240 * footprint of the vmas. When max_ptes_none is 0 khugepaged will not 241 * reduce the available free memory in the system as it 242 * runs. Increasing max_ptes_none will instead potentially reduce the 243 * free memory in the system during the khugepaged scan. 244 */ 245 static ssize_t max_ptes_none_show(struct kobject *kobj, 246 struct kobj_attribute *attr, 247 char *buf) 248 { 249 return sysfs_emit(buf, "%u\n", khugepaged_max_ptes_none); 250 } 251 static ssize_t max_ptes_none_store(struct kobject *kobj, 252 struct kobj_attribute *attr, 253 const char *buf, size_t count) 254 { 255 int err; 256 unsigned long max_ptes_none; 257 258 err = kstrtoul(buf, 10, &max_ptes_none); 259 if (err || max_ptes_none > HPAGE_PMD_NR - 1) 260 return -EINVAL; 261 262 khugepaged_max_ptes_none = max_ptes_none; 263 264 return count; 265 } 266 static struct kobj_attribute khugepaged_max_ptes_none_attr = 267 __ATTR_RW(max_ptes_none); 268 269 static ssize_t max_ptes_swap_show(struct kobject *kobj, 270 struct kobj_attribute *attr, 271 char *buf) 272 { 273 return sysfs_emit(buf, "%u\n", khugepaged_max_ptes_swap); 274 } 275 276 static ssize_t max_ptes_swap_store(struct kobject *kobj, 277 struct kobj_attribute *attr, 278 const char *buf, size_t count) 279 { 280 int err; 281 unsigned long max_ptes_swap; 282 283 err = kstrtoul(buf, 10, &max_ptes_swap); 284 if (err || max_ptes_swap > HPAGE_PMD_NR - 1) 285 return -EINVAL; 286 287 khugepaged_max_ptes_swap = max_ptes_swap; 288 289 return count; 290 } 291 292 static struct kobj_attribute khugepaged_max_ptes_swap_attr = 293 __ATTR_RW(max_ptes_swap); 294 295 static ssize_t max_ptes_shared_show(struct kobject *kobj, 296 struct kobj_attribute *attr, 297 char *buf) 298 { 299 return sysfs_emit(buf, "%u\n", khugepaged_max_ptes_shared); 300 } 301 302 static ssize_t max_ptes_shared_store(struct kobject *kobj, 303 struct kobj_attribute *attr, 304 const char *buf, size_t count) 305 { 306 int err; 307 unsigned long max_ptes_shared; 308 309 err = kstrtoul(buf, 10, &max_ptes_shared); 310 if (err || max_ptes_shared > HPAGE_PMD_NR - 1) 311 return -EINVAL; 312 313 khugepaged_max_ptes_shared = max_ptes_shared; 314 315 return count; 316 } 317 318 static struct kobj_attribute khugepaged_max_ptes_shared_attr = 319 __ATTR_RW(max_ptes_shared); 320 321 static struct attribute *khugepaged_attr[] = { 322 &khugepaged_defrag_attr.attr, 323 &khugepaged_max_ptes_none_attr.attr, 324 &khugepaged_max_ptes_swap_attr.attr, 325 &khugepaged_max_ptes_shared_attr.attr, 326 &pages_to_scan_attr.attr, 327 &pages_collapsed_attr.attr, 328 &full_scans_attr.attr, 329 &scan_sleep_millisecs_attr.attr, 330 &alloc_sleep_millisecs_attr.attr, 331 NULL, 332 }; 333 334 struct attribute_group khugepaged_attr_group = { 335 .attrs = khugepaged_attr, 336 .name = "khugepaged", 337 }; 338 #endif /* CONFIG_SYSFS */ 339 340 int hugepage_madvise(struct vm_area_struct *vma, 341 unsigned long *vm_flags, int advice) 342 { 343 switch (advice) { 344 case MADV_HUGEPAGE: 345 #ifdef CONFIG_S390 346 /* 347 * qemu blindly sets MADV_HUGEPAGE on all allocations, but s390 348 * can't handle this properly after s390_enable_sie, so we simply 349 * ignore the madvise to prevent qemu from causing a SIGSEGV. 350 */ 351 if (mm_has_pgste(vma->vm_mm)) 352 return 0; 353 #endif 354 *vm_flags &= ~VM_NOHUGEPAGE; 355 *vm_flags |= VM_HUGEPAGE; 356 /* 357 * If the vma become good for khugepaged to scan, 358 * register it here without waiting a page fault that 359 * may not happen any time soon. 360 */ 361 khugepaged_enter_vma(vma, *vm_flags); 362 break; 363 case MADV_NOHUGEPAGE: 364 *vm_flags &= ~VM_HUGEPAGE; 365 *vm_flags |= VM_NOHUGEPAGE; 366 /* 367 * Setting VM_NOHUGEPAGE will prevent khugepaged from scanning 368 * this vma even if we leave the mm registered in khugepaged if 369 * it got registered before VM_NOHUGEPAGE was set. 370 */ 371 break; 372 } 373 374 return 0; 375 } 376 377 int __init khugepaged_init(void) 378 { 379 mm_slot_cache = kmem_cache_create("khugepaged_mm_slot", 380 sizeof(struct mm_slot), 381 __alignof__(struct mm_slot), 0, NULL); 382 if (!mm_slot_cache) 383 return -ENOMEM; 384 385 khugepaged_pages_to_scan = HPAGE_PMD_NR * 8; 386 khugepaged_max_ptes_none = HPAGE_PMD_NR - 1; 387 khugepaged_max_ptes_swap = HPAGE_PMD_NR / 8; 388 khugepaged_max_ptes_shared = HPAGE_PMD_NR / 2; 389 390 return 0; 391 } 392 393 void __init khugepaged_destroy(void) 394 { 395 kmem_cache_destroy(mm_slot_cache); 396 } 397 398 static inline struct mm_slot *alloc_mm_slot(void) 399 { 400 if (!mm_slot_cache) /* initialization failed */ 401 return NULL; 402 return kmem_cache_zalloc(mm_slot_cache, GFP_KERNEL); 403 } 404 405 static inline void free_mm_slot(struct mm_slot *mm_slot) 406 { 407 kmem_cache_free(mm_slot_cache, mm_slot); 408 } 409 410 static struct mm_slot *get_mm_slot(struct mm_struct *mm) 411 { 412 struct mm_slot *mm_slot; 413 414 hash_for_each_possible(mm_slots_hash, mm_slot, hash, (unsigned long)mm) 415 if (mm == mm_slot->mm) 416 return mm_slot; 417 418 return NULL; 419 } 420 421 static void insert_to_mm_slots_hash(struct mm_struct *mm, 422 struct mm_slot *mm_slot) 423 { 424 mm_slot->mm = mm; 425 hash_add(mm_slots_hash, &mm_slot->hash, (long)mm); 426 } 427 428 static inline int khugepaged_test_exit(struct mm_struct *mm) 429 { 430 return atomic_read(&mm->mm_users) == 0; 431 } 432 433 bool hugepage_vma_check(struct vm_area_struct *vma, 434 unsigned long vm_flags) 435 { 436 if (!transhuge_vma_enabled(vma, vm_flags)) 437 return false; 438 439 if (vm_flags & VM_NO_KHUGEPAGED) 440 return false; 441 442 /* Don't run khugepaged against DAX vma */ 443 if (vma_is_dax(vma)) 444 return false; 445 446 if (vma->vm_file && !IS_ALIGNED((vma->vm_start >> PAGE_SHIFT) - 447 vma->vm_pgoff, HPAGE_PMD_NR)) 448 return false; 449 450 /* Enabled via shmem mount options or sysfs settings. */ 451 if (shmem_file(vma->vm_file)) 452 return shmem_huge_enabled(vma); 453 454 /* THP settings require madvise. */ 455 if (!(vm_flags & VM_HUGEPAGE) && !khugepaged_always()) 456 return false; 457 458 /* Only regular file is valid */ 459 if (file_thp_enabled(vma)) 460 return true; 461 462 if (!vma->anon_vma || !vma_is_anonymous(vma)) 463 return false; 464 if (vma_is_temporary_stack(vma)) 465 return false; 466 467 return true; 468 } 469 470 void __khugepaged_enter(struct mm_struct *mm) 471 { 472 struct mm_slot *mm_slot; 473 int wakeup; 474 475 mm_slot = alloc_mm_slot(); 476 if (!mm_slot) 477 return; 478 479 /* __khugepaged_exit() must not run from under us */ 480 VM_BUG_ON_MM(khugepaged_test_exit(mm), mm); 481 if (unlikely(test_and_set_bit(MMF_VM_HUGEPAGE, &mm->flags))) { 482 free_mm_slot(mm_slot); 483 return; 484 } 485 486 spin_lock(&khugepaged_mm_lock); 487 insert_to_mm_slots_hash(mm, mm_slot); 488 /* 489 * Insert just behind the scanning cursor, to let the area settle 490 * down a little. 491 */ 492 wakeup = list_empty(&khugepaged_scan.mm_head); 493 list_add_tail(&mm_slot->mm_node, &khugepaged_scan.mm_head); 494 spin_unlock(&khugepaged_mm_lock); 495 496 mmgrab(mm); 497 if (wakeup) 498 wake_up_interruptible(&khugepaged_wait); 499 } 500 501 void khugepaged_enter_vma(struct vm_area_struct *vma, 502 unsigned long vm_flags) 503 { 504 if (!test_bit(MMF_VM_HUGEPAGE, &vma->vm_mm->flags) && 505 khugepaged_enabled() && 506 (((vma->vm_start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK) < 507 (vma->vm_end & HPAGE_PMD_MASK))) { 508 if (hugepage_vma_check(vma, vm_flags)) 509 __khugepaged_enter(vma->vm_mm); 510 } 511 } 512 513 void __khugepaged_exit(struct mm_struct *mm) 514 { 515 struct mm_slot *mm_slot; 516 int free = 0; 517 518 spin_lock(&khugepaged_mm_lock); 519 mm_slot = get_mm_slot(mm); 520 if (mm_slot && khugepaged_scan.mm_slot != mm_slot) { 521 hash_del(&mm_slot->hash); 522 list_del(&mm_slot->mm_node); 523 free = 1; 524 } 525 spin_unlock(&khugepaged_mm_lock); 526 527 if (free) { 528 clear_bit(MMF_VM_HUGEPAGE, &mm->flags); 529 free_mm_slot(mm_slot); 530 mmdrop(mm); 531 } else if (mm_slot) { 532 /* 533 * This is required to serialize against 534 * khugepaged_test_exit() (which is guaranteed to run 535 * under mmap sem read mode). Stop here (after we 536 * return all pagetables will be destroyed) until 537 * khugepaged has finished working on the pagetables 538 * under the mmap_lock. 539 */ 540 mmap_write_lock(mm); 541 mmap_write_unlock(mm); 542 } 543 } 544 545 static void release_pte_page(struct page *page) 546 { 547 mod_node_page_state(page_pgdat(page), 548 NR_ISOLATED_ANON + page_is_file_lru(page), 549 -compound_nr(page)); 550 unlock_page(page); 551 putback_lru_page(page); 552 } 553 554 static void release_pte_pages(pte_t *pte, pte_t *_pte, 555 struct list_head *compound_pagelist) 556 { 557 struct page *page, *tmp; 558 559 while (--_pte >= pte) { 560 pte_t pteval = *_pte; 561 562 page = pte_page(pteval); 563 if (!pte_none(pteval) && !is_zero_pfn(pte_pfn(pteval)) && 564 !PageCompound(page)) 565 release_pte_page(page); 566 } 567 568 list_for_each_entry_safe(page, tmp, compound_pagelist, lru) { 569 list_del(&page->lru); 570 release_pte_page(page); 571 } 572 } 573 574 static bool is_refcount_suitable(struct page *page) 575 { 576 int expected_refcount; 577 578 expected_refcount = total_mapcount(page); 579 if (PageSwapCache(page)) 580 expected_refcount += compound_nr(page); 581 582 return page_count(page) == expected_refcount; 583 } 584 585 static int __collapse_huge_page_isolate(struct vm_area_struct *vma, 586 unsigned long address, 587 pte_t *pte, 588 struct list_head *compound_pagelist) 589 { 590 struct page *page = NULL; 591 pte_t *_pte; 592 int none_or_zero = 0, shared = 0, result = 0, referenced = 0; 593 bool writable = false; 594 595 for (_pte = pte; _pte < pte + HPAGE_PMD_NR; 596 _pte++, address += PAGE_SIZE) { 597 pte_t pteval = *_pte; 598 if (pte_none(pteval) || (pte_present(pteval) && 599 is_zero_pfn(pte_pfn(pteval)))) { 600 if (!userfaultfd_armed(vma) && 601 ++none_or_zero <= khugepaged_max_ptes_none) { 602 continue; 603 } else { 604 result = SCAN_EXCEED_NONE_PTE; 605 count_vm_event(THP_SCAN_EXCEED_NONE_PTE); 606 goto out; 607 } 608 } 609 if (!pte_present(pteval)) { 610 result = SCAN_PTE_NON_PRESENT; 611 goto out; 612 } 613 page = vm_normal_page(vma, address, pteval); 614 if (unlikely(!page)) { 615 result = SCAN_PAGE_NULL; 616 goto out; 617 } 618 619 VM_BUG_ON_PAGE(!PageAnon(page), page); 620 621 if (page_mapcount(page) > 1 && 622 ++shared > khugepaged_max_ptes_shared) { 623 result = SCAN_EXCEED_SHARED_PTE; 624 count_vm_event(THP_SCAN_EXCEED_SHARED_PTE); 625 goto out; 626 } 627 628 if (PageCompound(page)) { 629 struct page *p; 630 page = compound_head(page); 631 632 /* 633 * Check if we have dealt with the compound page 634 * already 635 */ 636 list_for_each_entry(p, compound_pagelist, lru) { 637 if (page == p) 638 goto next; 639 } 640 } 641 642 /* 643 * We can do it before isolate_lru_page because the 644 * page can't be freed from under us. NOTE: PG_lock 645 * is needed to serialize against split_huge_page 646 * when invoked from the VM. 647 */ 648 if (!trylock_page(page)) { 649 result = SCAN_PAGE_LOCK; 650 goto out; 651 } 652 653 /* 654 * Check if the page has any GUP (or other external) pins. 655 * 656 * The page table that maps the page has been already unlinked 657 * from the page table tree and this process cannot get 658 * an additional pin on the page. 659 * 660 * New pins can come later if the page is shared across fork, 661 * but not from this process. The other process cannot write to 662 * the page, only trigger CoW. 663 */ 664 if (!is_refcount_suitable(page)) { 665 unlock_page(page); 666 result = SCAN_PAGE_COUNT; 667 goto out; 668 } 669 670 /* 671 * Isolate the page to avoid collapsing an hugepage 672 * currently in use by the VM. 673 */ 674 if (isolate_lru_page(page)) { 675 unlock_page(page); 676 result = SCAN_DEL_PAGE_LRU; 677 goto out; 678 } 679 mod_node_page_state(page_pgdat(page), 680 NR_ISOLATED_ANON + page_is_file_lru(page), 681 compound_nr(page)); 682 VM_BUG_ON_PAGE(!PageLocked(page), page); 683 VM_BUG_ON_PAGE(PageLRU(page), page); 684 685 if (PageCompound(page)) 686 list_add_tail(&page->lru, compound_pagelist); 687 next: 688 /* There should be enough young pte to collapse the page */ 689 if (pte_young(pteval) || 690 page_is_young(page) || PageReferenced(page) || 691 mmu_notifier_test_young(vma->vm_mm, address)) 692 referenced++; 693 694 if (pte_write(pteval)) 695 writable = true; 696 } 697 698 if (unlikely(!writable)) { 699 result = SCAN_PAGE_RO; 700 } else if (unlikely(!referenced)) { 701 result = SCAN_LACK_REFERENCED_PAGE; 702 } else { 703 result = SCAN_SUCCEED; 704 trace_mm_collapse_huge_page_isolate(page, none_or_zero, 705 referenced, writable, result); 706 return 1; 707 } 708 out: 709 release_pte_pages(pte, _pte, compound_pagelist); 710 trace_mm_collapse_huge_page_isolate(page, none_or_zero, 711 referenced, writable, result); 712 return 0; 713 } 714 715 static void __collapse_huge_page_copy(pte_t *pte, struct page *page, 716 struct vm_area_struct *vma, 717 unsigned long address, 718 spinlock_t *ptl, 719 struct list_head *compound_pagelist) 720 { 721 struct page *src_page, *tmp; 722 pte_t *_pte; 723 for (_pte = pte; _pte < pte + HPAGE_PMD_NR; 724 _pte++, page++, address += PAGE_SIZE) { 725 pte_t pteval = *_pte; 726 727 if (pte_none(pteval) || is_zero_pfn(pte_pfn(pteval))) { 728 clear_user_highpage(page, address); 729 add_mm_counter(vma->vm_mm, MM_ANONPAGES, 1); 730 if (is_zero_pfn(pte_pfn(pteval))) { 731 /* 732 * ptl mostly unnecessary. 733 */ 734 spin_lock(ptl); 735 ptep_clear(vma->vm_mm, address, _pte); 736 spin_unlock(ptl); 737 } 738 } else { 739 src_page = pte_page(pteval); 740 copy_user_highpage(page, src_page, address, vma); 741 if (!PageCompound(src_page)) 742 release_pte_page(src_page); 743 /* 744 * ptl mostly unnecessary, but preempt has to 745 * be disabled to update the per-cpu stats 746 * inside page_remove_rmap(). 747 */ 748 spin_lock(ptl); 749 ptep_clear(vma->vm_mm, address, _pte); 750 page_remove_rmap(src_page, vma, false); 751 spin_unlock(ptl); 752 free_page_and_swap_cache(src_page); 753 } 754 } 755 756 list_for_each_entry_safe(src_page, tmp, compound_pagelist, lru) { 757 list_del(&src_page->lru); 758 release_pte_page(src_page); 759 } 760 } 761 762 static void khugepaged_alloc_sleep(void) 763 { 764 DEFINE_WAIT(wait); 765 766 add_wait_queue(&khugepaged_wait, &wait); 767 freezable_schedule_timeout_interruptible( 768 msecs_to_jiffies(khugepaged_alloc_sleep_millisecs)); 769 remove_wait_queue(&khugepaged_wait, &wait); 770 } 771 772 static int khugepaged_node_load[MAX_NUMNODES]; 773 774 static bool khugepaged_scan_abort(int nid) 775 { 776 int i; 777 778 /* 779 * If node_reclaim_mode is disabled, then no extra effort is made to 780 * allocate memory locally. 781 */ 782 if (!node_reclaim_enabled()) 783 return false; 784 785 /* If there is a count for this node already, it must be acceptable */ 786 if (khugepaged_node_load[nid]) 787 return false; 788 789 for (i = 0; i < MAX_NUMNODES; i++) { 790 if (!khugepaged_node_load[i]) 791 continue; 792 if (node_distance(nid, i) > node_reclaim_distance) 793 return true; 794 } 795 return false; 796 } 797 798 /* Defrag for khugepaged will enter direct reclaim/compaction if necessary */ 799 static inline gfp_t alloc_hugepage_khugepaged_gfpmask(void) 800 { 801 return khugepaged_defrag() ? GFP_TRANSHUGE : GFP_TRANSHUGE_LIGHT; 802 } 803 804 #ifdef CONFIG_NUMA 805 static int khugepaged_find_target_node(void) 806 { 807 static int last_khugepaged_target_node = NUMA_NO_NODE; 808 int nid, target_node = 0, max_value = 0; 809 810 /* find first node with max normal pages hit */ 811 for (nid = 0; nid < MAX_NUMNODES; nid++) 812 if (khugepaged_node_load[nid] > max_value) { 813 max_value = khugepaged_node_load[nid]; 814 target_node = nid; 815 } 816 817 /* do some balance if several nodes have the same hit record */ 818 if (target_node <= last_khugepaged_target_node) 819 for (nid = last_khugepaged_target_node + 1; nid < MAX_NUMNODES; 820 nid++) 821 if (max_value == khugepaged_node_load[nid]) { 822 target_node = nid; 823 break; 824 } 825 826 last_khugepaged_target_node = target_node; 827 return target_node; 828 } 829 830 static bool khugepaged_prealloc_page(struct page **hpage, bool *wait) 831 { 832 if (IS_ERR(*hpage)) { 833 if (!*wait) 834 return false; 835 836 *wait = false; 837 *hpage = NULL; 838 khugepaged_alloc_sleep(); 839 } else if (*hpage) { 840 put_page(*hpage); 841 *hpage = NULL; 842 } 843 844 return true; 845 } 846 847 static struct page * 848 khugepaged_alloc_page(struct page **hpage, gfp_t gfp, int node) 849 { 850 VM_BUG_ON_PAGE(*hpage, *hpage); 851 852 *hpage = __alloc_pages_node(node, gfp, HPAGE_PMD_ORDER); 853 if (unlikely(!*hpage)) { 854 count_vm_event(THP_COLLAPSE_ALLOC_FAILED); 855 *hpage = ERR_PTR(-ENOMEM); 856 return NULL; 857 } 858 859 prep_transhuge_page(*hpage); 860 count_vm_event(THP_COLLAPSE_ALLOC); 861 return *hpage; 862 } 863 #else 864 static int khugepaged_find_target_node(void) 865 { 866 return 0; 867 } 868 869 static inline struct page *alloc_khugepaged_hugepage(void) 870 { 871 struct page *page; 872 873 page = alloc_pages(alloc_hugepage_khugepaged_gfpmask(), 874 HPAGE_PMD_ORDER); 875 if (page) 876 prep_transhuge_page(page); 877 return page; 878 } 879 880 static struct page *khugepaged_alloc_hugepage(bool *wait) 881 { 882 struct page *hpage; 883 884 do { 885 hpage = alloc_khugepaged_hugepage(); 886 if (!hpage) { 887 count_vm_event(THP_COLLAPSE_ALLOC_FAILED); 888 if (!*wait) 889 return NULL; 890 891 *wait = false; 892 khugepaged_alloc_sleep(); 893 } else 894 count_vm_event(THP_COLLAPSE_ALLOC); 895 } while (unlikely(!hpage) && likely(khugepaged_enabled())); 896 897 return hpage; 898 } 899 900 static bool khugepaged_prealloc_page(struct page **hpage, bool *wait) 901 { 902 /* 903 * If the hpage allocated earlier was briefly exposed in page cache 904 * before collapse_file() failed, it is possible that racing lookups 905 * have not yet completed, and would then be unpleasantly surprised by 906 * finding the hpage reused for the same mapping at a different offset. 907 * Just release the previous allocation if there is any danger of that. 908 */ 909 if (*hpage && page_count(*hpage) > 1) { 910 put_page(*hpage); 911 *hpage = NULL; 912 } 913 914 if (!*hpage) 915 *hpage = khugepaged_alloc_hugepage(wait); 916 917 if (unlikely(!*hpage)) 918 return false; 919 920 return true; 921 } 922 923 static struct page * 924 khugepaged_alloc_page(struct page **hpage, gfp_t gfp, int node) 925 { 926 VM_BUG_ON(!*hpage); 927 928 return *hpage; 929 } 930 #endif 931 932 /* 933 * If mmap_lock temporarily dropped, revalidate vma 934 * before taking mmap_lock. 935 * Return 0 if succeeds, otherwise return none-zero 936 * value (scan code). 937 */ 938 939 static int hugepage_vma_revalidate(struct mm_struct *mm, unsigned long address, 940 struct vm_area_struct **vmap) 941 { 942 struct vm_area_struct *vma; 943 unsigned long hstart, hend; 944 945 if (unlikely(khugepaged_test_exit(mm))) 946 return SCAN_ANY_PROCESS; 947 948 *vmap = vma = find_vma(mm, address); 949 if (!vma) 950 return SCAN_VMA_NULL; 951 952 hstart = (vma->vm_start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK; 953 hend = vma->vm_end & HPAGE_PMD_MASK; 954 if (address < hstart || address + HPAGE_PMD_SIZE > hend) 955 return SCAN_ADDRESS_RANGE; 956 if (!hugepage_vma_check(vma, vma->vm_flags)) 957 return SCAN_VMA_CHECK; 958 /* Anon VMA expected */ 959 if (!vma->anon_vma || !vma_is_anonymous(vma)) 960 return SCAN_VMA_CHECK; 961 return 0; 962 } 963 964 /* 965 * Bring missing pages in from swap, to complete THP collapse. 966 * Only done if khugepaged_scan_pmd believes it is worthwhile. 967 * 968 * Called and returns without pte mapped or spinlocks held. 969 * Note that if false is returned, mmap_lock will be released. 970 */ 971 972 static bool __collapse_huge_page_swapin(struct mm_struct *mm, 973 struct vm_area_struct *vma, 974 unsigned long haddr, pmd_t *pmd, 975 int referenced) 976 { 977 int swapped_in = 0; 978 vm_fault_t ret = 0; 979 unsigned long address, end = haddr + (HPAGE_PMD_NR * PAGE_SIZE); 980 981 for (address = haddr; address < end; address += PAGE_SIZE) { 982 struct vm_fault vmf = { 983 .vma = vma, 984 .address = address, 985 .pgoff = linear_page_index(vma, haddr), 986 .flags = FAULT_FLAG_ALLOW_RETRY, 987 .pmd = pmd, 988 }; 989 990 vmf.pte = pte_offset_map(pmd, address); 991 vmf.orig_pte = *vmf.pte; 992 if (!is_swap_pte(vmf.orig_pte)) { 993 pte_unmap(vmf.pte); 994 continue; 995 } 996 ret = do_swap_page(&vmf); 997 998 /* 999 * do_swap_page returns VM_FAULT_RETRY with released mmap_lock. 1000 * Note we treat VM_FAULT_RETRY as VM_FAULT_ERROR here because 1001 * we do not retry here and swap entry will remain in pagetable 1002 * resulting in later failure. 1003 */ 1004 if (ret & VM_FAULT_RETRY) { 1005 trace_mm_collapse_huge_page_swapin(mm, swapped_in, referenced, 0); 1006 return false; 1007 } 1008 if (ret & VM_FAULT_ERROR) { 1009 mmap_read_unlock(mm); 1010 trace_mm_collapse_huge_page_swapin(mm, swapped_in, referenced, 0); 1011 return false; 1012 } 1013 swapped_in++; 1014 } 1015 1016 /* Drain LRU add pagevec to remove extra pin on the swapped in pages */ 1017 if (swapped_in) 1018 lru_add_drain(); 1019 1020 trace_mm_collapse_huge_page_swapin(mm, swapped_in, referenced, 1); 1021 return true; 1022 } 1023 1024 static void collapse_huge_page(struct mm_struct *mm, 1025 unsigned long address, 1026 struct page **hpage, 1027 int node, int referenced, int unmapped) 1028 { 1029 LIST_HEAD(compound_pagelist); 1030 pmd_t *pmd, _pmd; 1031 pte_t *pte; 1032 pgtable_t pgtable; 1033 struct page *new_page; 1034 spinlock_t *pmd_ptl, *pte_ptl; 1035 int isolated = 0, result = 0; 1036 struct vm_area_struct *vma; 1037 struct mmu_notifier_range range; 1038 gfp_t gfp; 1039 1040 VM_BUG_ON(address & ~HPAGE_PMD_MASK); 1041 1042 /* Only allocate from the target node */ 1043 gfp = alloc_hugepage_khugepaged_gfpmask() | __GFP_THISNODE; 1044 1045 /* 1046 * Before allocating the hugepage, release the mmap_lock read lock. 1047 * The allocation can take potentially a long time if it involves 1048 * sync compaction, and we do not need to hold the mmap_lock during 1049 * that. We will recheck the vma after taking it again in write mode. 1050 */ 1051 mmap_read_unlock(mm); 1052 new_page = khugepaged_alloc_page(hpage, gfp, node); 1053 if (!new_page) { 1054 result = SCAN_ALLOC_HUGE_PAGE_FAIL; 1055 goto out_nolock; 1056 } 1057 1058 if (unlikely(mem_cgroup_charge(page_folio(new_page), mm, gfp))) { 1059 result = SCAN_CGROUP_CHARGE_FAIL; 1060 goto out_nolock; 1061 } 1062 count_memcg_page_event(new_page, THP_COLLAPSE_ALLOC); 1063 1064 mmap_read_lock(mm); 1065 result = hugepage_vma_revalidate(mm, address, &vma); 1066 if (result) { 1067 mmap_read_unlock(mm); 1068 goto out_nolock; 1069 } 1070 1071 pmd = mm_find_pmd(mm, address); 1072 if (!pmd) { 1073 result = SCAN_PMD_NULL; 1074 mmap_read_unlock(mm); 1075 goto out_nolock; 1076 } 1077 1078 /* 1079 * __collapse_huge_page_swapin will return with mmap_lock released 1080 * when it fails. So we jump out_nolock directly in that case. 1081 * Continuing to collapse causes inconsistency. 1082 */ 1083 if (unmapped && !__collapse_huge_page_swapin(mm, vma, address, 1084 pmd, referenced)) { 1085 goto out_nolock; 1086 } 1087 1088 mmap_read_unlock(mm); 1089 /* 1090 * Prevent all access to pagetables with the exception of 1091 * gup_fast later handled by the ptep_clear_flush and the VM 1092 * handled by the anon_vma lock + PG_lock. 1093 */ 1094 mmap_write_lock(mm); 1095 result = hugepage_vma_revalidate(mm, address, &vma); 1096 if (result) 1097 goto out_up_write; 1098 /* check if the pmd is still valid */ 1099 if (mm_find_pmd(mm, address) != pmd) 1100 goto out_up_write; 1101 1102 anon_vma_lock_write(vma->anon_vma); 1103 1104 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, NULL, mm, 1105 address, address + HPAGE_PMD_SIZE); 1106 mmu_notifier_invalidate_range_start(&range); 1107 1108 pte = pte_offset_map(pmd, address); 1109 pte_ptl = pte_lockptr(mm, pmd); 1110 1111 pmd_ptl = pmd_lock(mm, pmd); /* probably unnecessary */ 1112 /* 1113 * After this gup_fast can't run anymore. This also removes 1114 * any huge TLB entry from the CPU so we won't allow 1115 * huge and small TLB entries for the same virtual address 1116 * to avoid the risk of CPU bugs in that area. 1117 */ 1118 _pmd = pmdp_collapse_flush(vma, address, pmd); 1119 spin_unlock(pmd_ptl); 1120 mmu_notifier_invalidate_range_end(&range); 1121 1122 spin_lock(pte_ptl); 1123 isolated = __collapse_huge_page_isolate(vma, address, pte, 1124 &compound_pagelist); 1125 spin_unlock(pte_ptl); 1126 1127 if (unlikely(!isolated)) { 1128 pte_unmap(pte); 1129 spin_lock(pmd_ptl); 1130 BUG_ON(!pmd_none(*pmd)); 1131 /* 1132 * We can only use set_pmd_at when establishing 1133 * hugepmds and never for establishing regular pmds that 1134 * points to regular pagetables. Use pmd_populate for that 1135 */ 1136 pmd_populate(mm, pmd, pmd_pgtable(_pmd)); 1137 spin_unlock(pmd_ptl); 1138 anon_vma_unlock_write(vma->anon_vma); 1139 result = SCAN_FAIL; 1140 goto out_up_write; 1141 } 1142 1143 /* 1144 * All pages are isolated and locked so anon_vma rmap 1145 * can't run anymore. 1146 */ 1147 anon_vma_unlock_write(vma->anon_vma); 1148 1149 __collapse_huge_page_copy(pte, new_page, vma, address, pte_ptl, 1150 &compound_pagelist); 1151 pte_unmap(pte); 1152 /* 1153 * spin_lock() below is not the equivalent of smp_wmb(), but 1154 * the smp_wmb() inside __SetPageUptodate() can be reused to 1155 * avoid the copy_huge_page writes to become visible after 1156 * the set_pmd_at() write. 1157 */ 1158 __SetPageUptodate(new_page); 1159 pgtable = pmd_pgtable(_pmd); 1160 1161 _pmd = mk_huge_pmd(new_page, vma->vm_page_prot); 1162 _pmd = maybe_pmd_mkwrite(pmd_mkdirty(_pmd), vma); 1163 1164 spin_lock(pmd_ptl); 1165 BUG_ON(!pmd_none(*pmd)); 1166 page_add_new_anon_rmap(new_page, vma, address); 1167 lru_cache_add_inactive_or_unevictable(new_page, vma); 1168 pgtable_trans_huge_deposit(mm, pmd, pgtable); 1169 set_pmd_at(mm, address, pmd, _pmd); 1170 update_mmu_cache_pmd(vma, address, pmd); 1171 spin_unlock(pmd_ptl); 1172 1173 *hpage = NULL; 1174 1175 khugepaged_pages_collapsed++; 1176 result = SCAN_SUCCEED; 1177 out_up_write: 1178 mmap_write_unlock(mm); 1179 out_nolock: 1180 if (!IS_ERR_OR_NULL(*hpage)) 1181 mem_cgroup_uncharge(page_folio(*hpage)); 1182 trace_mm_collapse_huge_page(mm, isolated, result); 1183 return; 1184 } 1185 1186 static int khugepaged_scan_pmd(struct mm_struct *mm, 1187 struct vm_area_struct *vma, 1188 unsigned long address, 1189 struct page **hpage) 1190 { 1191 pmd_t *pmd; 1192 pte_t *pte, *_pte; 1193 int ret = 0, result = 0, referenced = 0; 1194 int none_or_zero = 0, shared = 0; 1195 struct page *page = NULL; 1196 unsigned long _address; 1197 spinlock_t *ptl; 1198 int node = NUMA_NO_NODE, unmapped = 0; 1199 bool writable = false; 1200 1201 VM_BUG_ON(address & ~HPAGE_PMD_MASK); 1202 1203 pmd = mm_find_pmd(mm, address); 1204 if (!pmd) { 1205 result = SCAN_PMD_NULL; 1206 goto out; 1207 } 1208 1209 memset(khugepaged_node_load, 0, sizeof(khugepaged_node_load)); 1210 pte = pte_offset_map_lock(mm, pmd, address, &ptl); 1211 for (_address = address, _pte = pte; _pte < pte + HPAGE_PMD_NR; 1212 _pte++, _address += PAGE_SIZE) { 1213 pte_t pteval = *_pte; 1214 if (is_swap_pte(pteval)) { 1215 if (++unmapped <= khugepaged_max_ptes_swap) { 1216 /* 1217 * Always be strict with uffd-wp 1218 * enabled swap entries. Please see 1219 * comment below for pte_uffd_wp(). 1220 */ 1221 if (pte_swp_uffd_wp(pteval)) { 1222 result = SCAN_PTE_UFFD_WP; 1223 goto out_unmap; 1224 } 1225 continue; 1226 } else { 1227 result = SCAN_EXCEED_SWAP_PTE; 1228 count_vm_event(THP_SCAN_EXCEED_SWAP_PTE); 1229 goto out_unmap; 1230 } 1231 } 1232 if (pte_none(pteval) || is_zero_pfn(pte_pfn(pteval))) { 1233 if (!userfaultfd_armed(vma) && 1234 ++none_or_zero <= khugepaged_max_ptes_none) { 1235 continue; 1236 } else { 1237 result = SCAN_EXCEED_NONE_PTE; 1238 count_vm_event(THP_SCAN_EXCEED_NONE_PTE); 1239 goto out_unmap; 1240 } 1241 } 1242 if (pte_uffd_wp(pteval)) { 1243 /* 1244 * Don't collapse the page if any of the small 1245 * PTEs are armed with uffd write protection. 1246 * Here we can also mark the new huge pmd as 1247 * write protected if any of the small ones is 1248 * marked but that could bring unknown 1249 * userfault messages that falls outside of 1250 * the registered range. So, just be simple. 1251 */ 1252 result = SCAN_PTE_UFFD_WP; 1253 goto out_unmap; 1254 } 1255 if (pte_write(pteval)) 1256 writable = true; 1257 1258 page = vm_normal_page(vma, _address, pteval); 1259 if (unlikely(!page)) { 1260 result = SCAN_PAGE_NULL; 1261 goto out_unmap; 1262 } 1263 1264 if (page_mapcount(page) > 1 && 1265 ++shared > khugepaged_max_ptes_shared) { 1266 result = SCAN_EXCEED_SHARED_PTE; 1267 count_vm_event(THP_SCAN_EXCEED_SHARED_PTE); 1268 goto out_unmap; 1269 } 1270 1271 page = compound_head(page); 1272 1273 /* 1274 * Record which node the original page is from and save this 1275 * information to khugepaged_node_load[]. 1276 * Khugepaged will allocate hugepage from the node has the max 1277 * hit record. 1278 */ 1279 node = page_to_nid(page); 1280 if (khugepaged_scan_abort(node)) { 1281 result = SCAN_SCAN_ABORT; 1282 goto out_unmap; 1283 } 1284 khugepaged_node_load[node]++; 1285 if (!PageLRU(page)) { 1286 result = SCAN_PAGE_LRU; 1287 goto out_unmap; 1288 } 1289 if (PageLocked(page)) { 1290 result = SCAN_PAGE_LOCK; 1291 goto out_unmap; 1292 } 1293 if (!PageAnon(page)) { 1294 result = SCAN_PAGE_ANON; 1295 goto out_unmap; 1296 } 1297 1298 /* 1299 * Check if the page has any GUP (or other external) pins. 1300 * 1301 * Here the check is racy it may see total_mapcount > refcount 1302 * in some cases. 1303 * For example, one process with one forked child process. 1304 * The parent has the PMD split due to MADV_DONTNEED, then 1305 * the child is trying unmap the whole PMD, but khugepaged 1306 * may be scanning the parent between the child has 1307 * PageDoubleMap flag cleared and dec the mapcount. So 1308 * khugepaged may see total_mapcount > refcount. 1309 * 1310 * But such case is ephemeral we could always retry collapse 1311 * later. However it may report false positive if the page 1312 * has excessive GUP pins (i.e. 512). Anyway the same check 1313 * will be done again later the risk seems low. 1314 */ 1315 if (!is_refcount_suitable(page)) { 1316 result = SCAN_PAGE_COUNT; 1317 goto out_unmap; 1318 } 1319 if (pte_young(pteval) || 1320 page_is_young(page) || PageReferenced(page) || 1321 mmu_notifier_test_young(vma->vm_mm, address)) 1322 referenced++; 1323 } 1324 if (!writable) { 1325 result = SCAN_PAGE_RO; 1326 } else if (!referenced || (unmapped && referenced < HPAGE_PMD_NR/2)) { 1327 result = SCAN_LACK_REFERENCED_PAGE; 1328 } else { 1329 result = SCAN_SUCCEED; 1330 ret = 1; 1331 } 1332 out_unmap: 1333 pte_unmap_unlock(pte, ptl); 1334 if (ret) { 1335 node = khugepaged_find_target_node(); 1336 /* collapse_huge_page will return with the mmap_lock released */ 1337 collapse_huge_page(mm, address, hpage, node, 1338 referenced, unmapped); 1339 } 1340 out: 1341 trace_mm_khugepaged_scan_pmd(mm, page, writable, referenced, 1342 none_or_zero, result, unmapped); 1343 return ret; 1344 } 1345 1346 static void collect_mm_slot(struct mm_slot *mm_slot) 1347 { 1348 struct mm_struct *mm = mm_slot->mm; 1349 1350 lockdep_assert_held(&khugepaged_mm_lock); 1351 1352 if (khugepaged_test_exit(mm)) { 1353 /* free mm_slot */ 1354 hash_del(&mm_slot->hash); 1355 list_del(&mm_slot->mm_node); 1356 1357 /* 1358 * Not strictly needed because the mm exited already. 1359 * 1360 * clear_bit(MMF_VM_HUGEPAGE, &mm->flags); 1361 */ 1362 1363 /* khugepaged_mm_lock actually not necessary for the below */ 1364 free_mm_slot(mm_slot); 1365 mmdrop(mm); 1366 } 1367 } 1368 1369 #ifdef CONFIG_SHMEM 1370 /* 1371 * Notify khugepaged that given addr of the mm is pte-mapped THP. Then 1372 * khugepaged should try to collapse the page table. 1373 */ 1374 static void khugepaged_add_pte_mapped_thp(struct mm_struct *mm, 1375 unsigned long addr) 1376 { 1377 struct mm_slot *mm_slot; 1378 1379 VM_BUG_ON(addr & ~HPAGE_PMD_MASK); 1380 1381 spin_lock(&khugepaged_mm_lock); 1382 mm_slot = get_mm_slot(mm); 1383 if (likely(mm_slot && mm_slot->nr_pte_mapped_thp < MAX_PTE_MAPPED_THP)) 1384 mm_slot->pte_mapped_thp[mm_slot->nr_pte_mapped_thp++] = addr; 1385 spin_unlock(&khugepaged_mm_lock); 1386 } 1387 1388 static void collapse_and_free_pmd(struct mm_struct *mm, struct vm_area_struct *vma, 1389 unsigned long addr, pmd_t *pmdp) 1390 { 1391 spinlock_t *ptl; 1392 pmd_t pmd; 1393 1394 mmap_assert_write_locked(mm); 1395 ptl = pmd_lock(vma->vm_mm, pmdp); 1396 pmd = pmdp_collapse_flush(vma, addr, pmdp); 1397 spin_unlock(ptl); 1398 mm_dec_nr_ptes(mm); 1399 page_table_check_pte_clear_range(mm, addr, pmd); 1400 pte_free(mm, pmd_pgtable(pmd)); 1401 } 1402 1403 /** 1404 * collapse_pte_mapped_thp - Try to collapse a pte-mapped THP for mm at 1405 * address haddr. 1406 * 1407 * @mm: process address space where collapse happens 1408 * @addr: THP collapse address 1409 * 1410 * This function checks whether all the PTEs in the PMD are pointing to the 1411 * right THP. If so, retract the page table so the THP can refault in with 1412 * as pmd-mapped. 1413 */ 1414 void collapse_pte_mapped_thp(struct mm_struct *mm, unsigned long addr) 1415 { 1416 unsigned long haddr = addr & HPAGE_PMD_MASK; 1417 struct vm_area_struct *vma = find_vma(mm, haddr); 1418 struct page *hpage; 1419 pte_t *start_pte, *pte; 1420 pmd_t *pmd; 1421 spinlock_t *ptl; 1422 int count = 0; 1423 int i; 1424 1425 if (!vma || !vma->vm_file || 1426 !range_in_vma(vma, haddr, haddr + HPAGE_PMD_SIZE)) 1427 return; 1428 1429 /* 1430 * This vm_flags may not have VM_HUGEPAGE if the page was not 1431 * collapsed by this mm. But we can still collapse if the page is 1432 * the valid THP. Add extra VM_HUGEPAGE so hugepage_vma_check() 1433 * will not fail the vma for missing VM_HUGEPAGE 1434 */ 1435 if (!hugepage_vma_check(vma, vma->vm_flags | VM_HUGEPAGE)) 1436 return; 1437 1438 /* Keep pmd pgtable for uffd-wp; see comment in retract_page_tables() */ 1439 if (userfaultfd_wp(vma)) 1440 return; 1441 1442 hpage = find_lock_page(vma->vm_file->f_mapping, 1443 linear_page_index(vma, haddr)); 1444 if (!hpage) 1445 return; 1446 1447 if (!PageHead(hpage)) 1448 goto drop_hpage; 1449 1450 pmd = mm_find_pmd(mm, haddr); 1451 if (!pmd) 1452 goto drop_hpage; 1453 1454 start_pte = pte_offset_map_lock(mm, pmd, haddr, &ptl); 1455 1456 /* step 1: check all mapped PTEs are to the right huge page */ 1457 for (i = 0, addr = haddr, pte = start_pte; 1458 i < HPAGE_PMD_NR; i++, addr += PAGE_SIZE, pte++) { 1459 struct page *page; 1460 1461 /* empty pte, skip */ 1462 if (pte_none(*pte)) 1463 continue; 1464 1465 /* page swapped out, abort */ 1466 if (!pte_present(*pte)) 1467 goto abort; 1468 1469 page = vm_normal_page(vma, addr, *pte); 1470 1471 /* 1472 * Note that uprobe, debugger, or MAP_PRIVATE may change the 1473 * page table, but the new page will not be a subpage of hpage. 1474 */ 1475 if (hpage + i != page) 1476 goto abort; 1477 count++; 1478 } 1479 1480 /* step 2: adjust rmap */ 1481 for (i = 0, addr = haddr, pte = start_pte; 1482 i < HPAGE_PMD_NR; i++, addr += PAGE_SIZE, pte++) { 1483 struct page *page; 1484 1485 if (pte_none(*pte)) 1486 continue; 1487 page = vm_normal_page(vma, addr, *pte); 1488 page_remove_rmap(page, vma, false); 1489 } 1490 1491 pte_unmap_unlock(start_pte, ptl); 1492 1493 /* step 3: set proper refcount and mm_counters. */ 1494 if (count) { 1495 page_ref_sub(hpage, count); 1496 add_mm_counter(vma->vm_mm, mm_counter_file(hpage), -count); 1497 } 1498 1499 /* step 4: collapse pmd */ 1500 collapse_and_free_pmd(mm, vma, haddr, pmd); 1501 drop_hpage: 1502 unlock_page(hpage); 1503 put_page(hpage); 1504 return; 1505 1506 abort: 1507 pte_unmap_unlock(start_pte, ptl); 1508 goto drop_hpage; 1509 } 1510 1511 static void khugepaged_collapse_pte_mapped_thps(struct mm_slot *mm_slot) 1512 { 1513 struct mm_struct *mm = mm_slot->mm; 1514 int i; 1515 1516 if (likely(mm_slot->nr_pte_mapped_thp == 0)) 1517 return; 1518 1519 if (!mmap_write_trylock(mm)) 1520 return; 1521 1522 if (unlikely(khugepaged_test_exit(mm))) 1523 goto out; 1524 1525 for (i = 0; i < mm_slot->nr_pte_mapped_thp; i++) 1526 collapse_pte_mapped_thp(mm, mm_slot->pte_mapped_thp[i]); 1527 1528 out: 1529 mm_slot->nr_pte_mapped_thp = 0; 1530 mmap_write_unlock(mm); 1531 } 1532 1533 static void retract_page_tables(struct address_space *mapping, pgoff_t pgoff) 1534 { 1535 struct vm_area_struct *vma; 1536 struct mm_struct *mm; 1537 unsigned long addr; 1538 pmd_t *pmd; 1539 1540 i_mmap_lock_write(mapping); 1541 vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff, pgoff) { 1542 /* 1543 * Check vma->anon_vma to exclude MAP_PRIVATE mappings that 1544 * got written to. These VMAs are likely not worth investing 1545 * mmap_write_lock(mm) as PMD-mapping is likely to be split 1546 * later. 1547 * 1548 * Note that vma->anon_vma check is racy: it can be set up after 1549 * the check but before we took mmap_lock by the fault path. 1550 * But page lock would prevent establishing any new ptes of the 1551 * page, so we are safe. 1552 * 1553 * An alternative would be drop the check, but check that page 1554 * table is clear before calling pmdp_collapse_flush() under 1555 * ptl. It has higher chance to recover THP for the VMA, but 1556 * has higher cost too. 1557 */ 1558 if (vma->anon_vma) 1559 continue; 1560 addr = vma->vm_start + ((pgoff - vma->vm_pgoff) << PAGE_SHIFT); 1561 if (addr & ~HPAGE_PMD_MASK) 1562 continue; 1563 if (vma->vm_end < addr + HPAGE_PMD_SIZE) 1564 continue; 1565 mm = vma->vm_mm; 1566 pmd = mm_find_pmd(mm, addr); 1567 if (!pmd) 1568 continue; 1569 /* 1570 * We need exclusive mmap_lock to retract page table. 1571 * 1572 * We use trylock due to lock inversion: we need to acquire 1573 * mmap_lock while holding page lock. Fault path does it in 1574 * reverse order. Trylock is a way to avoid deadlock. 1575 */ 1576 if (mmap_write_trylock(mm)) { 1577 /* 1578 * When a vma is registered with uffd-wp, we can't 1579 * recycle the pmd pgtable because there can be pte 1580 * markers installed. Skip it only, so the rest mm/vma 1581 * can still have the same file mapped hugely, however 1582 * it'll always mapped in small page size for uffd-wp 1583 * registered ranges. 1584 */ 1585 if (!khugepaged_test_exit(mm) && !userfaultfd_wp(vma)) 1586 collapse_and_free_pmd(mm, vma, addr, pmd); 1587 mmap_write_unlock(mm); 1588 } else { 1589 /* Try again later */ 1590 khugepaged_add_pte_mapped_thp(mm, addr); 1591 } 1592 } 1593 i_mmap_unlock_write(mapping); 1594 } 1595 1596 /** 1597 * collapse_file - collapse filemap/tmpfs/shmem pages into huge one. 1598 * 1599 * @mm: process address space where collapse happens 1600 * @file: file that collapse on 1601 * @start: collapse start address 1602 * @hpage: new allocated huge page for collapse 1603 * @node: appointed node the new huge page allocate from 1604 * 1605 * Basic scheme is simple, details are more complex: 1606 * - allocate and lock a new huge page; 1607 * - scan page cache replacing old pages with the new one 1608 * + swap/gup in pages if necessary; 1609 * + fill in gaps; 1610 * + keep old pages around in case rollback is required; 1611 * - if replacing succeeds: 1612 * + copy data over; 1613 * + free old pages; 1614 * + unlock huge page; 1615 * - if replacing failed; 1616 * + put all pages back and unfreeze them; 1617 * + restore gaps in the page cache; 1618 * + unlock and free huge page; 1619 */ 1620 static void collapse_file(struct mm_struct *mm, 1621 struct file *file, pgoff_t start, 1622 struct page **hpage, int node) 1623 { 1624 struct address_space *mapping = file->f_mapping; 1625 gfp_t gfp; 1626 struct page *new_page; 1627 pgoff_t index, end = start + HPAGE_PMD_NR; 1628 LIST_HEAD(pagelist); 1629 XA_STATE_ORDER(xas, &mapping->i_pages, start, HPAGE_PMD_ORDER); 1630 int nr_none = 0, result = SCAN_SUCCEED; 1631 bool is_shmem = shmem_file(file); 1632 int nr; 1633 1634 VM_BUG_ON(!IS_ENABLED(CONFIG_READ_ONLY_THP_FOR_FS) && !is_shmem); 1635 VM_BUG_ON(start & (HPAGE_PMD_NR - 1)); 1636 1637 /* Only allocate from the target node */ 1638 gfp = alloc_hugepage_khugepaged_gfpmask() | __GFP_THISNODE; 1639 1640 new_page = khugepaged_alloc_page(hpage, gfp, node); 1641 if (!new_page) { 1642 result = SCAN_ALLOC_HUGE_PAGE_FAIL; 1643 goto out; 1644 } 1645 1646 if (unlikely(mem_cgroup_charge(page_folio(new_page), mm, gfp))) { 1647 result = SCAN_CGROUP_CHARGE_FAIL; 1648 goto out; 1649 } 1650 count_memcg_page_event(new_page, THP_COLLAPSE_ALLOC); 1651 1652 /* 1653 * Ensure we have slots for all the pages in the range. This is 1654 * almost certainly a no-op because most of the pages must be present 1655 */ 1656 do { 1657 xas_lock_irq(&xas); 1658 xas_create_range(&xas); 1659 if (!xas_error(&xas)) 1660 break; 1661 xas_unlock_irq(&xas); 1662 if (!xas_nomem(&xas, GFP_KERNEL)) { 1663 result = SCAN_FAIL; 1664 goto out; 1665 } 1666 } while (1); 1667 1668 __SetPageLocked(new_page); 1669 if (is_shmem) 1670 __SetPageSwapBacked(new_page); 1671 new_page->index = start; 1672 new_page->mapping = mapping; 1673 1674 /* 1675 * At this point the new_page is locked and not up-to-date. 1676 * It's safe to insert it into the page cache, because nobody would 1677 * be able to map it or use it in another way until we unlock it. 1678 */ 1679 1680 xas_set(&xas, start); 1681 for (index = start; index < end; index++) { 1682 struct page *page = xas_next(&xas); 1683 1684 VM_BUG_ON(index != xas.xa_index); 1685 if (is_shmem) { 1686 if (!page) { 1687 /* 1688 * Stop if extent has been truncated or 1689 * hole-punched, and is now completely 1690 * empty. 1691 */ 1692 if (index == start) { 1693 if (!xas_next_entry(&xas, end - 1)) { 1694 result = SCAN_TRUNCATED; 1695 goto xa_locked; 1696 } 1697 xas_set(&xas, index); 1698 } 1699 if (!shmem_charge(mapping->host, 1)) { 1700 result = SCAN_FAIL; 1701 goto xa_locked; 1702 } 1703 xas_store(&xas, new_page); 1704 nr_none++; 1705 continue; 1706 } 1707 1708 if (xa_is_value(page) || !PageUptodate(page)) { 1709 xas_unlock_irq(&xas); 1710 /* swap in or instantiate fallocated page */ 1711 if (shmem_getpage(mapping->host, index, &page, 1712 SGP_NOALLOC)) { 1713 result = SCAN_FAIL; 1714 goto xa_unlocked; 1715 } 1716 } else if (trylock_page(page)) { 1717 get_page(page); 1718 xas_unlock_irq(&xas); 1719 } else { 1720 result = SCAN_PAGE_LOCK; 1721 goto xa_locked; 1722 } 1723 } else { /* !is_shmem */ 1724 if (!page || xa_is_value(page)) { 1725 xas_unlock_irq(&xas); 1726 page_cache_sync_readahead(mapping, &file->f_ra, 1727 file, index, 1728 end - index); 1729 /* drain pagevecs to help isolate_lru_page() */ 1730 lru_add_drain(); 1731 page = find_lock_page(mapping, index); 1732 if (unlikely(page == NULL)) { 1733 result = SCAN_FAIL; 1734 goto xa_unlocked; 1735 } 1736 } else if (PageDirty(page)) { 1737 /* 1738 * khugepaged only works on read-only fd, 1739 * so this page is dirty because it hasn't 1740 * been flushed since first write. There 1741 * won't be new dirty pages. 1742 * 1743 * Trigger async flush here and hope the 1744 * writeback is done when khugepaged 1745 * revisits this page. 1746 * 1747 * This is a one-off situation. We are not 1748 * forcing writeback in loop. 1749 */ 1750 xas_unlock_irq(&xas); 1751 filemap_flush(mapping); 1752 result = SCAN_FAIL; 1753 goto xa_unlocked; 1754 } else if (PageWriteback(page)) { 1755 xas_unlock_irq(&xas); 1756 result = SCAN_FAIL; 1757 goto xa_unlocked; 1758 } else if (trylock_page(page)) { 1759 get_page(page); 1760 xas_unlock_irq(&xas); 1761 } else { 1762 result = SCAN_PAGE_LOCK; 1763 goto xa_locked; 1764 } 1765 } 1766 1767 /* 1768 * The page must be locked, so we can drop the i_pages lock 1769 * without racing with truncate. 1770 */ 1771 VM_BUG_ON_PAGE(!PageLocked(page), page); 1772 1773 /* make sure the page is up to date */ 1774 if (unlikely(!PageUptodate(page))) { 1775 result = SCAN_FAIL; 1776 goto out_unlock; 1777 } 1778 1779 /* 1780 * If file was truncated then extended, or hole-punched, before 1781 * we locked the first page, then a THP might be there already. 1782 */ 1783 if (PageTransCompound(page)) { 1784 result = SCAN_PAGE_COMPOUND; 1785 goto out_unlock; 1786 } 1787 1788 if (page_mapping(page) != mapping) { 1789 result = SCAN_TRUNCATED; 1790 goto out_unlock; 1791 } 1792 1793 if (!is_shmem && (PageDirty(page) || 1794 PageWriteback(page))) { 1795 /* 1796 * khugepaged only works on read-only fd, so this 1797 * page is dirty because it hasn't been flushed 1798 * since first write. 1799 */ 1800 result = SCAN_FAIL; 1801 goto out_unlock; 1802 } 1803 1804 if (isolate_lru_page(page)) { 1805 result = SCAN_DEL_PAGE_LRU; 1806 goto out_unlock; 1807 } 1808 1809 if (page_has_private(page) && 1810 !try_to_release_page(page, GFP_KERNEL)) { 1811 result = SCAN_PAGE_HAS_PRIVATE; 1812 putback_lru_page(page); 1813 goto out_unlock; 1814 } 1815 1816 if (page_mapped(page)) 1817 try_to_unmap(page_folio(page), 1818 TTU_IGNORE_MLOCK | TTU_BATCH_FLUSH); 1819 1820 xas_lock_irq(&xas); 1821 xas_set(&xas, index); 1822 1823 VM_BUG_ON_PAGE(page != xas_load(&xas), page); 1824 1825 /* 1826 * The page is expected to have page_count() == 3: 1827 * - we hold a pin on it; 1828 * - one reference from page cache; 1829 * - one from isolate_lru_page; 1830 */ 1831 if (!page_ref_freeze(page, 3)) { 1832 result = SCAN_PAGE_COUNT; 1833 xas_unlock_irq(&xas); 1834 putback_lru_page(page); 1835 goto out_unlock; 1836 } 1837 1838 /* 1839 * Add the page to the list to be able to undo the collapse if 1840 * something go wrong. 1841 */ 1842 list_add_tail(&page->lru, &pagelist); 1843 1844 /* Finally, replace with the new page. */ 1845 xas_store(&xas, new_page); 1846 continue; 1847 out_unlock: 1848 unlock_page(page); 1849 put_page(page); 1850 goto xa_unlocked; 1851 } 1852 nr = thp_nr_pages(new_page); 1853 1854 if (is_shmem) 1855 __mod_lruvec_page_state(new_page, NR_SHMEM_THPS, nr); 1856 else { 1857 __mod_lruvec_page_state(new_page, NR_FILE_THPS, nr); 1858 filemap_nr_thps_inc(mapping); 1859 /* 1860 * Paired with smp_mb() in do_dentry_open() to ensure 1861 * i_writecount is up to date and the update to nr_thps is 1862 * visible. Ensures the page cache will be truncated if the 1863 * file is opened writable. 1864 */ 1865 smp_mb(); 1866 if (inode_is_open_for_write(mapping->host)) { 1867 result = SCAN_FAIL; 1868 __mod_lruvec_page_state(new_page, NR_FILE_THPS, -nr); 1869 filemap_nr_thps_dec(mapping); 1870 goto xa_locked; 1871 } 1872 } 1873 1874 if (nr_none) { 1875 __mod_lruvec_page_state(new_page, NR_FILE_PAGES, nr_none); 1876 /* nr_none is always 0 for non-shmem. */ 1877 __mod_lruvec_page_state(new_page, NR_SHMEM, nr_none); 1878 } 1879 1880 /* Join all the small entries into a single multi-index entry */ 1881 xas_set_order(&xas, start, HPAGE_PMD_ORDER); 1882 xas_store(&xas, new_page); 1883 xa_locked: 1884 xas_unlock_irq(&xas); 1885 xa_unlocked: 1886 1887 /* 1888 * If collapse is successful, flush must be done now before copying. 1889 * If collapse is unsuccessful, does flush actually need to be done? 1890 * Do it anyway, to clear the state. 1891 */ 1892 try_to_unmap_flush(); 1893 1894 if (result == SCAN_SUCCEED) { 1895 struct page *page, *tmp; 1896 1897 /* 1898 * Replacing old pages with new one has succeeded, now we 1899 * need to copy the content and free the old pages. 1900 */ 1901 index = start; 1902 list_for_each_entry_safe(page, tmp, &pagelist, lru) { 1903 while (index < page->index) { 1904 clear_highpage(new_page + (index % HPAGE_PMD_NR)); 1905 index++; 1906 } 1907 copy_highpage(new_page + (page->index % HPAGE_PMD_NR), 1908 page); 1909 list_del(&page->lru); 1910 page->mapping = NULL; 1911 page_ref_unfreeze(page, 1); 1912 ClearPageActive(page); 1913 ClearPageUnevictable(page); 1914 unlock_page(page); 1915 put_page(page); 1916 index++; 1917 } 1918 while (index < end) { 1919 clear_highpage(new_page + (index % HPAGE_PMD_NR)); 1920 index++; 1921 } 1922 1923 SetPageUptodate(new_page); 1924 page_ref_add(new_page, HPAGE_PMD_NR - 1); 1925 if (is_shmem) 1926 set_page_dirty(new_page); 1927 lru_cache_add(new_page); 1928 1929 /* 1930 * Remove pte page tables, so we can re-fault the page as huge. 1931 */ 1932 retract_page_tables(mapping, start); 1933 *hpage = NULL; 1934 1935 khugepaged_pages_collapsed++; 1936 } else { 1937 struct page *page; 1938 1939 /* Something went wrong: roll back page cache changes */ 1940 xas_lock_irq(&xas); 1941 if (nr_none) { 1942 mapping->nrpages -= nr_none; 1943 shmem_uncharge(mapping->host, nr_none); 1944 } 1945 1946 xas_set(&xas, start); 1947 xas_for_each(&xas, page, end - 1) { 1948 page = list_first_entry_or_null(&pagelist, 1949 struct page, lru); 1950 if (!page || xas.xa_index < page->index) { 1951 if (!nr_none) 1952 break; 1953 nr_none--; 1954 /* Put holes back where they were */ 1955 xas_store(&xas, NULL); 1956 continue; 1957 } 1958 1959 VM_BUG_ON_PAGE(page->index != xas.xa_index, page); 1960 1961 /* Unfreeze the page. */ 1962 list_del(&page->lru); 1963 page_ref_unfreeze(page, 2); 1964 xas_store(&xas, page); 1965 xas_pause(&xas); 1966 xas_unlock_irq(&xas); 1967 unlock_page(page); 1968 putback_lru_page(page); 1969 xas_lock_irq(&xas); 1970 } 1971 VM_BUG_ON(nr_none); 1972 xas_unlock_irq(&xas); 1973 1974 new_page->mapping = NULL; 1975 } 1976 1977 unlock_page(new_page); 1978 out: 1979 VM_BUG_ON(!list_empty(&pagelist)); 1980 if (!IS_ERR_OR_NULL(*hpage)) 1981 mem_cgroup_uncharge(page_folio(*hpage)); 1982 /* TODO: tracepoints */ 1983 } 1984 1985 static void khugepaged_scan_file(struct mm_struct *mm, 1986 struct file *file, pgoff_t start, struct page **hpage) 1987 { 1988 struct page *page = NULL; 1989 struct address_space *mapping = file->f_mapping; 1990 XA_STATE(xas, &mapping->i_pages, start); 1991 int present, swap; 1992 int node = NUMA_NO_NODE; 1993 int result = SCAN_SUCCEED; 1994 1995 present = 0; 1996 swap = 0; 1997 memset(khugepaged_node_load, 0, sizeof(khugepaged_node_load)); 1998 rcu_read_lock(); 1999 xas_for_each(&xas, page, start + HPAGE_PMD_NR - 1) { 2000 if (xas_retry(&xas, page)) 2001 continue; 2002 2003 if (xa_is_value(page)) { 2004 if (++swap > khugepaged_max_ptes_swap) { 2005 result = SCAN_EXCEED_SWAP_PTE; 2006 count_vm_event(THP_SCAN_EXCEED_SWAP_PTE); 2007 break; 2008 } 2009 continue; 2010 } 2011 2012 /* 2013 * XXX: khugepaged should compact smaller compound pages 2014 * into a PMD sized page 2015 */ 2016 if (PageTransCompound(page)) { 2017 result = SCAN_PAGE_COMPOUND; 2018 break; 2019 } 2020 2021 node = page_to_nid(page); 2022 if (khugepaged_scan_abort(node)) { 2023 result = SCAN_SCAN_ABORT; 2024 break; 2025 } 2026 khugepaged_node_load[node]++; 2027 2028 if (!PageLRU(page)) { 2029 result = SCAN_PAGE_LRU; 2030 break; 2031 } 2032 2033 if (page_count(page) != 2034 1 + page_mapcount(page) + page_has_private(page)) { 2035 result = SCAN_PAGE_COUNT; 2036 break; 2037 } 2038 2039 /* 2040 * We probably should check if the page is referenced here, but 2041 * nobody would transfer pte_young() to PageReferenced() for us. 2042 * And rmap walk here is just too costly... 2043 */ 2044 2045 present++; 2046 2047 if (need_resched()) { 2048 xas_pause(&xas); 2049 cond_resched_rcu(); 2050 } 2051 } 2052 rcu_read_unlock(); 2053 2054 if (result == SCAN_SUCCEED) { 2055 if (present < HPAGE_PMD_NR - khugepaged_max_ptes_none) { 2056 result = SCAN_EXCEED_NONE_PTE; 2057 count_vm_event(THP_SCAN_EXCEED_NONE_PTE); 2058 } else { 2059 node = khugepaged_find_target_node(); 2060 collapse_file(mm, file, start, hpage, node); 2061 } 2062 } 2063 2064 /* TODO: tracepoints */ 2065 } 2066 #else 2067 static void khugepaged_scan_file(struct mm_struct *mm, 2068 struct file *file, pgoff_t start, struct page **hpage) 2069 { 2070 BUILD_BUG(); 2071 } 2072 2073 static void khugepaged_collapse_pte_mapped_thps(struct mm_slot *mm_slot) 2074 { 2075 } 2076 #endif 2077 2078 static unsigned int khugepaged_scan_mm_slot(unsigned int pages, 2079 struct page **hpage) 2080 __releases(&khugepaged_mm_lock) 2081 __acquires(&khugepaged_mm_lock) 2082 { 2083 struct mm_slot *mm_slot; 2084 struct mm_struct *mm; 2085 struct vm_area_struct *vma; 2086 int progress = 0; 2087 2088 VM_BUG_ON(!pages); 2089 lockdep_assert_held(&khugepaged_mm_lock); 2090 2091 if (khugepaged_scan.mm_slot) 2092 mm_slot = khugepaged_scan.mm_slot; 2093 else { 2094 mm_slot = list_entry(khugepaged_scan.mm_head.next, 2095 struct mm_slot, mm_node); 2096 khugepaged_scan.address = 0; 2097 khugepaged_scan.mm_slot = mm_slot; 2098 } 2099 spin_unlock(&khugepaged_mm_lock); 2100 khugepaged_collapse_pte_mapped_thps(mm_slot); 2101 2102 mm = mm_slot->mm; 2103 /* 2104 * Don't wait for semaphore (to avoid long wait times). Just move to 2105 * the next mm on the list. 2106 */ 2107 vma = NULL; 2108 if (unlikely(!mmap_read_trylock(mm))) 2109 goto breakouterloop_mmap_lock; 2110 if (likely(!khugepaged_test_exit(mm))) 2111 vma = find_vma(mm, khugepaged_scan.address); 2112 2113 progress++; 2114 for (; vma; vma = vma->vm_next) { 2115 unsigned long hstart, hend; 2116 2117 cond_resched(); 2118 if (unlikely(khugepaged_test_exit(mm))) { 2119 progress++; 2120 break; 2121 } 2122 if (!hugepage_vma_check(vma, vma->vm_flags)) { 2123 skip: 2124 progress++; 2125 continue; 2126 } 2127 hstart = (vma->vm_start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK; 2128 hend = vma->vm_end & HPAGE_PMD_MASK; 2129 if (hstart >= hend) 2130 goto skip; 2131 if (khugepaged_scan.address > hend) 2132 goto skip; 2133 if (khugepaged_scan.address < hstart) 2134 khugepaged_scan.address = hstart; 2135 VM_BUG_ON(khugepaged_scan.address & ~HPAGE_PMD_MASK); 2136 2137 while (khugepaged_scan.address < hend) { 2138 int ret; 2139 cond_resched(); 2140 if (unlikely(khugepaged_test_exit(mm))) 2141 goto breakouterloop; 2142 2143 VM_BUG_ON(khugepaged_scan.address < hstart || 2144 khugepaged_scan.address + HPAGE_PMD_SIZE > 2145 hend); 2146 if (IS_ENABLED(CONFIG_SHMEM) && vma->vm_file) { 2147 struct file *file = get_file(vma->vm_file); 2148 pgoff_t pgoff = linear_page_index(vma, 2149 khugepaged_scan.address); 2150 2151 mmap_read_unlock(mm); 2152 ret = 1; 2153 khugepaged_scan_file(mm, file, pgoff, hpage); 2154 fput(file); 2155 } else { 2156 ret = khugepaged_scan_pmd(mm, vma, 2157 khugepaged_scan.address, 2158 hpage); 2159 } 2160 /* move to next address */ 2161 khugepaged_scan.address += HPAGE_PMD_SIZE; 2162 progress += HPAGE_PMD_NR; 2163 if (ret) 2164 /* we released mmap_lock so break loop */ 2165 goto breakouterloop_mmap_lock; 2166 if (progress >= pages) 2167 goto breakouterloop; 2168 } 2169 } 2170 breakouterloop: 2171 mmap_read_unlock(mm); /* exit_mmap will destroy ptes after this */ 2172 breakouterloop_mmap_lock: 2173 2174 spin_lock(&khugepaged_mm_lock); 2175 VM_BUG_ON(khugepaged_scan.mm_slot != mm_slot); 2176 /* 2177 * Release the current mm_slot if this mm is about to die, or 2178 * if we scanned all vmas of this mm. 2179 */ 2180 if (khugepaged_test_exit(mm) || !vma) { 2181 /* 2182 * Make sure that if mm_users is reaching zero while 2183 * khugepaged runs here, khugepaged_exit will find 2184 * mm_slot not pointing to the exiting mm. 2185 */ 2186 if (mm_slot->mm_node.next != &khugepaged_scan.mm_head) { 2187 khugepaged_scan.mm_slot = list_entry( 2188 mm_slot->mm_node.next, 2189 struct mm_slot, mm_node); 2190 khugepaged_scan.address = 0; 2191 } else { 2192 khugepaged_scan.mm_slot = NULL; 2193 khugepaged_full_scans++; 2194 } 2195 2196 collect_mm_slot(mm_slot); 2197 } 2198 2199 return progress; 2200 } 2201 2202 static int khugepaged_has_work(void) 2203 { 2204 return !list_empty(&khugepaged_scan.mm_head) && 2205 khugepaged_enabled(); 2206 } 2207 2208 static int khugepaged_wait_event(void) 2209 { 2210 return !list_empty(&khugepaged_scan.mm_head) || 2211 kthread_should_stop(); 2212 } 2213 2214 static void khugepaged_do_scan(void) 2215 { 2216 struct page *hpage = NULL; 2217 unsigned int progress = 0, pass_through_head = 0; 2218 unsigned int pages = READ_ONCE(khugepaged_pages_to_scan); 2219 bool wait = true; 2220 2221 lru_add_drain_all(); 2222 2223 while (progress < pages) { 2224 if (!khugepaged_prealloc_page(&hpage, &wait)) 2225 break; 2226 2227 cond_resched(); 2228 2229 if (unlikely(kthread_should_stop() || try_to_freeze())) 2230 break; 2231 2232 spin_lock(&khugepaged_mm_lock); 2233 if (!khugepaged_scan.mm_slot) 2234 pass_through_head++; 2235 if (khugepaged_has_work() && 2236 pass_through_head < 2) 2237 progress += khugepaged_scan_mm_slot(pages - progress, 2238 &hpage); 2239 else 2240 progress = pages; 2241 spin_unlock(&khugepaged_mm_lock); 2242 } 2243 2244 if (!IS_ERR_OR_NULL(hpage)) 2245 put_page(hpage); 2246 } 2247 2248 static bool khugepaged_should_wakeup(void) 2249 { 2250 return kthread_should_stop() || 2251 time_after_eq(jiffies, khugepaged_sleep_expire); 2252 } 2253 2254 static void khugepaged_wait_work(void) 2255 { 2256 if (khugepaged_has_work()) { 2257 const unsigned long scan_sleep_jiffies = 2258 msecs_to_jiffies(khugepaged_scan_sleep_millisecs); 2259 2260 if (!scan_sleep_jiffies) 2261 return; 2262 2263 khugepaged_sleep_expire = jiffies + scan_sleep_jiffies; 2264 wait_event_freezable_timeout(khugepaged_wait, 2265 khugepaged_should_wakeup(), 2266 scan_sleep_jiffies); 2267 return; 2268 } 2269 2270 if (khugepaged_enabled()) 2271 wait_event_freezable(khugepaged_wait, khugepaged_wait_event()); 2272 } 2273 2274 static int khugepaged(void *none) 2275 { 2276 struct mm_slot *mm_slot; 2277 2278 set_freezable(); 2279 set_user_nice(current, MAX_NICE); 2280 2281 while (!kthread_should_stop()) { 2282 khugepaged_do_scan(); 2283 khugepaged_wait_work(); 2284 } 2285 2286 spin_lock(&khugepaged_mm_lock); 2287 mm_slot = khugepaged_scan.mm_slot; 2288 khugepaged_scan.mm_slot = NULL; 2289 if (mm_slot) 2290 collect_mm_slot(mm_slot); 2291 spin_unlock(&khugepaged_mm_lock); 2292 return 0; 2293 } 2294 2295 static void set_recommended_min_free_kbytes(void) 2296 { 2297 struct zone *zone; 2298 int nr_zones = 0; 2299 unsigned long recommended_min; 2300 2301 if (!khugepaged_enabled()) { 2302 calculate_min_free_kbytes(); 2303 goto update_wmarks; 2304 } 2305 2306 for_each_populated_zone(zone) { 2307 /* 2308 * We don't need to worry about fragmentation of 2309 * ZONE_MOVABLE since it only has movable pages. 2310 */ 2311 if (zone_idx(zone) > gfp_zone(GFP_USER)) 2312 continue; 2313 2314 nr_zones++; 2315 } 2316 2317 /* Ensure 2 pageblocks are free to assist fragmentation avoidance */ 2318 recommended_min = pageblock_nr_pages * nr_zones * 2; 2319 2320 /* 2321 * Make sure that on average at least two pageblocks are almost free 2322 * of another type, one for a migratetype to fall back to and a 2323 * second to avoid subsequent fallbacks of other types There are 3 2324 * MIGRATE_TYPES we care about. 2325 */ 2326 recommended_min += pageblock_nr_pages * nr_zones * 2327 MIGRATE_PCPTYPES * MIGRATE_PCPTYPES; 2328 2329 /* don't ever allow to reserve more than 5% of the lowmem */ 2330 recommended_min = min(recommended_min, 2331 (unsigned long) nr_free_buffer_pages() / 20); 2332 recommended_min <<= (PAGE_SHIFT-10); 2333 2334 if (recommended_min > min_free_kbytes) { 2335 if (user_min_free_kbytes >= 0) 2336 pr_info("raising min_free_kbytes from %d to %lu to help transparent hugepage allocations\n", 2337 min_free_kbytes, recommended_min); 2338 2339 min_free_kbytes = recommended_min; 2340 } 2341 2342 update_wmarks: 2343 setup_per_zone_wmarks(); 2344 } 2345 2346 int start_stop_khugepaged(void) 2347 { 2348 int err = 0; 2349 2350 mutex_lock(&khugepaged_mutex); 2351 if (khugepaged_enabled()) { 2352 if (!khugepaged_thread) 2353 khugepaged_thread = kthread_run(khugepaged, NULL, 2354 "khugepaged"); 2355 if (IS_ERR(khugepaged_thread)) { 2356 pr_err("khugepaged: kthread_run(khugepaged) failed\n"); 2357 err = PTR_ERR(khugepaged_thread); 2358 khugepaged_thread = NULL; 2359 goto fail; 2360 } 2361 2362 if (!list_empty(&khugepaged_scan.mm_head)) 2363 wake_up_interruptible(&khugepaged_wait); 2364 } else if (khugepaged_thread) { 2365 kthread_stop(khugepaged_thread); 2366 khugepaged_thread = NULL; 2367 } 2368 set_recommended_min_free_kbytes(); 2369 fail: 2370 mutex_unlock(&khugepaged_mutex); 2371 return err; 2372 } 2373 2374 void khugepaged_min_free_kbytes_update(void) 2375 { 2376 mutex_lock(&khugepaged_mutex); 2377 if (khugepaged_enabled() && khugepaged_thread) 2378 set_recommended_min_free_kbytes(); 2379 mutex_unlock(&khugepaged_mutex); 2380 } 2381