1 // SPDX-License-Identifier: GPL-2.0 2 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 3 4 #include <linux/mm.h> 5 #include <linux/sched.h> 6 #include <linux/sched/mm.h> 7 #include <linux/sched/coredump.h> 8 #include <linux/mmu_notifier.h> 9 #include <linux/rmap.h> 10 #include <linux/swap.h> 11 #include <linux/mm_inline.h> 12 #include <linux/kthread.h> 13 #include <linux/khugepaged.h> 14 #include <linux/freezer.h> 15 #include <linux/mman.h> 16 #include <linux/hashtable.h> 17 #include <linux/userfaultfd_k.h> 18 #include <linux/page_idle.h> 19 #include <linux/page_table_check.h> 20 #include <linux/swapops.h> 21 #include <linux/shmem_fs.h> 22 23 #include <asm/tlb.h> 24 #include <asm/pgalloc.h> 25 #include "internal.h" 26 27 enum scan_result { 28 SCAN_FAIL, 29 SCAN_SUCCEED, 30 SCAN_PMD_NULL, 31 SCAN_EXCEED_NONE_PTE, 32 SCAN_EXCEED_SWAP_PTE, 33 SCAN_EXCEED_SHARED_PTE, 34 SCAN_PTE_NON_PRESENT, 35 SCAN_PTE_UFFD_WP, 36 SCAN_PAGE_RO, 37 SCAN_LACK_REFERENCED_PAGE, 38 SCAN_PAGE_NULL, 39 SCAN_SCAN_ABORT, 40 SCAN_PAGE_COUNT, 41 SCAN_PAGE_LRU, 42 SCAN_PAGE_LOCK, 43 SCAN_PAGE_ANON, 44 SCAN_PAGE_COMPOUND, 45 SCAN_ANY_PROCESS, 46 SCAN_VMA_NULL, 47 SCAN_VMA_CHECK, 48 SCAN_ADDRESS_RANGE, 49 SCAN_DEL_PAGE_LRU, 50 SCAN_ALLOC_HUGE_PAGE_FAIL, 51 SCAN_CGROUP_CHARGE_FAIL, 52 SCAN_TRUNCATED, 53 SCAN_PAGE_HAS_PRIVATE, 54 }; 55 56 #define CREATE_TRACE_POINTS 57 #include <trace/events/huge_memory.h> 58 59 static struct task_struct *khugepaged_thread __read_mostly; 60 static DEFINE_MUTEX(khugepaged_mutex); 61 62 /* default scan 8*512 pte (or vmas) every 30 second */ 63 static unsigned int khugepaged_pages_to_scan __read_mostly; 64 static unsigned int khugepaged_pages_collapsed; 65 static unsigned int khugepaged_full_scans; 66 static unsigned int khugepaged_scan_sleep_millisecs __read_mostly = 10000; 67 /* during fragmentation poll the hugepage allocator once every minute */ 68 static unsigned int khugepaged_alloc_sleep_millisecs __read_mostly = 60000; 69 static unsigned long khugepaged_sleep_expire; 70 static DEFINE_SPINLOCK(khugepaged_mm_lock); 71 static DECLARE_WAIT_QUEUE_HEAD(khugepaged_wait); 72 /* 73 * default collapse hugepages if there is at least one pte mapped like 74 * it would have happened if the vma was large enough during page 75 * fault. 76 */ 77 static unsigned int khugepaged_max_ptes_none __read_mostly; 78 static unsigned int khugepaged_max_ptes_swap __read_mostly; 79 static unsigned int khugepaged_max_ptes_shared __read_mostly; 80 81 #define MM_SLOTS_HASH_BITS 10 82 static __read_mostly DEFINE_HASHTABLE(mm_slots_hash, MM_SLOTS_HASH_BITS); 83 84 static struct kmem_cache *mm_slot_cache __read_mostly; 85 86 #define MAX_PTE_MAPPED_THP 8 87 88 /** 89 * struct mm_slot - hash lookup from mm to mm_slot 90 * @hash: hash collision list 91 * @mm_node: khugepaged scan list headed in khugepaged_scan.mm_head 92 * @mm: the mm that this information is valid for 93 * @nr_pte_mapped_thp: number of pte mapped THP 94 * @pte_mapped_thp: address array corresponding pte mapped THP 95 */ 96 struct mm_slot { 97 struct hlist_node hash; 98 struct list_head mm_node; 99 struct mm_struct *mm; 100 101 /* pte-mapped THP in this mm */ 102 int nr_pte_mapped_thp; 103 unsigned long pte_mapped_thp[MAX_PTE_MAPPED_THP]; 104 }; 105 106 /** 107 * struct khugepaged_scan - cursor for scanning 108 * @mm_head: the head of the mm list to scan 109 * @mm_slot: the current mm_slot we are scanning 110 * @address: the next address inside that to be scanned 111 * 112 * There is only the one khugepaged_scan instance of this cursor structure. 113 */ 114 struct khugepaged_scan { 115 struct list_head mm_head; 116 struct mm_slot *mm_slot; 117 unsigned long address; 118 }; 119 120 static struct khugepaged_scan khugepaged_scan = { 121 .mm_head = LIST_HEAD_INIT(khugepaged_scan.mm_head), 122 }; 123 124 #ifdef CONFIG_SYSFS 125 static ssize_t scan_sleep_millisecs_show(struct kobject *kobj, 126 struct kobj_attribute *attr, 127 char *buf) 128 { 129 return sysfs_emit(buf, "%u\n", khugepaged_scan_sleep_millisecs); 130 } 131 132 static ssize_t scan_sleep_millisecs_store(struct kobject *kobj, 133 struct kobj_attribute *attr, 134 const char *buf, size_t count) 135 { 136 unsigned int msecs; 137 int err; 138 139 err = kstrtouint(buf, 10, &msecs); 140 if (err) 141 return -EINVAL; 142 143 khugepaged_scan_sleep_millisecs = msecs; 144 khugepaged_sleep_expire = 0; 145 wake_up_interruptible(&khugepaged_wait); 146 147 return count; 148 } 149 static struct kobj_attribute scan_sleep_millisecs_attr = 150 __ATTR_RW(scan_sleep_millisecs); 151 152 static ssize_t alloc_sleep_millisecs_show(struct kobject *kobj, 153 struct kobj_attribute *attr, 154 char *buf) 155 { 156 return sysfs_emit(buf, "%u\n", khugepaged_alloc_sleep_millisecs); 157 } 158 159 static ssize_t alloc_sleep_millisecs_store(struct kobject *kobj, 160 struct kobj_attribute *attr, 161 const char *buf, size_t count) 162 { 163 unsigned int msecs; 164 int err; 165 166 err = kstrtouint(buf, 10, &msecs); 167 if (err) 168 return -EINVAL; 169 170 khugepaged_alloc_sleep_millisecs = msecs; 171 khugepaged_sleep_expire = 0; 172 wake_up_interruptible(&khugepaged_wait); 173 174 return count; 175 } 176 static struct kobj_attribute alloc_sleep_millisecs_attr = 177 __ATTR_RW(alloc_sleep_millisecs); 178 179 static ssize_t pages_to_scan_show(struct kobject *kobj, 180 struct kobj_attribute *attr, 181 char *buf) 182 { 183 return sysfs_emit(buf, "%u\n", khugepaged_pages_to_scan); 184 } 185 static ssize_t pages_to_scan_store(struct kobject *kobj, 186 struct kobj_attribute *attr, 187 const char *buf, size_t count) 188 { 189 unsigned int pages; 190 int err; 191 192 err = kstrtouint(buf, 10, &pages); 193 if (err || !pages) 194 return -EINVAL; 195 196 khugepaged_pages_to_scan = pages; 197 198 return count; 199 } 200 static struct kobj_attribute pages_to_scan_attr = 201 __ATTR_RW(pages_to_scan); 202 203 static ssize_t pages_collapsed_show(struct kobject *kobj, 204 struct kobj_attribute *attr, 205 char *buf) 206 { 207 return sysfs_emit(buf, "%u\n", khugepaged_pages_collapsed); 208 } 209 static struct kobj_attribute pages_collapsed_attr = 210 __ATTR_RO(pages_collapsed); 211 212 static ssize_t full_scans_show(struct kobject *kobj, 213 struct kobj_attribute *attr, 214 char *buf) 215 { 216 return sysfs_emit(buf, "%u\n", khugepaged_full_scans); 217 } 218 static struct kobj_attribute full_scans_attr = 219 __ATTR_RO(full_scans); 220 221 static ssize_t defrag_show(struct kobject *kobj, 222 struct kobj_attribute *attr, char *buf) 223 { 224 return single_hugepage_flag_show(kobj, attr, buf, 225 TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG); 226 } 227 static ssize_t defrag_store(struct kobject *kobj, 228 struct kobj_attribute *attr, 229 const char *buf, size_t count) 230 { 231 return single_hugepage_flag_store(kobj, attr, buf, count, 232 TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG); 233 } 234 static struct kobj_attribute khugepaged_defrag_attr = 235 __ATTR_RW(defrag); 236 237 /* 238 * max_ptes_none controls if khugepaged should collapse hugepages over 239 * any unmapped ptes in turn potentially increasing the memory 240 * footprint of the vmas. When max_ptes_none is 0 khugepaged will not 241 * reduce the available free memory in the system as it 242 * runs. Increasing max_ptes_none will instead potentially reduce the 243 * free memory in the system during the khugepaged scan. 244 */ 245 static ssize_t max_ptes_none_show(struct kobject *kobj, 246 struct kobj_attribute *attr, 247 char *buf) 248 { 249 return sysfs_emit(buf, "%u\n", khugepaged_max_ptes_none); 250 } 251 static ssize_t max_ptes_none_store(struct kobject *kobj, 252 struct kobj_attribute *attr, 253 const char *buf, size_t count) 254 { 255 int err; 256 unsigned long max_ptes_none; 257 258 err = kstrtoul(buf, 10, &max_ptes_none); 259 if (err || max_ptes_none > HPAGE_PMD_NR - 1) 260 return -EINVAL; 261 262 khugepaged_max_ptes_none = max_ptes_none; 263 264 return count; 265 } 266 static struct kobj_attribute khugepaged_max_ptes_none_attr = 267 __ATTR_RW(max_ptes_none); 268 269 static ssize_t max_ptes_swap_show(struct kobject *kobj, 270 struct kobj_attribute *attr, 271 char *buf) 272 { 273 return sysfs_emit(buf, "%u\n", khugepaged_max_ptes_swap); 274 } 275 276 static ssize_t max_ptes_swap_store(struct kobject *kobj, 277 struct kobj_attribute *attr, 278 const char *buf, size_t count) 279 { 280 int err; 281 unsigned long max_ptes_swap; 282 283 err = kstrtoul(buf, 10, &max_ptes_swap); 284 if (err || max_ptes_swap > HPAGE_PMD_NR - 1) 285 return -EINVAL; 286 287 khugepaged_max_ptes_swap = max_ptes_swap; 288 289 return count; 290 } 291 292 static struct kobj_attribute khugepaged_max_ptes_swap_attr = 293 __ATTR_RW(max_ptes_swap); 294 295 static ssize_t max_ptes_shared_show(struct kobject *kobj, 296 struct kobj_attribute *attr, 297 char *buf) 298 { 299 return sysfs_emit(buf, "%u\n", khugepaged_max_ptes_shared); 300 } 301 302 static ssize_t max_ptes_shared_store(struct kobject *kobj, 303 struct kobj_attribute *attr, 304 const char *buf, size_t count) 305 { 306 int err; 307 unsigned long max_ptes_shared; 308 309 err = kstrtoul(buf, 10, &max_ptes_shared); 310 if (err || max_ptes_shared > HPAGE_PMD_NR - 1) 311 return -EINVAL; 312 313 khugepaged_max_ptes_shared = max_ptes_shared; 314 315 return count; 316 } 317 318 static struct kobj_attribute khugepaged_max_ptes_shared_attr = 319 __ATTR_RW(max_ptes_shared); 320 321 static struct attribute *khugepaged_attr[] = { 322 &khugepaged_defrag_attr.attr, 323 &khugepaged_max_ptes_none_attr.attr, 324 &khugepaged_max_ptes_swap_attr.attr, 325 &khugepaged_max_ptes_shared_attr.attr, 326 &pages_to_scan_attr.attr, 327 &pages_collapsed_attr.attr, 328 &full_scans_attr.attr, 329 &scan_sleep_millisecs_attr.attr, 330 &alloc_sleep_millisecs_attr.attr, 331 NULL, 332 }; 333 334 struct attribute_group khugepaged_attr_group = { 335 .attrs = khugepaged_attr, 336 .name = "khugepaged", 337 }; 338 #endif /* CONFIG_SYSFS */ 339 340 int hugepage_madvise(struct vm_area_struct *vma, 341 unsigned long *vm_flags, int advice) 342 { 343 switch (advice) { 344 case MADV_HUGEPAGE: 345 #ifdef CONFIG_S390 346 /* 347 * qemu blindly sets MADV_HUGEPAGE on all allocations, but s390 348 * can't handle this properly after s390_enable_sie, so we simply 349 * ignore the madvise to prevent qemu from causing a SIGSEGV. 350 */ 351 if (mm_has_pgste(vma->vm_mm)) 352 return 0; 353 #endif 354 *vm_flags &= ~VM_NOHUGEPAGE; 355 *vm_flags |= VM_HUGEPAGE; 356 /* 357 * If the vma become good for khugepaged to scan, 358 * register it here without waiting a page fault that 359 * may not happen any time soon. 360 */ 361 khugepaged_enter_vma(vma, *vm_flags); 362 break; 363 case MADV_NOHUGEPAGE: 364 *vm_flags &= ~VM_HUGEPAGE; 365 *vm_flags |= VM_NOHUGEPAGE; 366 /* 367 * Setting VM_NOHUGEPAGE will prevent khugepaged from scanning 368 * this vma even if we leave the mm registered in khugepaged if 369 * it got registered before VM_NOHUGEPAGE was set. 370 */ 371 break; 372 } 373 374 return 0; 375 } 376 377 int __init khugepaged_init(void) 378 { 379 mm_slot_cache = kmem_cache_create("khugepaged_mm_slot", 380 sizeof(struct mm_slot), 381 __alignof__(struct mm_slot), 0, NULL); 382 if (!mm_slot_cache) 383 return -ENOMEM; 384 385 khugepaged_pages_to_scan = HPAGE_PMD_NR * 8; 386 khugepaged_max_ptes_none = HPAGE_PMD_NR - 1; 387 khugepaged_max_ptes_swap = HPAGE_PMD_NR / 8; 388 khugepaged_max_ptes_shared = HPAGE_PMD_NR / 2; 389 390 return 0; 391 } 392 393 void __init khugepaged_destroy(void) 394 { 395 kmem_cache_destroy(mm_slot_cache); 396 } 397 398 static inline struct mm_slot *alloc_mm_slot(void) 399 { 400 if (!mm_slot_cache) /* initialization failed */ 401 return NULL; 402 return kmem_cache_zalloc(mm_slot_cache, GFP_KERNEL); 403 } 404 405 static inline void free_mm_slot(struct mm_slot *mm_slot) 406 { 407 kmem_cache_free(mm_slot_cache, mm_slot); 408 } 409 410 static struct mm_slot *get_mm_slot(struct mm_struct *mm) 411 { 412 struct mm_slot *mm_slot; 413 414 hash_for_each_possible(mm_slots_hash, mm_slot, hash, (unsigned long)mm) 415 if (mm == mm_slot->mm) 416 return mm_slot; 417 418 return NULL; 419 } 420 421 static void insert_to_mm_slots_hash(struct mm_struct *mm, 422 struct mm_slot *mm_slot) 423 { 424 mm_slot->mm = mm; 425 hash_add(mm_slots_hash, &mm_slot->hash, (long)mm); 426 } 427 428 static inline int khugepaged_test_exit(struct mm_struct *mm) 429 { 430 return atomic_read(&mm->mm_users) == 0; 431 } 432 433 bool hugepage_vma_check(struct vm_area_struct *vma, 434 unsigned long vm_flags) 435 { 436 if (!transhuge_vma_enabled(vma, vm_flags)) 437 return false; 438 439 if (vm_flags & VM_NO_KHUGEPAGED) 440 return false; 441 442 /* Don't run khugepaged against DAX vma */ 443 if (vma_is_dax(vma)) 444 return false; 445 446 /* Check alignment for file vma and size for both file and anon vma */ 447 if (!transhuge_vma_suitable(vma, (vma->vm_end - HPAGE_PMD_SIZE))) 448 return false; 449 450 /* Enabled via shmem mount options or sysfs settings. */ 451 if (shmem_file(vma->vm_file)) 452 return shmem_huge_enabled(vma); 453 454 if (!khugepaged_enabled()) 455 return false; 456 457 /* THP settings require madvise. */ 458 if (!(vm_flags & VM_HUGEPAGE) && !khugepaged_always()) 459 return false; 460 461 /* Only regular file is valid */ 462 if (file_thp_enabled(vma)) 463 return true; 464 465 if (!vma->anon_vma || !vma_is_anonymous(vma)) 466 return false; 467 if (vma_is_temporary_stack(vma)) 468 return false; 469 470 return true; 471 } 472 473 void __khugepaged_enter(struct mm_struct *mm) 474 { 475 struct mm_slot *mm_slot; 476 int wakeup; 477 478 mm_slot = alloc_mm_slot(); 479 if (!mm_slot) 480 return; 481 482 /* __khugepaged_exit() must not run from under us */ 483 VM_BUG_ON_MM(khugepaged_test_exit(mm), mm); 484 if (unlikely(test_and_set_bit(MMF_VM_HUGEPAGE, &mm->flags))) { 485 free_mm_slot(mm_slot); 486 return; 487 } 488 489 spin_lock(&khugepaged_mm_lock); 490 insert_to_mm_slots_hash(mm, mm_slot); 491 /* 492 * Insert just behind the scanning cursor, to let the area settle 493 * down a little. 494 */ 495 wakeup = list_empty(&khugepaged_scan.mm_head); 496 list_add_tail(&mm_slot->mm_node, &khugepaged_scan.mm_head); 497 spin_unlock(&khugepaged_mm_lock); 498 499 mmgrab(mm); 500 if (wakeup) 501 wake_up_interruptible(&khugepaged_wait); 502 } 503 504 void khugepaged_enter_vma(struct vm_area_struct *vma, 505 unsigned long vm_flags) 506 { 507 if (!test_bit(MMF_VM_HUGEPAGE, &vma->vm_mm->flags) && 508 khugepaged_enabled()) { 509 if (hugepage_vma_check(vma, vm_flags)) 510 __khugepaged_enter(vma->vm_mm); 511 } 512 } 513 514 void __khugepaged_exit(struct mm_struct *mm) 515 { 516 struct mm_slot *mm_slot; 517 int free = 0; 518 519 spin_lock(&khugepaged_mm_lock); 520 mm_slot = get_mm_slot(mm); 521 if (mm_slot && khugepaged_scan.mm_slot != mm_slot) { 522 hash_del(&mm_slot->hash); 523 list_del(&mm_slot->mm_node); 524 free = 1; 525 } 526 spin_unlock(&khugepaged_mm_lock); 527 528 if (free) { 529 clear_bit(MMF_VM_HUGEPAGE, &mm->flags); 530 free_mm_slot(mm_slot); 531 mmdrop(mm); 532 } else if (mm_slot) { 533 /* 534 * This is required to serialize against 535 * khugepaged_test_exit() (which is guaranteed to run 536 * under mmap sem read mode). Stop here (after we 537 * return all pagetables will be destroyed) until 538 * khugepaged has finished working on the pagetables 539 * under the mmap_lock. 540 */ 541 mmap_write_lock(mm); 542 mmap_write_unlock(mm); 543 } 544 } 545 546 static void release_pte_page(struct page *page) 547 { 548 mod_node_page_state(page_pgdat(page), 549 NR_ISOLATED_ANON + page_is_file_lru(page), 550 -compound_nr(page)); 551 unlock_page(page); 552 putback_lru_page(page); 553 } 554 555 static void release_pte_pages(pte_t *pte, pte_t *_pte, 556 struct list_head *compound_pagelist) 557 { 558 struct page *page, *tmp; 559 560 while (--_pte >= pte) { 561 pte_t pteval = *_pte; 562 563 page = pte_page(pteval); 564 if (!pte_none(pteval) && !is_zero_pfn(pte_pfn(pteval)) && 565 !PageCompound(page)) 566 release_pte_page(page); 567 } 568 569 list_for_each_entry_safe(page, tmp, compound_pagelist, lru) { 570 list_del(&page->lru); 571 release_pte_page(page); 572 } 573 } 574 575 static bool is_refcount_suitable(struct page *page) 576 { 577 int expected_refcount; 578 579 expected_refcount = total_mapcount(page); 580 if (PageSwapCache(page)) 581 expected_refcount += compound_nr(page); 582 583 return page_count(page) == expected_refcount; 584 } 585 586 static int __collapse_huge_page_isolate(struct vm_area_struct *vma, 587 unsigned long address, 588 pte_t *pte, 589 struct list_head *compound_pagelist) 590 { 591 struct page *page = NULL; 592 pte_t *_pte; 593 int none_or_zero = 0, shared = 0, result = 0, referenced = 0; 594 bool writable = false; 595 596 for (_pte = pte; _pte < pte + HPAGE_PMD_NR; 597 _pte++, address += PAGE_SIZE) { 598 pte_t pteval = *_pte; 599 if (pte_none(pteval) || (pte_present(pteval) && 600 is_zero_pfn(pte_pfn(pteval)))) { 601 if (!userfaultfd_armed(vma) && 602 ++none_or_zero <= khugepaged_max_ptes_none) { 603 continue; 604 } else { 605 result = SCAN_EXCEED_NONE_PTE; 606 count_vm_event(THP_SCAN_EXCEED_NONE_PTE); 607 goto out; 608 } 609 } 610 if (!pte_present(pteval)) { 611 result = SCAN_PTE_NON_PRESENT; 612 goto out; 613 } 614 page = vm_normal_page(vma, address, pteval); 615 if (unlikely(!page) || unlikely(is_zone_device_page(page))) { 616 result = SCAN_PAGE_NULL; 617 goto out; 618 } 619 620 VM_BUG_ON_PAGE(!PageAnon(page), page); 621 622 if (page_mapcount(page) > 1 && 623 ++shared > khugepaged_max_ptes_shared) { 624 result = SCAN_EXCEED_SHARED_PTE; 625 count_vm_event(THP_SCAN_EXCEED_SHARED_PTE); 626 goto out; 627 } 628 629 if (PageCompound(page)) { 630 struct page *p; 631 page = compound_head(page); 632 633 /* 634 * Check if we have dealt with the compound page 635 * already 636 */ 637 list_for_each_entry(p, compound_pagelist, lru) { 638 if (page == p) 639 goto next; 640 } 641 } 642 643 /* 644 * We can do it before isolate_lru_page because the 645 * page can't be freed from under us. NOTE: PG_lock 646 * is needed to serialize against split_huge_page 647 * when invoked from the VM. 648 */ 649 if (!trylock_page(page)) { 650 result = SCAN_PAGE_LOCK; 651 goto out; 652 } 653 654 /* 655 * Check if the page has any GUP (or other external) pins. 656 * 657 * The page table that maps the page has been already unlinked 658 * from the page table tree and this process cannot get 659 * an additional pin on the page. 660 * 661 * New pins can come later if the page is shared across fork, 662 * but not from this process. The other process cannot write to 663 * the page, only trigger CoW. 664 */ 665 if (!is_refcount_suitable(page)) { 666 unlock_page(page); 667 result = SCAN_PAGE_COUNT; 668 goto out; 669 } 670 671 /* 672 * Isolate the page to avoid collapsing an hugepage 673 * currently in use by the VM. 674 */ 675 if (isolate_lru_page(page)) { 676 unlock_page(page); 677 result = SCAN_DEL_PAGE_LRU; 678 goto out; 679 } 680 mod_node_page_state(page_pgdat(page), 681 NR_ISOLATED_ANON + page_is_file_lru(page), 682 compound_nr(page)); 683 VM_BUG_ON_PAGE(!PageLocked(page), page); 684 VM_BUG_ON_PAGE(PageLRU(page), page); 685 686 if (PageCompound(page)) 687 list_add_tail(&page->lru, compound_pagelist); 688 next: 689 /* There should be enough young pte to collapse the page */ 690 if (pte_young(pteval) || 691 page_is_young(page) || PageReferenced(page) || 692 mmu_notifier_test_young(vma->vm_mm, address)) 693 referenced++; 694 695 if (pte_write(pteval)) 696 writable = true; 697 } 698 699 if (unlikely(!writable)) { 700 result = SCAN_PAGE_RO; 701 } else if (unlikely(!referenced)) { 702 result = SCAN_LACK_REFERENCED_PAGE; 703 } else { 704 result = SCAN_SUCCEED; 705 trace_mm_collapse_huge_page_isolate(page, none_or_zero, 706 referenced, writable, result); 707 return 1; 708 } 709 out: 710 release_pte_pages(pte, _pte, compound_pagelist); 711 trace_mm_collapse_huge_page_isolate(page, none_or_zero, 712 referenced, writable, result); 713 return 0; 714 } 715 716 static void __collapse_huge_page_copy(pte_t *pte, struct page *page, 717 struct vm_area_struct *vma, 718 unsigned long address, 719 spinlock_t *ptl, 720 struct list_head *compound_pagelist) 721 { 722 struct page *src_page, *tmp; 723 pte_t *_pte; 724 for (_pte = pte; _pte < pte + HPAGE_PMD_NR; 725 _pte++, page++, address += PAGE_SIZE) { 726 pte_t pteval = *_pte; 727 728 if (pte_none(pteval) || is_zero_pfn(pte_pfn(pteval))) { 729 clear_user_highpage(page, address); 730 add_mm_counter(vma->vm_mm, MM_ANONPAGES, 1); 731 if (is_zero_pfn(pte_pfn(pteval))) { 732 /* 733 * ptl mostly unnecessary. 734 */ 735 spin_lock(ptl); 736 ptep_clear(vma->vm_mm, address, _pte); 737 spin_unlock(ptl); 738 } 739 } else { 740 src_page = pte_page(pteval); 741 copy_user_highpage(page, src_page, address, vma); 742 if (!PageCompound(src_page)) 743 release_pte_page(src_page); 744 /* 745 * ptl mostly unnecessary, but preempt has to 746 * be disabled to update the per-cpu stats 747 * inside page_remove_rmap(). 748 */ 749 spin_lock(ptl); 750 ptep_clear(vma->vm_mm, address, _pte); 751 page_remove_rmap(src_page, vma, false); 752 spin_unlock(ptl); 753 free_page_and_swap_cache(src_page); 754 } 755 } 756 757 list_for_each_entry_safe(src_page, tmp, compound_pagelist, lru) { 758 list_del(&src_page->lru); 759 mod_node_page_state(page_pgdat(src_page), 760 NR_ISOLATED_ANON + page_is_file_lru(src_page), 761 -compound_nr(src_page)); 762 unlock_page(src_page); 763 free_swap_cache(src_page); 764 putback_lru_page(src_page); 765 } 766 } 767 768 static void khugepaged_alloc_sleep(void) 769 { 770 DEFINE_WAIT(wait); 771 772 add_wait_queue(&khugepaged_wait, &wait); 773 freezable_schedule_timeout_interruptible( 774 msecs_to_jiffies(khugepaged_alloc_sleep_millisecs)); 775 remove_wait_queue(&khugepaged_wait, &wait); 776 } 777 778 static int khugepaged_node_load[MAX_NUMNODES]; 779 780 static bool khugepaged_scan_abort(int nid) 781 { 782 int i; 783 784 /* 785 * If node_reclaim_mode is disabled, then no extra effort is made to 786 * allocate memory locally. 787 */ 788 if (!node_reclaim_enabled()) 789 return false; 790 791 /* If there is a count for this node already, it must be acceptable */ 792 if (khugepaged_node_load[nid]) 793 return false; 794 795 for (i = 0; i < MAX_NUMNODES; i++) { 796 if (!khugepaged_node_load[i]) 797 continue; 798 if (node_distance(nid, i) > node_reclaim_distance) 799 return true; 800 } 801 return false; 802 } 803 804 /* Defrag for khugepaged will enter direct reclaim/compaction if necessary */ 805 static inline gfp_t alloc_hugepage_khugepaged_gfpmask(void) 806 { 807 return khugepaged_defrag() ? GFP_TRANSHUGE : GFP_TRANSHUGE_LIGHT; 808 } 809 810 #ifdef CONFIG_NUMA 811 static int khugepaged_find_target_node(void) 812 { 813 static int last_khugepaged_target_node = NUMA_NO_NODE; 814 int nid, target_node = 0, max_value = 0; 815 816 /* find first node with max normal pages hit */ 817 for (nid = 0; nid < MAX_NUMNODES; nid++) 818 if (khugepaged_node_load[nid] > max_value) { 819 max_value = khugepaged_node_load[nid]; 820 target_node = nid; 821 } 822 823 /* do some balance if several nodes have the same hit record */ 824 if (target_node <= last_khugepaged_target_node) 825 for (nid = last_khugepaged_target_node + 1; nid < MAX_NUMNODES; 826 nid++) 827 if (max_value == khugepaged_node_load[nid]) { 828 target_node = nid; 829 break; 830 } 831 832 last_khugepaged_target_node = target_node; 833 return target_node; 834 } 835 836 static bool khugepaged_prealloc_page(struct page **hpage, bool *wait) 837 { 838 if (IS_ERR(*hpage)) { 839 if (!*wait) 840 return false; 841 842 *wait = false; 843 *hpage = NULL; 844 khugepaged_alloc_sleep(); 845 } else if (*hpage) { 846 put_page(*hpage); 847 *hpage = NULL; 848 } 849 850 return true; 851 } 852 853 static struct page * 854 khugepaged_alloc_page(struct page **hpage, gfp_t gfp, int node) 855 { 856 VM_BUG_ON_PAGE(*hpage, *hpage); 857 858 *hpage = __alloc_pages_node(node, gfp, HPAGE_PMD_ORDER); 859 if (unlikely(!*hpage)) { 860 count_vm_event(THP_COLLAPSE_ALLOC_FAILED); 861 *hpage = ERR_PTR(-ENOMEM); 862 return NULL; 863 } 864 865 prep_transhuge_page(*hpage); 866 count_vm_event(THP_COLLAPSE_ALLOC); 867 return *hpage; 868 } 869 #else 870 static int khugepaged_find_target_node(void) 871 { 872 return 0; 873 } 874 875 static inline struct page *alloc_khugepaged_hugepage(void) 876 { 877 struct page *page; 878 879 page = alloc_pages(alloc_hugepage_khugepaged_gfpmask(), 880 HPAGE_PMD_ORDER); 881 if (page) 882 prep_transhuge_page(page); 883 return page; 884 } 885 886 static struct page *khugepaged_alloc_hugepage(bool *wait) 887 { 888 struct page *hpage; 889 890 do { 891 hpage = alloc_khugepaged_hugepage(); 892 if (!hpage) { 893 count_vm_event(THP_COLLAPSE_ALLOC_FAILED); 894 if (!*wait) 895 return NULL; 896 897 *wait = false; 898 khugepaged_alloc_sleep(); 899 } else 900 count_vm_event(THP_COLLAPSE_ALLOC); 901 } while (unlikely(!hpage) && likely(khugepaged_enabled())); 902 903 return hpage; 904 } 905 906 static bool khugepaged_prealloc_page(struct page **hpage, bool *wait) 907 { 908 /* 909 * If the hpage allocated earlier was briefly exposed in page cache 910 * before collapse_file() failed, it is possible that racing lookups 911 * have not yet completed, and would then be unpleasantly surprised by 912 * finding the hpage reused for the same mapping at a different offset. 913 * Just release the previous allocation if there is any danger of that. 914 */ 915 if (*hpage && page_count(*hpage) > 1) { 916 put_page(*hpage); 917 *hpage = NULL; 918 } 919 920 if (!*hpage) 921 *hpage = khugepaged_alloc_hugepage(wait); 922 923 if (unlikely(!*hpage)) 924 return false; 925 926 return true; 927 } 928 929 static struct page * 930 khugepaged_alloc_page(struct page **hpage, gfp_t gfp, int node) 931 { 932 VM_BUG_ON(!*hpage); 933 934 return *hpage; 935 } 936 #endif 937 938 /* 939 * If mmap_lock temporarily dropped, revalidate vma 940 * before taking mmap_lock. 941 * Return 0 if succeeds, otherwise return none-zero 942 * value (scan code). 943 */ 944 945 static int hugepage_vma_revalidate(struct mm_struct *mm, unsigned long address, 946 struct vm_area_struct **vmap) 947 { 948 struct vm_area_struct *vma; 949 950 if (unlikely(khugepaged_test_exit(mm))) 951 return SCAN_ANY_PROCESS; 952 953 *vmap = vma = find_vma(mm, address); 954 if (!vma) 955 return SCAN_VMA_NULL; 956 957 if (!transhuge_vma_suitable(vma, address)) 958 return SCAN_ADDRESS_RANGE; 959 if (!hugepage_vma_check(vma, vma->vm_flags)) 960 return SCAN_VMA_CHECK; 961 /* Anon VMA expected */ 962 if (!vma->anon_vma || !vma_is_anonymous(vma)) 963 return SCAN_VMA_CHECK; 964 return 0; 965 } 966 967 /* 968 * Bring missing pages in from swap, to complete THP collapse. 969 * Only done if khugepaged_scan_pmd believes it is worthwhile. 970 * 971 * Called and returns without pte mapped or spinlocks held. 972 * Note that if false is returned, mmap_lock will be released. 973 */ 974 975 static bool __collapse_huge_page_swapin(struct mm_struct *mm, 976 struct vm_area_struct *vma, 977 unsigned long haddr, pmd_t *pmd, 978 int referenced) 979 { 980 int swapped_in = 0; 981 vm_fault_t ret = 0; 982 unsigned long address, end = haddr + (HPAGE_PMD_NR * PAGE_SIZE); 983 984 for (address = haddr; address < end; address += PAGE_SIZE) { 985 struct vm_fault vmf = { 986 .vma = vma, 987 .address = address, 988 .pgoff = linear_page_index(vma, haddr), 989 .flags = FAULT_FLAG_ALLOW_RETRY, 990 .pmd = pmd, 991 }; 992 993 vmf.pte = pte_offset_map(pmd, address); 994 vmf.orig_pte = *vmf.pte; 995 if (!is_swap_pte(vmf.orig_pte)) { 996 pte_unmap(vmf.pte); 997 continue; 998 } 999 ret = do_swap_page(&vmf); 1000 1001 /* 1002 * do_swap_page returns VM_FAULT_RETRY with released mmap_lock. 1003 * Note we treat VM_FAULT_RETRY as VM_FAULT_ERROR here because 1004 * we do not retry here and swap entry will remain in pagetable 1005 * resulting in later failure. 1006 */ 1007 if (ret & VM_FAULT_RETRY) { 1008 trace_mm_collapse_huge_page_swapin(mm, swapped_in, referenced, 0); 1009 return false; 1010 } 1011 if (ret & VM_FAULT_ERROR) { 1012 mmap_read_unlock(mm); 1013 trace_mm_collapse_huge_page_swapin(mm, swapped_in, referenced, 0); 1014 return false; 1015 } 1016 swapped_in++; 1017 } 1018 1019 /* Drain LRU add pagevec to remove extra pin on the swapped in pages */ 1020 if (swapped_in) 1021 lru_add_drain(); 1022 1023 trace_mm_collapse_huge_page_swapin(mm, swapped_in, referenced, 1); 1024 return true; 1025 } 1026 1027 static void collapse_huge_page(struct mm_struct *mm, 1028 unsigned long address, 1029 struct page **hpage, 1030 int node, int referenced, int unmapped) 1031 { 1032 LIST_HEAD(compound_pagelist); 1033 pmd_t *pmd, _pmd; 1034 pte_t *pte; 1035 pgtable_t pgtable; 1036 struct page *new_page; 1037 spinlock_t *pmd_ptl, *pte_ptl; 1038 int isolated = 0, result = 0; 1039 struct vm_area_struct *vma; 1040 struct mmu_notifier_range range; 1041 gfp_t gfp; 1042 1043 VM_BUG_ON(address & ~HPAGE_PMD_MASK); 1044 1045 /* Only allocate from the target node */ 1046 gfp = alloc_hugepage_khugepaged_gfpmask() | __GFP_THISNODE; 1047 1048 /* 1049 * Before allocating the hugepage, release the mmap_lock read lock. 1050 * The allocation can take potentially a long time if it involves 1051 * sync compaction, and we do not need to hold the mmap_lock during 1052 * that. We will recheck the vma after taking it again in write mode. 1053 */ 1054 mmap_read_unlock(mm); 1055 new_page = khugepaged_alloc_page(hpage, gfp, node); 1056 if (!new_page) { 1057 result = SCAN_ALLOC_HUGE_PAGE_FAIL; 1058 goto out_nolock; 1059 } 1060 1061 if (unlikely(mem_cgroup_charge(page_folio(new_page), mm, gfp))) { 1062 result = SCAN_CGROUP_CHARGE_FAIL; 1063 goto out_nolock; 1064 } 1065 count_memcg_page_event(new_page, THP_COLLAPSE_ALLOC); 1066 1067 mmap_read_lock(mm); 1068 result = hugepage_vma_revalidate(mm, address, &vma); 1069 if (result) { 1070 mmap_read_unlock(mm); 1071 goto out_nolock; 1072 } 1073 1074 pmd = mm_find_pmd(mm, address); 1075 if (!pmd) { 1076 result = SCAN_PMD_NULL; 1077 mmap_read_unlock(mm); 1078 goto out_nolock; 1079 } 1080 1081 /* 1082 * __collapse_huge_page_swapin will return with mmap_lock released 1083 * when it fails. So we jump out_nolock directly in that case. 1084 * Continuing to collapse causes inconsistency. 1085 */ 1086 if (unmapped && !__collapse_huge_page_swapin(mm, vma, address, 1087 pmd, referenced)) { 1088 goto out_nolock; 1089 } 1090 1091 mmap_read_unlock(mm); 1092 /* 1093 * Prevent all access to pagetables with the exception of 1094 * gup_fast later handled by the ptep_clear_flush and the VM 1095 * handled by the anon_vma lock + PG_lock. 1096 */ 1097 mmap_write_lock(mm); 1098 result = hugepage_vma_revalidate(mm, address, &vma); 1099 if (result) 1100 goto out_up_write; 1101 /* check if the pmd is still valid */ 1102 if (mm_find_pmd(mm, address) != pmd) 1103 goto out_up_write; 1104 1105 anon_vma_lock_write(vma->anon_vma); 1106 1107 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, NULL, mm, 1108 address, address + HPAGE_PMD_SIZE); 1109 mmu_notifier_invalidate_range_start(&range); 1110 1111 pte = pte_offset_map(pmd, address); 1112 pte_ptl = pte_lockptr(mm, pmd); 1113 1114 pmd_ptl = pmd_lock(mm, pmd); /* probably unnecessary */ 1115 /* 1116 * After this gup_fast can't run anymore. This also removes 1117 * any huge TLB entry from the CPU so we won't allow 1118 * huge and small TLB entries for the same virtual address 1119 * to avoid the risk of CPU bugs in that area. 1120 */ 1121 _pmd = pmdp_collapse_flush(vma, address, pmd); 1122 spin_unlock(pmd_ptl); 1123 mmu_notifier_invalidate_range_end(&range); 1124 1125 spin_lock(pte_ptl); 1126 isolated = __collapse_huge_page_isolate(vma, address, pte, 1127 &compound_pagelist); 1128 spin_unlock(pte_ptl); 1129 1130 if (unlikely(!isolated)) { 1131 pte_unmap(pte); 1132 spin_lock(pmd_ptl); 1133 BUG_ON(!pmd_none(*pmd)); 1134 /* 1135 * We can only use set_pmd_at when establishing 1136 * hugepmds and never for establishing regular pmds that 1137 * points to regular pagetables. Use pmd_populate for that 1138 */ 1139 pmd_populate(mm, pmd, pmd_pgtable(_pmd)); 1140 spin_unlock(pmd_ptl); 1141 anon_vma_unlock_write(vma->anon_vma); 1142 result = SCAN_FAIL; 1143 goto out_up_write; 1144 } 1145 1146 /* 1147 * All pages are isolated and locked so anon_vma rmap 1148 * can't run anymore. 1149 */ 1150 anon_vma_unlock_write(vma->anon_vma); 1151 1152 __collapse_huge_page_copy(pte, new_page, vma, address, pte_ptl, 1153 &compound_pagelist); 1154 pte_unmap(pte); 1155 /* 1156 * spin_lock() below is not the equivalent of smp_wmb(), but 1157 * the smp_wmb() inside __SetPageUptodate() can be reused to 1158 * avoid the copy_huge_page writes to become visible after 1159 * the set_pmd_at() write. 1160 */ 1161 __SetPageUptodate(new_page); 1162 pgtable = pmd_pgtable(_pmd); 1163 1164 _pmd = mk_huge_pmd(new_page, vma->vm_page_prot); 1165 _pmd = maybe_pmd_mkwrite(pmd_mkdirty(_pmd), vma); 1166 1167 spin_lock(pmd_ptl); 1168 BUG_ON(!pmd_none(*pmd)); 1169 page_add_new_anon_rmap(new_page, vma, address); 1170 lru_cache_add_inactive_or_unevictable(new_page, vma); 1171 pgtable_trans_huge_deposit(mm, pmd, pgtable); 1172 set_pmd_at(mm, address, pmd, _pmd); 1173 update_mmu_cache_pmd(vma, address, pmd); 1174 spin_unlock(pmd_ptl); 1175 1176 *hpage = NULL; 1177 1178 khugepaged_pages_collapsed++; 1179 result = SCAN_SUCCEED; 1180 out_up_write: 1181 mmap_write_unlock(mm); 1182 out_nolock: 1183 if (!IS_ERR_OR_NULL(*hpage)) 1184 mem_cgroup_uncharge(page_folio(*hpage)); 1185 trace_mm_collapse_huge_page(mm, isolated, result); 1186 return; 1187 } 1188 1189 static int khugepaged_scan_pmd(struct mm_struct *mm, 1190 struct vm_area_struct *vma, 1191 unsigned long address, 1192 struct page **hpage) 1193 { 1194 pmd_t *pmd; 1195 pte_t *pte, *_pte; 1196 int ret = 0, result = 0, referenced = 0; 1197 int none_or_zero = 0, shared = 0; 1198 struct page *page = NULL; 1199 unsigned long _address; 1200 spinlock_t *ptl; 1201 int node = NUMA_NO_NODE, unmapped = 0; 1202 bool writable = false; 1203 1204 VM_BUG_ON(address & ~HPAGE_PMD_MASK); 1205 1206 pmd = mm_find_pmd(mm, address); 1207 if (!pmd) { 1208 result = SCAN_PMD_NULL; 1209 goto out; 1210 } 1211 1212 memset(khugepaged_node_load, 0, sizeof(khugepaged_node_load)); 1213 pte = pte_offset_map_lock(mm, pmd, address, &ptl); 1214 for (_address = address, _pte = pte; _pte < pte + HPAGE_PMD_NR; 1215 _pte++, _address += PAGE_SIZE) { 1216 pte_t pteval = *_pte; 1217 if (is_swap_pte(pteval)) { 1218 if (++unmapped <= khugepaged_max_ptes_swap) { 1219 /* 1220 * Always be strict with uffd-wp 1221 * enabled swap entries. Please see 1222 * comment below for pte_uffd_wp(). 1223 */ 1224 if (pte_swp_uffd_wp(pteval)) { 1225 result = SCAN_PTE_UFFD_WP; 1226 goto out_unmap; 1227 } 1228 continue; 1229 } else { 1230 result = SCAN_EXCEED_SWAP_PTE; 1231 count_vm_event(THP_SCAN_EXCEED_SWAP_PTE); 1232 goto out_unmap; 1233 } 1234 } 1235 if (pte_none(pteval) || is_zero_pfn(pte_pfn(pteval))) { 1236 if (!userfaultfd_armed(vma) && 1237 ++none_or_zero <= khugepaged_max_ptes_none) { 1238 continue; 1239 } else { 1240 result = SCAN_EXCEED_NONE_PTE; 1241 count_vm_event(THP_SCAN_EXCEED_NONE_PTE); 1242 goto out_unmap; 1243 } 1244 } 1245 if (pte_uffd_wp(pteval)) { 1246 /* 1247 * Don't collapse the page if any of the small 1248 * PTEs are armed with uffd write protection. 1249 * Here we can also mark the new huge pmd as 1250 * write protected if any of the small ones is 1251 * marked but that could bring unknown 1252 * userfault messages that falls outside of 1253 * the registered range. So, just be simple. 1254 */ 1255 result = SCAN_PTE_UFFD_WP; 1256 goto out_unmap; 1257 } 1258 if (pte_write(pteval)) 1259 writable = true; 1260 1261 page = vm_normal_page(vma, _address, pteval); 1262 if (unlikely(!page) || unlikely(is_zone_device_page(page))) { 1263 result = SCAN_PAGE_NULL; 1264 goto out_unmap; 1265 } 1266 1267 if (page_mapcount(page) > 1 && 1268 ++shared > khugepaged_max_ptes_shared) { 1269 result = SCAN_EXCEED_SHARED_PTE; 1270 count_vm_event(THP_SCAN_EXCEED_SHARED_PTE); 1271 goto out_unmap; 1272 } 1273 1274 page = compound_head(page); 1275 1276 /* 1277 * Record which node the original page is from and save this 1278 * information to khugepaged_node_load[]. 1279 * Khugepaged will allocate hugepage from the node has the max 1280 * hit record. 1281 */ 1282 node = page_to_nid(page); 1283 if (khugepaged_scan_abort(node)) { 1284 result = SCAN_SCAN_ABORT; 1285 goto out_unmap; 1286 } 1287 khugepaged_node_load[node]++; 1288 if (!PageLRU(page)) { 1289 result = SCAN_PAGE_LRU; 1290 goto out_unmap; 1291 } 1292 if (PageLocked(page)) { 1293 result = SCAN_PAGE_LOCK; 1294 goto out_unmap; 1295 } 1296 if (!PageAnon(page)) { 1297 result = SCAN_PAGE_ANON; 1298 goto out_unmap; 1299 } 1300 1301 /* 1302 * Check if the page has any GUP (or other external) pins. 1303 * 1304 * Here the check is racy it may see total_mapcount > refcount 1305 * in some cases. 1306 * For example, one process with one forked child process. 1307 * The parent has the PMD split due to MADV_DONTNEED, then 1308 * the child is trying unmap the whole PMD, but khugepaged 1309 * may be scanning the parent between the child has 1310 * PageDoubleMap flag cleared and dec the mapcount. So 1311 * khugepaged may see total_mapcount > refcount. 1312 * 1313 * But such case is ephemeral we could always retry collapse 1314 * later. However it may report false positive if the page 1315 * has excessive GUP pins (i.e. 512). Anyway the same check 1316 * will be done again later the risk seems low. 1317 */ 1318 if (!is_refcount_suitable(page)) { 1319 result = SCAN_PAGE_COUNT; 1320 goto out_unmap; 1321 } 1322 if (pte_young(pteval) || 1323 page_is_young(page) || PageReferenced(page) || 1324 mmu_notifier_test_young(vma->vm_mm, address)) 1325 referenced++; 1326 } 1327 if (!writable) { 1328 result = SCAN_PAGE_RO; 1329 } else if (!referenced || (unmapped && referenced < HPAGE_PMD_NR/2)) { 1330 result = SCAN_LACK_REFERENCED_PAGE; 1331 } else { 1332 result = SCAN_SUCCEED; 1333 ret = 1; 1334 } 1335 out_unmap: 1336 pte_unmap_unlock(pte, ptl); 1337 if (ret) { 1338 node = khugepaged_find_target_node(); 1339 /* collapse_huge_page will return with the mmap_lock released */ 1340 collapse_huge_page(mm, address, hpage, node, 1341 referenced, unmapped); 1342 } 1343 out: 1344 trace_mm_khugepaged_scan_pmd(mm, page, writable, referenced, 1345 none_or_zero, result, unmapped); 1346 return ret; 1347 } 1348 1349 static void collect_mm_slot(struct mm_slot *mm_slot) 1350 { 1351 struct mm_struct *mm = mm_slot->mm; 1352 1353 lockdep_assert_held(&khugepaged_mm_lock); 1354 1355 if (khugepaged_test_exit(mm)) { 1356 /* free mm_slot */ 1357 hash_del(&mm_slot->hash); 1358 list_del(&mm_slot->mm_node); 1359 1360 /* 1361 * Not strictly needed because the mm exited already. 1362 * 1363 * clear_bit(MMF_VM_HUGEPAGE, &mm->flags); 1364 */ 1365 1366 /* khugepaged_mm_lock actually not necessary for the below */ 1367 free_mm_slot(mm_slot); 1368 mmdrop(mm); 1369 } 1370 } 1371 1372 #ifdef CONFIG_SHMEM 1373 /* 1374 * Notify khugepaged that given addr of the mm is pte-mapped THP. Then 1375 * khugepaged should try to collapse the page table. 1376 */ 1377 static void khugepaged_add_pte_mapped_thp(struct mm_struct *mm, 1378 unsigned long addr) 1379 { 1380 struct mm_slot *mm_slot; 1381 1382 VM_BUG_ON(addr & ~HPAGE_PMD_MASK); 1383 1384 spin_lock(&khugepaged_mm_lock); 1385 mm_slot = get_mm_slot(mm); 1386 if (likely(mm_slot && mm_slot->nr_pte_mapped_thp < MAX_PTE_MAPPED_THP)) 1387 mm_slot->pte_mapped_thp[mm_slot->nr_pte_mapped_thp++] = addr; 1388 spin_unlock(&khugepaged_mm_lock); 1389 } 1390 1391 static void collapse_and_free_pmd(struct mm_struct *mm, struct vm_area_struct *vma, 1392 unsigned long addr, pmd_t *pmdp) 1393 { 1394 spinlock_t *ptl; 1395 pmd_t pmd; 1396 1397 mmap_assert_write_locked(mm); 1398 ptl = pmd_lock(vma->vm_mm, pmdp); 1399 pmd = pmdp_collapse_flush(vma, addr, pmdp); 1400 spin_unlock(ptl); 1401 mm_dec_nr_ptes(mm); 1402 page_table_check_pte_clear_range(mm, addr, pmd); 1403 pte_free(mm, pmd_pgtable(pmd)); 1404 } 1405 1406 /** 1407 * collapse_pte_mapped_thp - Try to collapse a pte-mapped THP for mm at 1408 * address haddr. 1409 * 1410 * @mm: process address space where collapse happens 1411 * @addr: THP collapse address 1412 * 1413 * This function checks whether all the PTEs in the PMD are pointing to the 1414 * right THP. If so, retract the page table so the THP can refault in with 1415 * as pmd-mapped. 1416 */ 1417 void collapse_pte_mapped_thp(struct mm_struct *mm, unsigned long addr) 1418 { 1419 unsigned long haddr = addr & HPAGE_PMD_MASK; 1420 struct vm_area_struct *vma = find_vma(mm, haddr); 1421 struct page *hpage; 1422 pte_t *start_pte, *pte; 1423 pmd_t *pmd; 1424 spinlock_t *ptl; 1425 int count = 0; 1426 int i; 1427 1428 if (!vma || !vma->vm_file || 1429 !range_in_vma(vma, haddr, haddr + HPAGE_PMD_SIZE)) 1430 return; 1431 1432 /* 1433 * This vm_flags may not have VM_HUGEPAGE if the page was not 1434 * collapsed by this mm. But we can still collapse if the page is 1435 * the valid THP. Add extra VM_HUGEPAGE so hugepage_vma_check() 1436 * will not fail the vma for missing VM_HUGEPAGE 1437 */ 1438 if (!hugepage_vma_check(vma, vma->vm_flags | VM_HUGEPAGE)) 1439 return; 1440 1441 /* Keep pmd pgtable for uffd-wp; see comment in retract_page_tables() */ 1442 if (userfaultfd_wp(vma)) 1443 return; 1444 1445 hpage = find_lock_page(vma->vm_file->f_mapping, 1446 linear_page_index(vma, haddr)); 1447 if (!hpage) 1448 return; 1449 1450 if (!PageHead(hpage)) 1451 goto drop_hpage; 1452 1453 pmd = mm_find_pmd(mm, haddr); 1454 if (!pmd) 1455 goto drop_hpage; 1456 1457 start_pte = pte_offset_map_lock(mm, pmd, haddr, &ptl); 1458 1459 /* step 1: check all mapped PTEs are to the right huge page */ 1460 for (i = 0, addr = haddr, pte = start_pte; 1461 i < HPAGE_PMD_NR; i++, addr += PAGE_SIZE, pte++) { 1462 struct page *page; 1463 1464 /* empty pte, skip */ 1465 if (pte_none(*pte)) 1466 continue; 1467 1468 /* page swapped out, abort */ 1469 if (!pte_present(*pte)) 1470 goto abort; 1471 1472 page = vm_normal_page(vma, addr, *pte); 1473 if (WARN_ON_ONCE(page && is_zone_device_page(page))) 1474 page = NULL; 1475 /* 1476 * Note that uprobe, debugger, or MAP_PRIVATE may change the 1477 * page table, but the new page will not be a subpage of hpage. 1478 */ 1479 if (hpage + i != page) 1480 goto abort; 1481 count++; 1482 } 1483 1484 /* step 2: adjust rmap */ 1485 for (i = 0, addr = haddr, pte = start_pte; 1486 i < HPAGE_PMD_NR; i++, addr += PAGE_SIZE, pte++) { 1487 struct page *page; 1488 1489 if (pte_none(*pte)) 1490 continue; 1491 page = vm_normal_page(vma, addr, *pte); 1492 if (WARN_ON_ONCE(page && is_zone_device_page(page))) 1493 goto abort; 1494 page_remove_rmap(page, vma, false); 1495 } 1496 1497 pte_unmap_unlock(start_pte, ptl); 1498 1499 /* step 3: set proper refcount and mm_counters. */ 1500 if (count) { 1501 page_ref_sub(hpage, count); 1502 add_mm_counter(vma->vm_mm, mm_counter_file(hpage), -count); 1503 } 1504 1505 /* step 4: collapse pmd */ 1506 collapse_and_free_pmd(mm, vma, haddr, pmd); 1507 drop_hpage: 1508 unlock_page(hpage); 1509 put_page(hpage); 1510 return; 1511 1512 abort: 1513 pte_unmap_unlock(start_pte, ptl); 1514 goto drop_hpage; 1515 } 1516 1517 static void khugepaged_collapse_pte_mapped_thps(struct mm_slot *mm_slot) 1518 { 1519 struct mm_struct *mm = mm_slot->mm; 1520 int i; 1521 1522 if (likely(mm_slot->nr_pte_mapped_thp == 0)) 1523 return; 1524 1525 if (!mmap_write_trylock(mm)) 1526 return; 1527 1528 if (unlikely(khugepaged_test_exit(mm))) 1529 goto out; 1530 1531 for (i = 0; i < mm_slot->nr_pte_mapped_thp; i++) 1532 collapse_pte_mapped_thp(mm, mm_slot->pte_mapped_thp[i]); 1533 1534 out: 1535 mm_slot->nr_pte_mapped_thp = 0; 1536 mmap_write_unlock(mm); 1537 } 1538 1539 static void retract_page_tables(struct address_space *mapping, pgoff_t pgoff) 1540 { 1541 struct vm_area_struct *vma; 1542 struct mm_struct *mm; 1543 unsigned long addr; 1544 pmd_t *pmd; 1545 1546 i_mmap_lock_write(mapping); 1547 vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff, pgoff) { 1548 /* 1549 * Check vma->anon_vma to exclude MAP_PRIVATE mappings that 1550 * got written to. These VMAs are likely not worth investing 1551 * mmap_write_lock(mm) as PMD-mapping is likely to be split 1552 * later. 1553 * 1554 * Note that vma->anon_vma check is racy: it can be set up after 1555 * the check but before we took mmap_lock by the fault path. 1556 * But page lock would prevent establishing any new ptes of the 1557 * page, so we are safe. 1558 * 1559 * An alternative would be drop the check, but check that page 1560 * table is clear before calling pmdp_collapse_flush() under 1561 * ptl. It has higher chance to recover THP for the VMA, but 1562 * has higher cost too. 1563 */ 1564 if (vma->anon_vma) 1565 continue; 1566 addr = vma->vm_start + ((pgoff - vma->vm_pgoff) << PAGE_SHIFT); 1567 if (addr & ~HPAGE_PMD_MASK) 1568 continue; 1569 if (vma->vm_end < addr + HPAGE_PMD_SIZE) 1570 continue; 1571 mm = vma->vm_mm; 1572 pmd = mm_find_pmd(mm, addr); 1573 if (!pmd) 1574 continue; 1575 /* 1576 * We need exclusive mmap_lock to retract page table. 1577 * 1578 * We use trylock due to lock inversion: we need to acquire 1579 * mmap_lock while holding page lock. Fault path does it in 1580 * reverse order. Trylock is a way to avoid deadlock. 1581 */ 1582 if (mmap_write_trylock(mm)) { 1583 /* 1584 * When a vma is registered with uffd-wp, we can't 1585 * recycle the pmd pgtable because there can be pte 1586 * markers installed. Skip it only, so the rest mm/vma 1587 * can still have the same file mapped hugely, however 1588 * it'll always mapped in small page size for uffd-wp 1589 * registered ranges. 1590 */ 1591 if (!khugepaged_test_exit(mm) && !userfaultfd_wp(vma)) 1592 collapse_and_free_pmd(mm, vma, addr, pmd); 1593 mmap_write_unlock(mm); 1594 } else { 1595 /* Try again later */ 1596 khugepaged_add_pte_mapped_thp(mm, addr); 1597 } 1598 } 1599 i_mmap_unlock_write(mapping); 1600 } 1601 1602 /** 1603 * collapse_file - collapse filemap/tmpfs/shmem pages into huge one. 1604 * 1605 * @mm: process address space where collapse happens 1606 * @file: file that collapse on 1607 * @start: collapse start address 1608 * @hpage: new allocated huge page for collapse 1609 * @node: appointed node the new huge page allocate from 1610 * 1611 * Basic scheme is simple, details are more complex: 1612 * - allocate and lock a new huge page; 1613 * - scan page cache replacing old pages with the new one 1614 * + swap/gup in pages if necessary; 1615 * + fill in gaps; 1616 * + keep old pages around in case rollback is required; 1617 * - if replacing succeeds: 1618 * + copy data over; 1619 * + free old pages; 1620 * + unlock huge page; 1621 * - if replacing failed; 1622 * + put all pages back and unfreeze them; 1623 * + restore gaps in the page cache; 1624 * + unlock and free huge page; 1625 */ 1626 static void collapse_file(struct mm_struct *mm, 1627 struct file *file, pgoff_t start, 1628 struct page **hpage, int node) 1629 { 1630 struct address_space *mapping = file->f_mapping; 1631 gfp_t gfp; 1632 struct page *new_page; 1633 pgoff_t index, end = start + HPAGE_PMD_NR; 1634 LIST_HEAD(pagelist); 1635 XA_STATE_ORDER(xas, &mapping->i_pages, start, HPAGE_PMD_ORDER); 1636 int nr_none = 0, result = SCAN_SUCCEED; 1637 bool is_shmem = shmem_file(file); 1638 int nr; 1639 1640 VM_BUG_ON(!IS_ENABLED(CONFIG_READ_ONLY_THP_FOR_FS) && !is_shmem); 1641 VM_BUG_ON(start & (HPAGE_PMD_NR - 1)); 1642 1643 /* Only allocate from the target node */ 1644 gfp = alloc_hugepage_khugepaged_gfpmask() | __GFP_THISNODE; 1645 1646 new_page = khugepaged_alloc_page(hpage, gfp, node); 1647 if (!new_page) { 1648 result = SCAN_ALLOC_HUGE_PAGE_FAIL; 1649 goto out; 1650 } 1651 1652 if (unlikely(mem_cgroup_charge(page_folio(new_page), mm, gfp))) { 1653 result = SCAN_CGROUP_CHARGE_FAIL; 1654 goto out; 1655 } 1656 count_memcg_page_event(new_page, THP_COLLAPSE_ALLOC); 1657 1658 /* 1659 * Ensure we have slots for all the pages in the range. This is 1660 * almost certainly a no-op because most of the pages must be present 1661 */ 1662 do { 1663 xas_lock_irq(&xas); 1664 xas_create_range(&xas); 1665 if (!xas_error(&xas)) 1666 break; 1667 xas_unlock_irq(&xas); 1668 if (!xas_nomem(&xas, GFP_KERNEL)) { 1669 result = SCAN_FAIL; 1670 goto out; 1671 } 1672 } while (1); 1673 1674 __SetPageLocked(new_page); 1675 if (is_shmem) 1676 __SetPageSwapBacked(new_page); 1677 new_page->index = start; 1678 new_page->mapping = mapping; 1679 1680 /* 1681 * At this point the new_page is locked and not up-to-date. 1682 * It's safe to insert it into the page cache, because nobody would 1683 * be able to map it or use it in another way until we unlock it. 1684 */ 1685 1686 xas_set(&xas, start); 1687 for (index = start; index < end; index++) { 1688 struct page *page = xas_next(&xas); 1689 1690 VM_BUG_ON(index != xas.xa_index); 1691 if (is_shmem) { 1692 if (!page) { 1693 /* 1694 * Stop if extent has been truncated or 1695 * hole-punched, and is now completely 1696 * empty. 1697 */ 1698 if (index == start) { 1699 if (!xas_next_entry(&xas, end - 1)) { 1700 result = SCAN_TRUNCATED; 1701 goto xa_locked; 1702 } 1703 xas_set(&xas, index); 1704 } 1705 if (!shmem_charge(mapping->host, 1)) { 1706 result = SCAN_FAIL; 1707 goto xa_locked; 1708 } 1709 xas_store(&xas, new_page); 1710 nr_none++; 1711 continue; 1712 } 1713 1714 if (xa_is_value(page) || !PageUptodate(page)) { 1715 xas_unlock_irq(&xas); 1716 /* swap in or instantiate fallocated page */ 1717 if (shmem_getpage(mapping->host, index, &page, 1718 SGP_NOALLOC)) { 1719 result = SCAN_FAIL; 1720 goto xa_unlocked; 1721 } 1722 } else if (trylock_page(page)) { 1723 get_page(page); 1724 xas_unlock_irq(&xas); 1725 } else { 1726 result = SCAN_PAGE_LOCK; 1727 goto xa_locked; 1728 } 1729 } else { /* !is_shmem */ 1730 if (!page || xa_is_value(page)) { 1731 xas_unlock_irq(&xas); 1732 page_cache_sync_readahead(mapping, &file->f_ra, 1733 file, index, 1734 end - index); 1735 /* drain pagevecs to help isolate_lru_page() */ 1736 lru_add_drain(); 1737 page = find_lock_page(mapping, index); 1738 if (unlikely(page == NULL)) { 1739 result = SCAN_FAIL; 1740 goto xa_unlocked; 1741 } 1742 } else if (PageDirty(page)) { 1743 /* 1744 * khugepaged only works on read-only fd, 1745 * so this page is dirty because it hasn't 1746 * been flushed since first write. There 1747 * won't be new dirty pages. 1748 * 1749 * Trigger async flush here and hope the 1750 * writeback is done when khugepaged 1751 * revisits this page. 1752 * 1753 * This is a one-off situation. We are not 1754 * forcing writeback in loop. 1755 */ 1756 xas_unlock_irq(&xas); 1757 filemap_flush(mapping); 1758 result = SCAN_FAIL; 1759 goto xa_unlocked; 1760 } else if (PageWriteback(page)) { 1761 xas_unlock_irq(&xas); 1762 result = SCAN_FAIL; 1763 goto xa_unlocked; 1764 } else if (trylock_page(page)) { 1765 get_page(page); 1766 xas_unlock_irq(&xas); 1767 } else { 1768 result = SCAN_PAGE_LOCK; 1769 goto xa_locked; 1770 } 1771 } 1772 1773 /* 1774 * The page must be locked, so we can drop the i_pages lock 1775 * without racing with truncate. 1776 */ 1777 VM_BUG_ON_PAGE(!PageLocked(page), page); 1778 1779 /* make sure the page is up to date */ 1780 if (unlikely(!PageUptodate(page))) { 1781 result = SCAN_FAIL; 1782 goto out_unlock; 1783 } 1784 1785 /* 1786 * If file was truncated then extended, or hole-punched, before 1787 * we locked the first page, then a THP might be there already. 1788 */ 1789 if (PageTransCompound(page)) { 1790 result = SCAN_PAGE_COMPOUND; 1791 goto out_unlock; 1792 } 1793 1794 if (page_mapping(page) != mapping) { 1795 result = SCAN_TRUNCATED; 1796 goto out_unlock; 1797 } 1798 1799 if (!is_shmem && (PageDirty(page) || 1800 PageWriteback(page))) { 1801 /* 1802 * khugepaged only works on read-only fd, so this 1803 * page is dirty because it hasn't been flushed 1804 * since first write. 1805 */ 1806 result = SCAN_FAIL; 1807 goto out_unlock; 1808 } 1809 1810 if (isolate_lru_page(page)) { 1811 result = SCAN_DEL_PAGE_LRU; 1812 goto out_unlock; 1813 } 1814 1815 if (page_has_private(page) && 1816 !try_to_release_page(page, GFP_KERNEL)) { 1817 result = SCAN_PAGE_HAS_PRIVATE; 1818 putback_lru_page(page); 1819 goto out_unlock; 1820 } 1821 1822 if (page_mapped(page)) 1823 try_to_unmap(page_folio(page), 1824 TTU_IGNORE_MLOCK | TTU_BATCH_FLUSH); 1825 1826 xas_lock_irq(&xas); 1827 xas_set(&xas, index); 1828 1829 VM_BUG_ON_PAGE(page != xas_load(&xas), page); 1830 1831 /* 1832 * The page is expected to have page_count() == 3: 1833 * - we hold a pin on it; 1834 * - one reference from page cache; 1835 * - one from isolate_lru_page; 1836 */ 1837 if (!page_ref_freeze(page, 3)) { 1838 result = SCAN_PAGE_COUNT; 1839 xas_unlock_irq(&xas); 1840 putback_lru_page(page); 1841 goto out_unlock; 1842 } 1843 1844 /* 1845 * Add the page to the list to be able to undo the collapse if 1846 * something go wrong. 1847 */ 1848 list_add_tail(&page->lru, &pagelist); 1849 1850 /* Finally, replace with the new page. */ 1851 xas_store(&xas, new_page); 1852 continue; 1853 out_unlock: 1854 unlock_page(page); 1855 put_page(page); 1856 goto xa_unlocked; 1857 } 1858 nr = thp_nr_pages(new_page); 1859 1860 if (is_shmem) 1861 __mod_lruvec_page_state(new_page, NR_SHMEM_THPS, nr); 1862 else { 1863 __mod_lruvec_page_state(new_page, NR_FILE_THPS, nr); 1864 filemap_nr_thps_inc(mapping); 1865 /* 1866 * Paired with smp_mb() in do_dentry_open() to ensure 1867 * i_writecount is up to date and the update to nr_thps is 1868 * visible. Ensures the page cache will be truncated if the 1869 * file is opened writable. 1870 */ 1871 smp_mb(); 1872 if (inode_is_open_for_write(mapping->host)) { 1873 result = SCAN_FAIL; 1874 __mod_lruvec_page_state(new_page, NR_FILE_THPS, -nr); 1875 filemap_nr_thps_dec(mapping); 1876 goto xa_locked; 1877 } 1878 } 1879 1880 if (nr_none) { 1881 __mod_lruvec_page_state(new_page, NR_FILE_PAGES, nr_none); 1882 /* nr_none is always 0 for non-shmem. */ 1883 __mod_lruvec_page_state(new_page, NR_SHMEM, nr_none); 1884 } 1885 1886 /* Join all the small entries into a single multi-index entry */ 1887 xas_set_order(&xas, start, HPAGE_PMD_ORDER); 1888 xas_store(&xas, new_page); 1889 xa_locked: 1890 xas_unlock_irq(&xas); 1891 xa_unlocked: 1892 1893 /* 1894 * If collapse is successful, flush must be done now before copying. 1895 * If collapse is unsuccessful, does flush actually need to be done? 1896 * Do it anyway, to clear the state. 1897 */ 1898 try_to_unmap_flush(); 1899 1900 if (result == SCAN_SUCCEED) { 1901 struct page *page, *tmp; 1902 1903 /* 1904 * Replacing old pages with new one has succeeded, now we 1905 * need to copy the content and free the old pages. 1906 */ 1907 index = start; 1908 list_for_each_entry_safe(page, tmp, &pagelist, lru) { 1909 while (index < page->index) { 1910 clear_highpage(new_page + (index % HPAGE_PMD_NR)); 1911 index++; 1912 } 1913 copy_highpage(new_page + (page->index % HPAGE_PMD_NR), 1914 page); 1915 list_del(&page->lru); 1916 page->mapping = NULL; 1917 page_ref_unfreeze(page, 1); 1918 ClearPageActive(page); 1919 ClearPageUnevictable(page); 1920 unlock_page(page); 1921 put_page(page); 1922 index++; 1923 } 1924 while (index < end) { 1925 clear_highpage(new_page + (index % HPAGE_PMD_NR)); 1926 index++; 1927 } 1928 1929 SetPageUptodate(new_page); 1930 page_ref_add(new_page, HPAGE_PMD_NR - 1); 1931 if (is_shmem) 1932 set_page_dirty(new_page); 1933 lru_cache_add(new_page); 1934 1935 /* 1936 * Remove pte page tables, so we can re-fault the page as huge. 1937 */ 1938 retract_page_tables(mapping, start); 1939 *hpage = NULL; 1940 1941 khugepaged_pages_collapsed++; 1942 } else { 1943 struct page *page; 1944 1945 /* Something went wrong: roll back page cache changes */ 1946 xas_lock_irq(&xas); 1947 if (nr_none) { 1948 mapping->nrpages -= nr_none; 1949 shmem_uncharge(mapping->host, nr_none); 1950 } 1951 1952 xas_set(&xas, start); 1953 xas_for_each(&xas, page, end - 1) { 1954 page = list_first_entry_or_null(&pagelist, 1955 struct page, lru); 1956 if (!page || xas.xa_index < page->index) { 1957 if (!nr_none) 1958 break; 1959 nr_none--; 1960 /* Put holes back where they were */ 1961 xas_store(&xas, NULL); 1962 continue; 1963 } 1964 1965 VM_BUG_ON_PAGE(page->index != xas.xa_index, page); 1966 1967 /* Unfreeze the page. */ 1968 list_del(&page->lru); 1969 page_ref_unfreeze(page, 2); 1970 xas_store(&xas, page); 1971 xas_pause(&xas); 1972 xas_unlock_irq(&xas); 1973 unlock_page(page); 1974 putback_lru_page(page); 1975 xas_lock_irq(&xas); 1976 } 1977 VM_BUG_ON(nr_none); 1978 xas_unlock_irq(&xas); 1979 1980 new_page->mapping = NULL; 1981 } 1982 1983 unlock_page(new_page); 1984 out: 1985 VM_BUG_ON(!list_empty(&pagelist)); 1986 if (!IS_ERR_OR_NULL(*hpage)) 1987 mem_cgroup_uncharge(page_folio(*hpage)); 1988 /* TODO: tracepoints */ 1989 } 1990 1991 static void khugepaged_scan_file(struct mm_struct *mm, 1992 struct file *file, pgoff_t start, struct page **hpage) 1993 { 1994 struct page *page = NULL; 1995 struct address_space *mapping = file->f_mapping; 1996 XA_STATE(xas, &mapping->i_pages, start); 1997 int present, swap; 1998 int node = NUMA_NO_NODE; 1999 int result = SCAN_SUCCEED; 2000 2001 present = 0; 2002 swap = 0; 2003 memset(khugepaged_node_load, 0, sizeof(khugepaged_node_load)); 2004 rcu_read_lock(); 2005 xas_for_each(&xas, page, start + HPAGE_PMD_NR - 1) { 2006 if (xas_retry(&xas, page)) 2007 continue; 2008 2009 if (xa_is_value(page)) { 2010 if (++swap > khugepaged_max_ptes_swap) { 2011 result = SCAN_EXCEED_SWAP_PTE; 2012 count_vm_event(THP_SCAN_EXCEED_SWAP_PTE); 2013 break; 2014 } 2015 continue; 2016 } 2017 2018 /* 2019 * XXX: khugepaged should compact smaller compound pages 2020 * into a PMD sized page 2021 */ 2022 if (PageTransCompound(page)) { 2023 result = SCAN_PAGE_COMPOUND; 2024 break; 2025 } 2026 2027 node = page_to_nid(page); 2028 if (khugepaged_scan_abort(node)) { 2029 result = SCAN_SCAN_ABORT; 2030 break; 2031 } 2032 khugepaged_node_load[node]++; 2033 2034 if (!PageLRU(page)) { 2035 result = SCAN_PAGE_LRU; 2036 break; 2037 } 2038 2039 if (page_count(page) != 2040 1 + page_mapcount(page) + page_has_private(page)) { 2041 result = SCAN_PAGE_COUNT; 2042 break; 2043 } 2044 2045 /* 2046 * We probably should check if the page is referenced here, but 2047 * nobody would transfer pte_young() to PageReferenced() for us. 2048 * And rmap walk here is just too costly... 2049 */ 2050 2051 present++; 2052 2053 if (need_resched()) { 2054 xas_pause(&xas); 2055 cond_resched_rcu(); 2056 } 2057 } 2058 rcu_read_unlock(); 2059 2060 if (result == SCAN_SUCCEED) { 2061 if (present < HPAGE_PMD_NR - khugepaged_max_ptes_none) { 2062 result = SCAN_EXCEED_NONE_PTE; 2063 count_vm_event(THP_SCAN_EXCEED_NONE_PTE); 2064 } else { 2065 node = khugepaged_find_target_node(); 2066 collapse_file(mm, file, start, hpage, node); 2067 } 2068 } 2069 2070 /* TODO: tracepoints */ 2071 } 2072 #else 2073 static void khugepaged_scan_file(struct mm_struct *mm, 2074 struct file *file, pgoff_t start, struct page **hpage) 2075 { 2076 BUILD_BUG(); 2077 } 2078 2079 static void khugepaged_collapse_pte_mapped_thps(struct mm_slot *mm_slot) 2080 { 2081 } 2082 #endif 2083 2084 static unsigned int khugepaged_scan_mm_slot(unsigned int pages, 2085 struct page **hpage) 2086 __releases(&khugepaged_mm_lock) 2087 __acquires(&khugepaged_mm_lock) 2088 { 2089 struct mm_slot *mm_slot; 2090 struct mm_struct *mm; 2091 struct vm_area_struct *vma; 2092 int progress = 0; 2093 2094 VM_BUG_ON(!pages); 2095 lockdep_assert_held(&khugepaged_mm_lock); 2096 2097 if (khugepaged_scan.mm_slot) 2098 mm_slot = khugepaged_scan.mm_slot; 2099 else { 2100 mm_slot = list_entry(khugepaged_scan.mm_head.next, 2101 struct mm_slot, mm_node); 2102 khugepaged_scan.address = 0; 2103 khugepaged_scan.mm_slot = mm_slot; 2104 } 2105 spin_unlock(&khugepaged_mm_lock); 2106 khugepaged_collapse_pte_mapped_thps(mm_slot); 2107 2108 mm = mm_slot->mm; 2109 /* 2110 * Don't wait for semaphore (to avoid long wait times). Just move to 2111 * the next mm on the list. 2112 */ 2113 vma = NULL; 2114 if (unlikely(!mmap_read_trylock(mm))) 2115 goto breakouterloop_mmap_lock; 2116 if (likely(!khugepaged_test_exit(mm))) 2117 vma = find_vma(mm, khugepaged_scan.address); 2118 2119 progress++; 2120 for (; vma; vma = vma->vm_next) { 2121 unsigned long hstart, hend; 2122 2123 cond_resched(); 2124 if (unlikely(khugepaged_test_exit(mm))) { 2125 progress++; 2126 break; 2127 } 2128 if (!hugepage_vma_check(vma, vma->vm_flags)) { 2129 skip: 2130 progress++; 2131 continue; 2132 } 2133 hstart = round_up(vma->vm_start, HPAGE_PMD_SIZE); 2134 hend = round_down(vma->vm_end, HPAGE_PMD_SIZE); 2135 if (khugepaged_scan.address > hend) 2136 goto skip; 2137 if (khugepaged_scan.address < hstart) 2138 khugepaged_scan.address = hstart; 2139 VM_BUG_ON(khugepaged_scan.address & ~HPAGE_PMD_MASK); 2140 2141 while (khugepaged_scan.address < hend) { 2142 int ret; 2143 cond_resched(); 2144 if (unlikely(khugepaged_test_exit(mm))) 2145 goto breakouterloop; 2146 2147 VM_BUG_ON(khugepaged_scan.address < hstart || 2148 khugepaged_scan.address + HPAGE_PMD_SIZE > 2149 hend); 2150 if (IS_ENABLED(CONFIG_SHMEM) && vma->vm_file) { 2151 struct file *file = get_file(vma->vm_file); 2152 pgoff_t pgoff = linear_page_index(vma, 2153 khugepaged_scan.address); 2154 2155 mmap_read_unlock(mm); 2156 ret = 1; 2157 khugepaged_scan_file(mm, file, pgoff, hpage); 2158 fput(file); 2159 } else { 2160 ret = khugepaged_scan_pmd(mm, vma, 2161 khugepaged_scan.address, 2162 hpage); 2163 } 2164 /* move to next address */ 2165 khugepaged_scan.address += HPAGE_PMD_SIZE; 2166 progress += HPAGE_PMD_NR; 2167 if (ret) 2168 /* we released mmap_lock so break loop */ 2169 goto breakouterloop_mmap_lock; 2170 if (progress >= pages) 2171 goto breakouterloop; 2172 } 2173 } 2174 breakouterloop: 2175 mmap_read_unlock(mm); /* exit_mmap will destroy ptes after this */ 2176 breakouterloop_mmap_lock: 2177 2178 spin_lock(&khugepaged_mm_lock); 2179 VM_BUG_ON(khugepaged_scan.mm_slot != mm_slot); 2180 /* 2181 * Release the current mm_slot if this mm is about to die, or 2182 * if we scanned all vmas of this mm. 2183 */ 2184 if (khugepaged_test_exit(mm) || !vma) { 2185 /* 2186 * Make sure that if mm_users is reaching zero while 2187 * khugepaged runs here, khugepaged_exit will find 2188 * mm_slot not pointing to the exiting mm. 2189 */ 2190 if (mm_slot->mm_node.next != &khugepaged_scan.mm_head) { 2191 khugepaged_scan.mm_slot = list_entry( 2192 mm_slot->mm_node.next, 2193 struct mm_slot, mm_node); 2194 khugepaged_scan.address = 0; 2195 } else { 2196 khugepaged_scan.mm_slot = NULL; 2197 khugepaged_full_scans++; 2198 } 2199 2200 collect_mm_slot(mm_slot); 2201 } 2202 2203 return progress; 2204 } 2205 2206 static int khugepaged_has_work(void) 2207 { 2208 return !list_empty(&khugepaged_scan.mm_head) && 2209 khugepaged_enabled(); 2210 } 2211 2212 static int khugepaged_wait_event(void) 2213 { 2214 return !list_empty(&khugepaged_scan.mm_head) || 2215 kthread_should_stop(); 2216 } 2217 2218 static void khugepaged_do_scan(void) 2219 { 2220 struct page *hpage = NULL; 2221 unsigned int progress = 0, pass_through_head = 0; 2222 unsigned int pages = READ_ONCE(khugepaged_pages_to_scan); 2223 bool wait = true; 2224 2225 lru_add_drain_all(); 2226 2227 while (progress < pages) { 2228 if (!khugepaged_prealloc_page(&hpage, &wait)) 2229 break; 2230 2231 cond_resched(); 2232 2233 if (unlikely(kthread_should_stop() || try_to_freeze())) 2234 break; 2235 2236 spin_lock(&khugepaged_mm_lock); 2237 if (!khugepaged_scan.mm_slot) 2238 pass_through_head++; 2239 if (khugepaged_has_work() && 2240 pass_through_head < 2) 2241 progress += khugepaged_scan_mm_slot(pages - progress, 2242 &hpage); 2243 else 2244 progress = pages; 2245 spin_unlock(&khugepaged_mm_lock); 2246 } 2247 2248 if (!IS_ERR_OR_NULL(hpage)) 2249 put_page(hpage); 2250 } 2251 2252 static bool khugepaged_should_wakeup(void) 2253 { 2254 return kthread_should_stop() || 2255 time_after_eq(jiffies, khugepaged_sleep_expire); 2256 } 2257 2258 static void khugepaged_wait_work(void) 2259 { 2260 if (khugepaged_has_work()) { 2261 const unsigned long scan_sleep_jiffies = 2262 msecs_to_jiffies(khugepaged_scan_sleep_millisecs); 2263 2264 if (!scan_sleep_jiffies) 2265 return; 2266 2267 khugepaged_sleep_expire = jiffies + scan_sleep_jiffies; 2268 wait_event_freezable_timeout(khugepaged_wait, 2269 khugepaged_should_wakeup(), 2270 scan_sleep_jiffies); 2271 return; 2272 } 2273 2274 if (khugepaged_enabled()) 2275 wait_event_freezable(khugepaged_wait, khugepaged_wait_event()); 2276 } 2277 2278 static int khugepaged(void *none) 2279 { 2280 struct mm_slot *mm_slot; 2281 2282 set_freezable(); 2283 set_user_nice(current, MAX_NICE); 2284 2285 while (!kthread_should_stop()) { 2286 khugepaged_do_scan(); 2287 khugepaged_wait_work(); 2288 } 2289 2290 spin_lock(&khugepaged_mm_lock); 2291 mm_slot = khugepaged_scan.mm_slot; 2292 khugepaged_scan.mm_slot = NULL; 2293 if (mm_slot) 2294 collect_mm_slot(mm_slot); 2295 spin_unlock(&khugepaged_mm_lock); 2296 return 0; 2297 } 2298 2299 static void set_recommended_min_free_kbytes(void) 2300 { 2301 struct zone *zone; 2302 int nr_zones = 0; 2303 unsigned long recommended_min; 2304 2305 if (!khugepaged_enabled()) { 2306 calculate_min_free_kbytes(); 2307 goto update_wmarks; 2308 } 2309 2310 for_each_populated_zone(zone) { 2311 /* 2312 * We don't need to worry about fragmentation of 2313 * ZONE_MOVABLE since it only has movable pages. 2314 */ 2315 if (zone_idx(zone) > gfp_zone(GFP_USER)) 2316 continue; 2317 2318 nr_zones++; 2319 } 2320 2321 /* Ensure 2 pageblocks are free to assist fragmentation avoidance */ 2322 recommended_min = pageblock_nr_pages * nr_zones * 2; 2323 2324 /* 2325 * Make sure that on average at least two pageblocks are almost free 2326 * of another type, one for a migratetype to fall back to and a 2327 * second to avoid subsequent fallbacks of other types There are 3 2328 * MIGRATE_TYPES we care about. 2329 */ 2330 recommended_min += pageblock_nr_pages * nr_zones * 2331 MIGRATE_PCPTYPES * MIGRATE_PCPTYPES; 2332 2333 /* don't ever allow to reserve more than 5% of the lowmem */ 2334 recommended_min = min(recommended_min, 2335 (unsigned long) nr_free_buffer_pages() / 20); 2336 recommended_min <<= (PAGE_SHIFT-10); 2337 2338 if (recommended_min > min_free_kbytes) { 2339 if (user_min_free_kbytes >= 0) 2340 pr_info("raising min_free_kbytes from %d to %lu to help transparent hugepage allocations\n", 2341 min_free_kbytes, recommended_min); 2342 2343 min_free_kbytes = recommended_min; 2344 } 2345 2346 update_wmarks: 2347 setup_per_zone_wmarks(); 2348 } 2349 2350 int start_stop_khugepaged(void) 2351 { 2352 int err = 0; 2353 2354 mutex_lock(&khugepaged_mutex); 2355 if (khugepaged_enabled()) { 2356 if (!khugepaged_thread) 2357 khugepaged_thread = kthread_run(khugepaged, NULL, 2358 "khugepaged"); 2359 if (IS_ERR(khugepaged_thread)) { 2360 pr_err("khugepaged: kthread_run(khugepaged) failed\n"); 2361 err = PTR_ERR(khugepaged_thread); 2362 khugepaged_thread = NULL; 2363 goto fail; 2364 } 2365 2366 if (!list_empty(&khugepaged_scan.mm_head)) 2367 wake_up_interruptible(&khugepaged_wait); 2368 } else if (khugepaged_thread) { 2369 kthread_stop(khugepaged_thread); 2370 khugepaged_thread = NULL; 2371 } 2372 set_recommended_min_free_kbytes(); 2373 fail: 2374 mutex_unlock(&khugepaged_mutex); 2375 return err; 2376 } 2377 2378 void khugepaged_min_free_kbytes_update(void) 2379 { 2380 mutex_lock(&khugepaged_mutex); 2381 if (khugepaged_enabled() && khugepaged_thread) 2382 set_recommended_min_free_kbytes(); 2383 mutex_unlock(&khugepaged_mutex); 2384 } 2385