1 // SPDX-License-Identifier: GPL-2.0 2 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 3 4 #include <linux/mm.h> 5 #include <linux/sched.h> 6 #include <linux/sched/mm.h> 7 #include <linux/sched/coredump.h> 8 #include <linux/mmu_notifier.h> 9 #include <linux/rmap.h> 10 #include <linux/swap.h> 11 #include <linux/mm_inline.h> 12 #include <linux/kthread.h> 13 #include <linux/khugepaged.h> 14 #include <linux/freezer.h> 15 #include <linux/mman.h> 16 #include <linux/hashtable.h> 17 #include <linux/userfaultfd_k.h> 18 #include <linux/page_idle.h> 19 #include <linux/page_table_check.h> 20 #include <linux/swapops.h> 21 #include <linux/shmem_fs.h> 22 23 #include <asm/tlb.h> 24 #include <asm/pgalloc.h> 25 #include "internal.h" 26 #include "mm_slot.h" 27 28 enum scan_result { 29 SCAN_FAIL, 30 SCAN_SUCCEED, 31 SCAN_PMD_NULL, 32 SCAN_PMD_NONE, 33 SCAN_PMD_MAPPED, 34 SCAN_EXCEED_NONE_PTE, 35 SCAN_EXCEED_SWAP_PTE, 36 SCAN_EXCEED_SHARED_PTE, 37 SCAN_PTE_NON_PRESENT, 38 SCAN_PTE_UFFD_WP, 39 SCAN_PTE_MAPPED_HUGEPAGE, 40 SCAN_PAGE_RO, 41 SCAN_LACK_REFERENCED_PAGE, 42 SCAN_PAGE_NULL, 43 SCAN_SCAN_ABORT, 44 SCAN_PAGE_COUNT, 45 SCAN_PAGE_LRU, 46 SCAN_PAGE_LOCK, 47 SCAN_PAGE_ANON, 48 SCAN_PAGE_COMPOUND, 49 SCAN_ANY_PROCESS, 50 SCAN_VMA_NULL, 51 SCAN_VMA_CHECK, 52 SCAN_ADDRESS_RANGE, 53 SCAN_DEL_PAGE_LRU, 54 SCAN_ALLOC_HUGE_PAGE_FAIL, 55 SCAN_CGROUP_CHARGE_FAIL, 56 SCAN_TRUNCATED, 57 SCAN_PAGE_HAS_PRIVATE, 58 }; 59 60 #define CREATE_TRACE_POINTS 61 #include <trace/events/huge_memory.h> 62 63 static struct task_struct *khugepaged_thread __read_mostly; 64 static DEFINE_MUTEX(khugepaged_mutex); 65 66 /* default scan 8*512 pte (or vmas) every 30 second */ 67 static unsigned int khugepaged_pages_to_scan __read_mostly; 68 static unsigned int khugepaged_pages_collapsed; 69 static unsigned int khugepaged_full_scans; 70 static unsigned int khugepaged_scan_sleep_millisecs __read_mostly = 10000; 71 /* during fragmentation poll the hugepage allocator once every minute */ 72 static unsigned int khugepaged_alloc_sleep_millisecs __read_mostly = 60000; 73 static unsigned long khugepaged_sleep_expire; 74 static DEFINE_SPINLOCK(khugepaged_mm_lock); 75 static DECLARE_WAIT_QUEUE_HEAD(khugepaged_wait); 76 /* 77 * default collapse hugepages if there is at least one pte mapped like 78 * it would have happened if the vma was large enough during page 79 * fault. 80 * 81 * Note that these are only respected if collapse was initiated by khugepaged. 82 */ 83 static unsigned int khugepaged_max_ptes_none __read_mostly; 84 static unsigned int khugepaged_max_ptes_swap __read_mostly; 85 static unsigned int khugepaged_max_ptes_shared __read_mostly; 86 87 #define MM_SLOTS_HASH_BITS 10 88 static __read_mostly DEFINE_HASHTABLE(mm_slots_hash, MM_SLOTS_HASH_BITS); 89 90 static struct kmem_cache *mm_slot_cache __read_mostly; 91 92 #define MAX_PTE_MAPPED_THP 8 93 94 struct collapse_control { 95 bool is_khugepaged; 96 97 /* Num pages scanned per node */ 98 u32 node_load[MAX_NUMNODES]; 99 100 /* nodemask for allocation fallback */ 101 nodemask_t alloc_nmask; 102 }; 103 104 /** 105 * struct khugepaged_mm_slot - khugepaged information per mm that is being scanned 106 * @slot: hash lookup from mm to mm_slot 107 * @nr_pte_mapped_thp: number of pte mapped THP 108 * @pte_mapped_thp: address array corresponding pte mapped THP 109 */ 110 struct khugepaged_mm_slot { 111 struct mm_slot slot; 112 113 /* pte-mapped THP in this mm */ 114 int nr_pte_mapped_thp; 115 unsigned long pte_mapped_thp[MAX_PTE_MAPPED_THP]; 116 }; 117 118 /** 119 * struct khugepaged_scan - cursor for scanning 120 * @mm_head: the head of the mm list to scan 121 * @mm_slot: the current mm_slot we are scanning 122 * @address: the next address inside that to be scanned 123 * 124 * There is only the one khugepaged_scan instance of this cursor structure. 125 */ 126 struct khugepaged_scan { 127 struct list_head mm_head; 128 struct khugepaged_mm_slot *mm_slot; 129 unsigned long address; 130 }; 131 132 static struct khugepaged_scan khugepaged_scan = { 133 .mm_head = LIST_HEAD_INIT(khugepaged_scan.mm_head), 134 }; 135 136 #ifdef CONFIG_SYSFS 137 static ssize_t scan_sleep_millisecs_show(struct kobject *kobj, 138 struct kobj_attribute *attr, 139 char *buf) 140 { 141 return sysfs_emit(buf, "%u\n", khugepaged_scan_sleep_millisecs); 142 } 143 144 static ssize_t scan_sleep_millisecs_store(struct kobject *kobj, 145 struct kobj_attribute *attr, 146 const char *buf, size_t count) 147 { 148 unsigned int msecs; 149 int err; 150 151 err = kstrtouint(buf, 10, &msecs); 152 if (err) 153 return -EINVAL; 154 155 khugepaged_scan_sleep_millisecs = msecs; 156 khugepaged_sleep_expire = 0; 157 wake_up_interruptible(&khugepaged_wait); 158 159 return count; 160 } 161 static struct kobj_attribute scan_sleep_millisecs_attr = 162 __ATTR_RW(scan_sleep_millisecs); 163 164 static ssize_t alloc_sleep_millisecs_show(struct kobject *kobj, 165 struct kobj_attribute *attr, 166 char *buf) 167 { 168 return sysfs_emit(buf, "%u\n", khugepaged_alloc_sleep_millisecs); 169 } 170 171 static ssize_t alloc_sleep_millisecs_store(struct kobject *kobj, 172 struct kobj_attribute *attr, 173 const char *buf, size_t count) 174 { 175 unsigned int msecs; 176 int err; 177 178 err = kstrtouint(buf, 10, &msecs); 179 if (err) 180 return -EINVAL; 181 182 khugepaged_alloc_sleep_millisecs = msecs; 183 khugepaged_sleep_expire = 0; 184 wake_up_interruptible(&khugepaged_wait); 185 186 return count; 187 } 188 static struct kobj_attribute alloc_sleep_millisecs_attr = 189 __ATTR_RW(alloc_sleep_millisecs); 190 191 static ssize_t pages_to_scan_show(struct kobject *kobj, 192 struct kobj_attribute *attr, 193 char *buf) 194 { 195 return sysfs_emit(buf, "%u\n", khugepaged_pages_to_scan); 196 } 197 static ssize_t pages_to_scan_store(struct kobject *kobj, 198 struct kobj_attribute *attr, 199 const char *buf, size_t count) 200 { 201 unsigned int pages; 202 int err; 203 204 err = kstrtouint(buf, 10, &pages); 205 if (err || !pages) 206 return -EINVAL; 207 208 khugepaged_pages_to_scan = pages; 209 210 return count; 211 } 212 static struct kobj_attribute pages_to_scan_attr = 213 __ATTR_RW(pages_to_scan); 214 215 static ssize_t pages_collapsed_show(struct kobject *kobj, 216 struct kobj_attribute *attr, 217 char *buf) 218 { 219 return sysfs_emit(buf, "%u\n", khugepaged_pages_collapsed); 220 } 221 static struct kobj_attribute pages_collapsed_attr = 222 __ATTR_RO(pages_collapsed); 223 224 static ssize_t full_scans_show(struct kobject *kobj, 225 struct kobj_attribute *attr, 226 char *buf) 227 { 228 return sysfs_emit(buf, "%u\n", khugepaged_full_scans); 229 } 230 static struct kobj_attribute full_scans_attr = 231 __ATTR_RO(full_scans); 232 233 static ssize_t defrag_show(struct kobject *kobj, 234 struct kobj_attribute *attr, char *buf) 235 { 236 return single_hugepage_flag_show(kobj, attr, buf, 237 TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG); 238 } 239 static ssize_t defrag_store(struct kobject *kobj, 240 struct kobj_attribute *attr, 241 const char *buf, size_t count) 242 { 243 return single_hugepage_flag_store(kobj, attr, buf, count, 244 TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG); 245 } 246 static struct kobj_attribute khugepaged_defrag_attr = 247 __ATTR_RW(defrag); 248 249 /* 250 * max_ptes_none controls if khugepaged should collapse hugepages over 251 * any unmapped ptes in turn potentially increasing the memory 252 * footprint of the vmas. When max_ptes_none is 0 khugepaged will not 253 * reduce the available free memory in the system as it 254 * runs. Increasing max_ptes_none will instead potentially reduce the 255 * free memory in the system during the khugepaged scan. 256 */ 257 static ssize_t max_ptes_none_show(struct kobject *kobj, 258 struct kobj_attribute *attr, 259 char *buf) 260 { 261 return sysfs_emit(buf, "%u\n", khugepaged_max_ptes_none); 262 } 263 static ssize_t max_ptes_none_store(struct kobject *kobj, 264 struct kobj_attribute *attr, 265 const char *buf, size_t count) 266 { 267 int err; 268 unsigned long max_ptes_none; 269 270 err = kstrtoul(buf, 10, &max_ptes_none); 271 if (err || max_ptes_none > HPAGE_PMD_NR - 1) 272 return -EINVAL; 273 274 khugepaged_max_ptes_none = max_ptes_none; 275 276 return count; 277 } 278 static struct kobj_attribute khugepaged_max_ptes_none_attr = 279 __ATTR_RW(max_ptes_none); 280 281 static ssize_t max_ptes_swap_show(struct kobject *kobj, 282 struct kobj_attribute *attr, 283 char *buf) 284 { 285 return sysfs_emit(buf, "%u\n", khugepaged_max_ptes_swap); 286 } 287 288 static ssize_t max_ptes_swap_store(struct kobject *kobj, 289 struct kobj_attribute *attr, 290 const char *buf, size_t count) 291 { 292 int err; 293 unsigned long max_ptes_swap; 294 295 err = kstrtoul(buf, 10, &max_ptes_swap); 296 if (err || max_ptes_swap > HPAGE_PMD_NR - 1) 297 return -EINVAL; 298 299 khugepaged_max_ptes_swap = max_ptes_swap; 300 301 return count; 302 } 303 304 static struct kobj_attribute khugepaged_max_ptes_swap_attr = 305 __ATTR_RW(max_ptes_swap); 306 307 static ssize_t max_ptes_shared_show(struct kobject *kobj, 308 struct kobj_attribute *attr, 309 char *buf) 310 { 311 return sysfs_emit(buf, "%u\n", khugepaged_max_ptes_shared); 312 } 313 314 static ssize_t max_ptes_shared_store(struct kobject *kobj, 315 struct kobj_attribute *attr, 316 const char *buf, size_t count) 317 { 318 int err; 319 unsigned long max_ptes_shared; 320 321 err = kstrtoul(buf, 10, &max_ptes_shared); 322 if (err || max_ptes_shared > HPAGE_PMD_NR - 1) 323 return -EINVAL; 324 325 khugepaged_max_ptes_shared = max_ptes_shared; 326 327 return count; 328 } 329 330 static struct kobj_attribute khugepaged_max_ptes_shared_attr = 331 __ATTR_RW(max_ptes_shared); 332 333 static struct attribute *khugepaged_attr[] = { 334 &khugepaged_defrag_attr.attr, 335 &khugepaged_max_ptes_none_attr.attr, 336 &khugepaged_max_ptes_swap_attr.attr, 337 &khugepaged_max_ptes_shared_attr.attr, 338 &pages_to_scan_attr.attr, 339 &pages_collapsed_attr.attr, 340 &full_scans_attr.attr, 341 &scan_sleep_millisecs_attr.attr, 342 &alloc_sleep_millisecs_attr.attr, 343 NULL, 344 }; 345 346 struct attribute_group khugepaged_attr_group = { 347 .attrs = khugepaged_attr, 348 .name = "khugepaged", 349 }; 350 #endif /* CONFIG_SYSFS */ 351 352 int hugepage_madvise(struct vm_area_struct *vma, 353 unsigned long *vm_flags, int advice) 354 { 355 switch (advice) { 356 case MADV_HUGEPAGE: 357 #ifdef CONFIG_S390 358 /* 359 * qemu blindly sets MADV_HUGEPAGE on all allocations, but s390 360 * can't handle this properly after s390_enable_sie, so we simply 361 * ignore the madvise to prevent qemu from causing a SIGSEGV. 362 */ 363 if (mm_has_pgste(vma->vm_mm)) 364 return 0; 365 #endif 366 *vm_flags &= ~VM_NOHUGEPAGE; 367 *vm_flags |= VM_HUGEPAGE; 368 /* 369 * If the vma become good for khugepaged to scan, 370 * register it here without waiting a page fault that 371 * may not happen any time soon. 372 */ 373 khugepaged_enter_vma(vma, *vm_flags); 374 break; 375 case MADV_NOHUGEPAGE: 376 *vm_flags &= ~VM_HUGEPAGE; 377 *vm_flags |= VM_NOHUGEPAGE; 378 /* 379 * Setting VM_NOHUGEPAGE will prevent khugepaged from scanning 380 * this vma even if we leave the mm registered in khugepaged if 381 * it got registered before VM_NOHUGEPAGE was set. 382 */ 383 break; 384 } 385 386 return 0; 387 } 388 389 int __init khugepaged_init(void) 390 { 391 mm_slot_cache = kmem_cache_create("khugepaged_mm_slot", 392 sizeof(struct khugepaged_mm_slot), 393 __alignof__(struct khugepaged_mm_slot), 394 0, NULL); 395 if (!mm_slot_cache) 396 return -ENOMEM; 397 398 khugepaged_pages_to_scan = HPAGE_PMD_NR * 8; 399 khugepaged_max_ptes_none = HPAGE_PMD_NR - 1; 400 khugepaged_max_ptes_swap = HPAGE_PMD_NR / 8; 401 khugepaged_max_ptes_shared = HPAGE_PMD_NR / 2; 402 403 return 0; 404 } 405 406 void __init khugepaged_destroy(void) 407 { 408 kmem_cache_destroy(mm_slot_cache); 409 } 410 411 static inline int hpage_collapse_test_exit(struct mm_struct *mm) 412 { 413 return atomic_read(&mm->mm_users) == 0; 414 } 415 416 void __khugepaged_enter(struct mm_struct *mm) 417 { 418 struct khugepaged_mm_slot *mm_slot; 419 struct mm_slot *slot; 420 int wakeup; 421 422 mm_slot = mm_slot_alloc(mm_slot_cache); 423 if (!mm_slot) 424 return; 425 426 slot = &mm_slot->slot; 427 428 /* __khugepaged_exit() must not run from under us */ 429 VM_BUG_ON_MM(hpage_collapse_test_exit(mm), mm); 430 if (unlikely(test_and_set_bit(MMF_VM_HUGEPAGE, &mm->flags))) { 431 mm_slot_free(mm_slot_cache, mm_slot); 432 return; 433 } 434 435 spin_lock(&khugepaged_mm_lock); 436 mm_slot_insert(mm_slots_hash, mm, slot); 437 /* 438 * Insert just behind the scanning cursor, to let the area settle 439 * down a little. 440 */ 441 wakeup = list_empty(&khugepaged_scan.mm_head); 442 list_add_tail(&slot->mm_node, &khugepaged_scan.mm_head); 443 spin_unlock(&khugepaged_mm_lock); 444 445 mmgrab(mm); 446 if (wakeup) 447 wake_up_interruptible(&khugepaged_wait); 448 } 449 450 void khugepaged_enter_vma(struct vm_area_struct *vma, 451 unsigned long vm_flags) 452 { 453 if (!test_bit(MMF_VM_HUGEPAGE, &vma->vm_mm->flags) && 454 hugepage_flags_enabled()) { 455 if (hugepage_vma_check(vma, vm_flags, false, false, true)) 456 __khugepaged_enter(vma->vm_mm); 457 } 458 } 459 460 void __khugepaged_exit(struct mm_struct *mm) 461 { 462 struct khugepaged_mm_slot *mm_slot; 463 struct mm_slot *slot; 464 int free = 0; 465 466 spin_lock(&khugepaged_mm_lock); 467 slot = mm_slot_lookup(mm_slots_hash, mm); 468 mm_slot = mm_slot_entry(slot, struct khugepaged_mm_slot, slot); 469 if (mm_slot && khugepaged_scan.mm_slot != mm_slot) { 470 hash_del(&slot->hash); 471 list_del(&slot->mm_node); 472 free = 1; 473 } 474 spin_unlock(&khugepaged_mm_lock); 475 476 if (free) { 477 clear_bit(MMF_VM_HUGEPAGE, &mm->flags); 478 mm_slot_free(mm_slot_cache, mm_slot); 479 mmdrop(mm); 480 } else if (mm_slot) { 481 /* 482 * This is required to serialize against 483 * hpage_collapse_test_exit() (which is guaranteed to run 484 * under mmap sem read mode). Stop here (after we return all 485 * pagetables will be destroyed) until khugepaged has finished 486 * working on the pagetables under the mmap_lock. 487 */ 488 mmap_write_lock(mm); 489 mmap_write_unlock(mm); 490 } 491 } 492 493 static void release_pte_page(struct page *page) 494 { 495 mod_node_page_state(page_pgdat(page), 496 NR_ISOLATED_ANON + page_is_file_lru(page), 497 -compound_nr(page)); 498 unlock_page(page); 499 putback_lru_page(page); 500 } 501 502 static void release_pte_pages(pte_t *pte, pte_t *_pte, 503 struct list_head *compound_pagelist) 504 { 505 struct page *page, *tmp; 506 507 while (--_pte >= pte) { 508 pte_t pteval = *_pte; 509 510 page = pte_page(pteval); 511 if (!pte_none(pteval) && !is_zero_pfn(pte_pfn(pteval)) && 512 !PageCompound(page)) 513 release_pte_page(page); 514 } 515 516 list_for_each_entry_safe(page, tmp, compound_pagelist, lru) { 517 list_del(&page->lru); 518 release_pte_page(page); 519 } 520 } 521 522 static bool is_refcount_suitable(struct page *page) 523 { 524 int expected_refcount; 525 526 expected_refcount = total_mapcount(page); 527 if (PageSwapCache(page)) 528 expected_refcount += compound_nr(page); 529 530 return page_count(page) == expected_refcount; 531 } 532 533 static int __collapse_huge_page_isolate(struct vm_area_struct *vma, 534 unsigned long address, 535 pte_t *pte, 536 struct collapse_control *cc, 537 struct list_head *compound_pagelist) 538 { 539 struct page *page = NULL; 540 pte_t *_pte; 541 int none_or_zero = 0, shared = 0, result = SCAN_FAIL, referenced = 0; 542 bool writable = false; 543 544 for (_pte = pte; _pte < pte + HPAGE_PMD_NR; 545 _pte++, address += PAGE_SIZE) { 546 pte_t pteval = *_pte; 547 if (pte_none(pteval) || (pte_present(pteval) && 548 is_zero_pfn(pte_pfn(pteval)))) { 549 ++none_or_zero; 550 if (!userfaultfd_armed(vma) && 551 (!cc->is_khugepaged || 552 none_or_zero <= khugepaged_max_ptes_none)) { 553 continue; 554 } else { 555 result = SCAN_EXCEED_NONE_PTE; 556 count_vm_event(THP_SCAN_EXCEED_NONE_PTE); 557 goto out; 558 } 559 } 560 if (!pte_present(pteval)) { 561 result = SCAN_PTE_NON_PRESENT; 562 goto out; 563 } 564 page = vm_normal_page(vma, address, pteval); 565 if (unlikely(!page) || unlikely(is_zone_device_page(page))) { 566 result = SCAN_PAGE_NULL; 567 goto out; 568 } 569 570 VM_BUG_ON_PAGE(!PageAnon(page), page); 571 572 if (page_mapcount(page) > 1) { 573 ++shared; 574 if (cc->is_khugepaged && 575 shared > khugepaged_max_ptes_shared) { 576 result = SCAN_EXCEED_SHARED_PTE; 577 count_vm_event(THP_SCAN_EXCEED_SHARED_PTE); 578 goto out; 579 } 580 } 581 582 if (PageCompound(page)) { 583 struct page *p; 584 page = compound_head(page); 585 586 /* 587 * Check if we have dealt with the compound page 588 * already 589 */ 590 list_for_each_entry(p, compound_pagelist, lru) { 591 if (page == p) 592 goto next; 593 } 594 } 595 596 /* 597 * We can do it before isolate_lru_page because the 598 * page can't be freed from under us. NOTE: PG_lock 599 * is needed to serialize against split_huge_page 600 * when invoked from the VM. 601 */ 602 if (!trylock_page(page)) { 603 result = SCAN_PAGE_LOCK; 604 goto out; 605 } 606 607 /* 608 * Check if the page has any GUP (or other external) pins. 609 * 610 * The page table that maps the page has been already unlinked 611 * from the page table tree and this process cannot get 612 * an additional pin on the page. 613 * 614 * New pins can come later if the page is shared across fork, 615 * but not from this process. The other process cannot write to 616 * the page, only trigger CoW. 617 */ 618 if (!is_refcount_suitable(page)) { 619 unlock_page(page); 620 result = SCAN_PAGE_COUNT; 621 goto out; 622 } 623 624 /* 625 * Isolate the page to avoid collapsing an hugepage 626 * currently in use by the VM. 627 */ 628 if (isolate_lru_page(page)) { 629 unlock_page(page); 630 result = SCAN_DEL_PAGE_LRU; 631 goto out; 632 } 633 mod_node_page_state(page_pgdat(page), 634 NR_ISOLATED_ANON + page_is_file_lru(page), 635 compound_nr(page)); 636 VM_BUG_ON_PAGE(!PageLocked(page), page); 637 VM_BUG_ON_PAGE(PageLRU(page), page); 638 639 if (PageCompound(page)) 640 list_add_tail(&page->lru, compound_pagelist); 641 next: 642 /* 643 * If collapse was initiated by khugepaged, check that there is 644 * enough young pte to justify collapsing the page 645 */ 646 if (cc->is_khugepaged && 647 (pte_young(pteval) || page_is_young(page) || 648 PageReferenced(page) || mmu_notifier_test_young(vma->vm_mm, 649 address))) 650 referenced++; 651 652 if (pte_write(pteval)) 653 writable = true; 654 } 655 656 if (unlikely(!writable)) { 657 result = SCAN_PAGE_RO; 658 } else if (unlikely(cc->is_khugepaged && !referenced)) { 659 result = SCAN_LACK_REFERENCED_PAGE; 660 } else { 661 result = SCAN_SUCCEED; 662 trace_mm_collapse_huge_page_isolate(page, none_or_zero, 663 referenced, writable, result); 664 return result; 665 } 666 out: 667 release_pte_pages(pte, _pte, compound_pagelist); 668 trace_mm_collapse_huge_page_isolate(page, none_or_zero, 669 referenced, writable, result); 670 return result; 671 } 672 673 static void __collapse_huge_page_copy(pte_t *pte, struct page *page, 674 struct vm_area_struct *vma, 675 unsigned long address, 676 spinlock_t *ptl, 677 struct list_head *compound_pagelist) 678 { 679 struct page *src_page, *tmp; 680 pte_t *_pte; 681 for (_pte = pte; _pte < pte + HPAGE_PMD_NR; 682 _pte++, page++, address += PAGE_SIZE) { 683 pte_t pteval = *_pte; 684 685 if (pte_none(pteval) || is_zero_pfn(pte_pfn(pteval))) { 686 clear_user_highpage(page, address); 687 add_mm_counter(vma->vm_mm, MM_ANONPAGES, 1); 688 if (is_zero_pfn(pte_pfn(pteval))) { 689 /* 690 * ptl mostly unnecessary. 691 */ 692 spin_lock(ptl); 693 ptep_clear(vma->vm_mm, address, _pte); 694 spin_unlock(ptl); 695 } 696 } else { 697 src_page = pte_page(pteval); 698 copy_user_highpage(page, src_page, address, vma); 699 if (!PageCompound(src_page)) 700 release_pte_page(src_page); 701 /* 702 * ptl mostly unnecessary, but preempt has to 703 * be disabled to update the per-cpu stats 704 * inside page_remove_rmap(). 705 */ 706 spin_lock(ptl); 707 ptep_clear(vma->vm_mm, address, _pte); 708 page_remove_rmap(src_page, vma, false); 709 spin_unlock(ptl); 710 free_page_and_swap_cache(src_page); 711 } 712 } 713 714 list_for_each_entry_safe(src_page, tmp, compound_pagelist, lru) { 715 list_del(&src_page->lru); 716 mod_node_page_state(page_pgdat(src_page), 717 NR_ISOLATED_ANON + page_is_file_lru(src_page), 718 -compound_nr(src_page)); 719 unlock_page(src_page); 720 free_swap_cache(src_page); 721 putback_lru_page(src_page); 722 } 723 } 724 725 static void khugepaged_alloc_sleep(void) 726 { 727 DEFINE_WAIT(wait); 728 729 add_wait_queue(&khugepaged_wait, &wait); 730 __set_current_state(TASK_INTERRUPTIBLE|TASK_FREEZABLE); 731 schedule_timeout(msecs_to_jiffies(khugepaged_alloc_sleep_millisecs)); 732 remove_wait_queue(&khugepaged_wait, &wait); 733 } 734 735 struct collapse_control khugepaged_collapse_control = { 736 .is_khugepaged = true, 737 }; 738 739 static bool hpage_collapse_scan_abort(int nid, struct collapse_control *cc) 740 { 741 int i; 742 743 /* 744 * If node_reclaim_mode is disabled, then no extra effort is made to 745 * allocate memory locally. 746 */ 747 if (!node_reclaim_enabled()) 748 return false; 749 750 /* If there is a count for this node already, it must be acceptable */ 751 if (cc->node_load[nid]) 752 return false; 753 754 for (i = 0; i < MAX_NUMNODES; i++) { 755 if (!cc->node_load[i]) 756 continue; 757 if (node_distance(nid, i) > node_reclaim_distance) 758 return true; 759 } 760 return false; 761 } 762 763 #define khugepaged_defrag() \ 764 (transparent_hugepage_flags & \ 765 (1<<TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG)) 766 767 /* Defrag for khugepaged will enter direct reclaim/compaction if necessary */ 768 static inline gfp_t alloc_hugepage_khugepaged_gfpmask(void) 769 { 770 return khugepaged_defrag() ? GFP_TRANSHUGE : GFP_TRANSHUGE_LIGHT; 771 } 772 773 #ifdef CONFIG_NUMA 774 static int hpage_collapse_find_target_node(struct collapse_control *cc) 775 { 776 int nid, target_node = 0, max_value = 0; 777 778 /* find first node with max normal pages hit */ 779 for (nid = 0; nid < MAX_NUMNODES; nid++) 780 if (cc->node_load[nid] > max_value) { 781 max_value = cc->node_load[nid]; 782 target_node = nid; 783 } 784 785 for_each_online_node(nid) { 786 if (max_value == cc->node_load[nid]) 787 node_set(nid, cc->alloc_nmask); 788 } 789 790 return target_node; 791 } 792 #else 793 static int hpage_collapse_find_target_node(struct collapse_control *cc) 794 { 795 return 0; 796 } 797 #endif 798 799 static bool hpage_collapse_alloc_page(struct page **hpage, gfp_t gfp, int node, 800 nodemask_t *nmask) 801 { 802 *hpage = __alloc_pages(gfp, HPAGE_PMD_ORDER, node, nmask); 803 if (unlikely(!*hpage)) { 804 count_vm_event(THP_COLLAPSE_ALLOC_FAILED); 805 return false; 806 } 807 808 prep_transhuge_page(*hpage); 809 count_vm_event(THP_COLLAPSE_ALLOC); 810 return true; 811 } 812 813 /* 814 * If mmap_lock temporarily dropped, revalidate vma 815 * before taking mmap_lock. 816 * Returns enum scan_result value. 817 */ 818 819 static int hugepage_vma_revalidate(struct mm_struct *mm, unsigned long address, 820 bool expect_anon, 821 struct vm_area_struct **vmap, 822 struct collapse_control *cc) 823 { 824 struct vm_area_struct *vma; 825 826 if (unlikely(hpage_collapse_test_exit(mm))) 827 return SCAN_ANY_PROCESS; 828 829 *vmap = vma = find_vma(mm, address); 830 if (!vma) 831 return SCAN_VMA_NULL; 832 833 if (!transhuge_vma_suitable(vma, address)) 834 return SCAN_ADDRESS_RANGE; 835 if (!hugepage_vma_check(vma, vma->vm_flags, false, false, 836 cc->is_khugepaged)) 837 return SCAN_VMA_CHECK; 838 /* 839 * Anon VMA expected, the address may be unmapped then 840 * remapped to file after khugepaged reaquired the mmap_lock. 841 * 842 * hugepage_vma_check may return true for qualified file 843 * vmas. 844 */ 845 if (expect_anon && (!(*vmap)->anon_vma || !vma_is_anonymous(*vmap))) 846 return SCAN_PAGE_ANON; 847 return SCAN_SUCCEED; 848 } 849 850 static int find_pmd_or_thp_or_none(struct mm_struct *mm, 851 unsigned long address, 852 pmd_t **pmd) 853 { 854 pmd_t pmde; 855 856 *pmd = mm_find_pmd(mm, address); 857 if (!*pmd) 858 return SCAN_PMD_NULL; 859 860 pmde = pmd_read_atomic(*pmd); 861 862 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 863 /* See comments in pmd_none_or_trans_huge_or_clear_bad() */ 864 barrier(); 865 #endif 866 if (pmd_none(pmde)) 867 return SCAN_PMD_NONE; 868 if (pmd_trans_huge(pmde)) 869 return SCAN_PMD_MAPPED; 870 if (pmd_bad(pmde)) 871 return SCAN_PMD_NULL; 872 return SCAN_SUCCEED; 873 } 874 875 static int check_pmd_still_valid(struct mm_struct *mm, 876 unsigned long address, 877 pmd_t *pmd) 878 { 879 pmd_t *new_pmd; 880 int result = find_pmd_or_thp_or_none(mm, address, &new_pmd); 881 882 if (result != SCAN_SUCCEED) 883 return result; 884 if (new_pmd != pmd) 885 return SCAN_FAIL; 886 return SCAN_SUCCEED; 887 } 888 889 /* 890 * Bring missing pages in from swap, to complete THP collapse. 891 * Only done if hpage_collapse_scan_pmd believes it is worthwhile. 892 * 893 * Called and returns without pte mapped or spinlocks held. 894 * Note that if false is returned, mmap_lock will be released. 895 */ 896 897 static int __collapse_huge_page_swapin(struct mm_struct *mm, 898 struct vm_area_struct *vma, 899 unsigned long haddr, pmd_t *pmd, 900 int referenced) 901 { 902 int swapped_in = 0; 903 vm_fault_t ret = 0; 904 unsigned long address, end = haddr + (HPAGE_PMD_NR * PAGE_SIZE); 905 906 for (address = haddr; address < end; address += PAGE_SIZE) { 907 struct vm_fault vmf = { 908 .vma = vma, 909 .address = address, 910 .pgoff = linear_page_index(vma, haddr), 911 .flags = FAULT_FLAG_ALLOW_RETRY, 912 .pmd = pmd, 913 }; 914 915 vmf.pte = pte_offset_map(pmd, address); 916 vmf.orig_pte = *vmf.pte; 917 if (!is_swap_pte(vmf.orig_pte)) { 918 pte_unmap(vmf.pte); 919 continue; 920 } 921 ret = do_swap_page(&vmf); 922 923 /* 924 * do_swap_page returns VM_FAULT_RETRY with released mmap_lock. 925 * Note we treat VM_FAULT_RETRY as VM_FAULT_ERROR here because 926 * we do not retry here and swap entry will remain in pagetable 927 * resulting in later failure. 928 */ 929 if (ret & VM_FAULT_RETRY) { 930 trace_mm_collapse_huge_page_swapin(mm, swapped_in, referenced, 0); 931 /* Likely, but not guaranteed, that page lock failed */ 932 return SCAN_PAGE_LOCK; 933 } 934 if (ret & VM_FAULT_ERROR) { 935 mmap_read_unlock(mm); 936 trace_mm_collapse_huge_page_swapin(mm, swapped_in, referenced, 0); 937 return SCAN_FAIL; 938 } 939 swapped_in++; 940 } 941 942 /* Drain LRU add pagevec to remove extra pin on the swapped in pages */ 943 if (swapped_in) 944 lru_add_drain(); 945 946 trace_mm_collapse_huge_page_swapin(mm, swapped_in, referenced, 1); 947 return SCAN_SUCCEED; 948 } 949 950 static int alloc_charge_hpage(struct page **hpage, struct mm_struct *mm, 951 struct collapse_control *cc) 952 { 953 gfp_t gfp = (cc->is_khugepaged ? alloc_hugepage_khugepaged_gfpmask() : 954 GFP_TRANSHUGE); 955 int node = hpage_collapse_find_target_node(cc); 956 957 if (!hpage_collapse_alloc_page(hpage, gfp, node, &cc->alloc_nmask)) 958 return SCAN_ALLOC_HUGE_PAGE_FAIL; 959 if (unlikely(mem_cgroup_charge(page_folio(*hpage), mm, gfp))) 960 return SCAN_CGROUP_CHARGE_FAIL; 961 count_memcg_page_event(*hpage, THP_COLLAPSE_ALLOC); 962 return SCAN_SUCCEED; 963 } 964 965 static int collapse_huge_page(struct mm_struct *mm, unsigned long address, 966 int referenced, int unmapped, 967 struct collapse_control *cc) 968 { 969 LIST_HEAD(compound_pagelist); 970 pmd_t *pmd, _pmd; 971 pte_t *pte; 972 pgtable_t pgtable; 973 struct page *hpage; 974 spinlock_t *pmd_ptl, *pte_ptl; 975 int result = SCAN_FAIL; 976 struct vm_area_struct *vma; 977 struct mmu_notifier_range range; 978 979 VM_BUG_ON(address & ~HPAGE_PMD_MASK); 980 981 /* 982 * Before allocating the hugepage, release the mmap_lock read lock. 983 * The allocation can take potentially a long time if it involves 984 * sync compaction, and we do not need to hold the mmap_lock during 985 * that. We will recheck the vma after taking it again in write mode. 986 */ 987 mmap_read_unlock(mm); 988 989 result = alloc_charge_hpage(&hpage, mm, cc); 990 if (result != SCAN_SUCCEED) 991 goto out_nolock; 992 993 mmap_read_lock(mm); 994 result = hugepage_vma_revalidate(mm, address, true, &vma, cc); 995 if (result != SCAN_SUCCEED) { 996 mmap_read_unlock(mm); 997 goto out_nolock; 998 } 999 1000 result = find_pmd_or_thp_or_none(mm, address, &pmd); 1001 if (result != SCAN_SUCCEED) { 1002 mmap_read_unlock(mm); 1003 goto out_nolock; 1004 } 1005 1006 if (unmapped) { 1007 /* 1008 * __collapse_huge_page_swapin will return with mmap_lock 1009 * released when it fails. So we jump out_nolock directly in 1010 * that case. Continuing to collapse causes inconsistency. 1011 */ 1012 result = __collapse_huge_page_swapin(mm, vma, address, pmd, 1013 referenced); 1014 if (result != SCAN_SUCCEED) 1015 goto out_nolock; 1016 } 1017 1018 mmap_read_unlock(mm); 1019 /* 1020 * Prevent all access to pagetables with the exception of 1021 * gup_fast later handled by the ptep_clear_flush and the VM 1022 * handled by the anon_vma lock + PG_lock. 1023 */ 1024 mmap_write_lock(mm); 1025 result = hugepage_vma_revalidate(mm, address, true, &vma, cc); 1026 if (result != SCAN_SUCCEED) 1027 goto out_up_write; 1028 /* check if the pmd is still valid */ 1029 result = check_pmd_still_valid(mm, address, pmd); 1030 if (result != SCAN_SUCCEED) 1031 goto out_up_write; 1032 1033 anon_vma_lock_write(vma->anon_vma); 1034 1035 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, NULL, mm, 1036 address, address + HPAGE_PMD_SIZE); 1037 mmu_notifier_invalidate_range_start(&range); 1038 1039 pte = pte_offset_map(pmd, address); 1040 pte_ptl = pte_lockptr(mm, pmd); 1041 1042 pmd_ptl = pmd_lock(mm, pmd); /* probably unnecessary */ 1043 /* 1044 * This removes any huge TLB entry from the CPU so we won't allow 1045 * huge and small TLB entries for the same virtual address to 1046 * avoid the risk of CPU bugs in that area. 1047 * 1048 * Parallel fast GUP is fine since fast GUP will back off when 1049 * it detects PMD is changed. 1050 */ 1051 _pmd = pmdp_collapse_flush(vma, address, pmd); 1052 spin_unlock(pmd_ptl); 1053 mmu_notifier_invalidate_range_end(&range); 1054 1055 spin_lock(pte_ptl); 1056 result = __collapse_huge_page_isolate(vma, address, pte, cc, 1057 &compound_pagelist); 1058 spin_unlock(pte_ptl); 1059 1060 if (unlikely(result != SCAN_SUCCEED)) { 1061 pte_unmap(pte); 1062 spin_lock(pmd_ptl); 1063 BUG_ON(!pmd_none(*pmd)); 1064 /* 1065 * We can only use set_pmd_at when establishing 1066 * hugepmds and never for establishing regular pmds that 1067 * points to regular pagetables. Use pmd_populate for that 1068 */ 1069 pmd_populate(mm, pmd, pmd_pgtable(_pmd)); 1070 spin_unlock(pmd_ptl); 1071 anon_vma_unlock_write(vma->anon_vma); 1072 goto out_up_write; 1073 } 1074 1075 /* 1076 * All pages are isolated and locked so anon_vma rmap 1077 * can't run anymore. 1078 */ 1079 anon_vma_unlock_write(vma->anon_vma); 1080 1081 __collapse_huge_page_copy(pte, hpage, vma, address, pte_ptl, 1082 &compound_pagelist); 1083 pte_unmap(pte); 1084 /* 1085 * spin_lock() below is not the equivalent of smp_wmb(), but 1086 * the smp_wmb() inside __SetPageUptodate() can be reused to 1087 * avoid the copy_huge_page writes to become visible after 1088 * the set_pmd_at() write. 1089 */ 1090 __SetPageUptodate(hpage); 1091 pgtable = pmd_pgtable(_pmd); 1092 1093 _pmd = mk_huge_pmd(hpage, vma->vm_page_prot); 1094 _pmd = maybe_pmd_mkwrite(pmd_mkdirty(_pmd), vma); 1095 1096 spin_lock(pmd_ptl); 1097 BUG_ON(!pmd_none(*pmd)); 1098 page_add_new_anon_rmap(hpage, vma, address); 1099 lru_cache_add_inactive_or_unevictable(hpage, vma); 1100 pgtable_trans_huge_deposit(mm, pmd, pgtable); 1101 set_pmd_at(mm, address, pmd, _pmd); 1102 update_mmu_cache_pmd(vma, address, pmd); 1103 spin_unlock(pmd_ptl); 1104 1105 hpage = NULL; 1106 1107 result = SCAN_SUCCEED; 1108 out_up_write: 1109 mmap_write_unlock(mm); 1110 out_nolock: 1111 if (hpage) { 1112 mem_cgroup_uncharge(page_folio(hpage)); 1113 put_page(hpage); 1114 } 1115 trace_mm_collapse_huge_page(mm, result == SCAN_SUCCEED, result); 1116 return result; 1117 } 1118 1119 static int hpage_collapse_scan_pmd(struct mm_struct *mm, 1120 struct vm_area_struct *vma, 1121 unsigned long address, bool *mmap_locked, 1122 struct collapse_control *cc) 1123 { 1124 pmd_t *pmd; 1125 pte_t *pte, *_pte; 1126 int result = SCAN_FAIL, referenced = 0; 1127 int none_or_zero = 0, shared = 0; 1128 struct page *page = NULL; 1129 unsigned long _address; 1130 spinlock_t *ptl; 1131 int node = NUMA_NO_NODE, unmapped = 0; 1132 bool writable = false; 1133 1134 VM_BUG_ON(address & ~HPAGE_PMD_MASK); 1135 1136 result = find_pmd_or_thp_or_none(mm, address, &pmd); 1137 if (result != SCAN_SUCCEED) 1138 goto out; 1139 1140 memset(cc->node_load, 0, sizeof(cc->node_load)); 1141 nodes_clear(cc->alloc_nmask); 1142 pte = pte_offset_map_lock(mm, pmd, address, &ptl); 1143 for (_address = address, _pte = pte; _pte < pte + HPAGE_PMD_NR; 1144 _pte++, _address += PAGE_SIZE) { 1145 pte_t pteval = *_pte; 1146 if (is_swap_pte(pteval)) { 1147 ++unmapped; 1148 if (!cc->is_khugepaged || 1149 unmapped <= khugepaged_max_ptes_swap) { 1150 /* 1151 * Always be strict with uffd-wp 1152 * enabled swap entries. Please see 1153 * comment below for pte_uffd_wp(). 1154 */ 1155 if (pte_swp_uffd_wp(pteval)) { 1156 result = SCAN_PTE_UFFD_WP; 1157 goto out_unmap; 1158 } 1159 continue; 1160 } else { 1161 result = SCAN_EXCEED_SWAP_PTE; 1162 count_vm_event(THP_SCAN_EXCEED_SWAP_PTE); 1163 goto out_unmap; 1164 } 1165 } 1166 if (pte_none(pteval) || is_zero_pfn(pte_pfn(pteval))) { 1167 ++none_or_zero; 1168 if (!userfaultfd_armed(vma) && 1169 (!cc->is_khugepaged || 1170 none_or_zero <= khugepaged_max_ptes_none)) { 1171 continue; 1172 } else { 1173 result = SCAN_EXCEED_NONE_PTE; 1174 count_vm_event(THP_SCAN_EXCEED_NONE_PTE); 1175 goto out_unmap; 1176 } 1177 } 1178 if (pte_uffd_wp(pteval)) { 1179 /* 1180 * Don't collapse the page if any of the small 1181 * PTEs are armed with uffd write protection. 1182 * Here we can also mark the new huge pmd as 1183 * write protected if any of the small ones is 1184 * marked but that could bring unknown 1185 * userfault messages that falls outside of 1186 * the registered range. So, just be simple. 1187 */ 1188 result = SCAN_PTE_UFFD_WP; 1189 goto out_unmap; 1190 } 1191 if (pte_write(pteval)) 1192 writable = true; 1193 1194 page = vm_normal_page(vma, _address, pteval); 1195 if (unlikely(!page) || unlikely(is_zone_device_page(page))) { 1196 result = SCAN_PAGE_NULL; 1197 goto out_unmap; 1198 } 1199 1200 if (page_mapcount(page) > 1) { 1201 ++shared; 1202 if (cc->is_khugepaged && 1203 shared > khugepaged_max_ptes_shared) { 1204 result = SCAN_EXCEED_SHARED_PTE; 1205 count_vm_event(THP_SCAN_EXCEED_SHARED_PTE); 1206 goto out_unmap; 1207 } 1208 } 1209 1210 page = compound_head(page); 1211 1212 /* 1213 * Record which node the original page is from and save this 1214 * information to cc->node_load[]. 1215 * Khugepaged will allocate hugepage from the node has the max 1216 * hit record. 1217 */ 1218 node = page_to_nid(page); 1219 if (hpage_collapse_scan_abort(node, cc)) { 1220 result = SCAN_SCAN_ABORT; 1221 goto out_unmap; 1222 } 1223 cc->node_load[node]++; 1224 if (!PageLRU(page)) { 1225 result = SCAN_PAGE_LRU; 1226 goto out_unmap; 1227 } 1228 if (PageLocked(page)) { 1229 result = SCAN_PAGE_LOCK; 1230 goto out_unmap; 1231 } 1232 if (!PageAnon(page)) { 1233 result = SCAN_PAGE_ANON; 1234 goto out_unmap; 1235 } 1236 1237 /* 1238 * Check if the page has any GUP (or other external) pins. 1239 * 1240 * Here the check is racy it may see total_mapcount > refcount 1241 * in some cases. 1242 * For example, one process with one forked child process. 1243 * The parent has the PMD split due to MADV_DONTNEED, then 1244 * the child is trying unmap the whole PMD, but khugepaged 1245 * may be scanning the parent between the child has 1246 * PageDoubleMap flag cleared and dec the mapcount. So 1247 * khugepaged may see total_mapcount > refcount. 1248 * 1249 * But such case is ephemeral we could always retry collapse 1250 * later. However it may report false positive if the page 1251 * has excessive GUP pins (i.e. 512). Anyway the same check 1252 * will be done again later the risk seems low. 1253 */ 1254 if (!is_refcount_suitable(page)) { 1255 result = SCAN_PAGE_COUNT; 1256 goto out_unmap; 1257 } 1258 1259 /* 1260 * If collapse was initiated by khugepaged, check that there is 1261 * enough young pte to justify collapsing the page 1262 */ 1263 if (cc->is_khugepaged && 1264 (pte_young(pteval) || page_is_young(page) || 1265 PageReferenced(page) || mmu_notifier_test_young(vma->vm_mm, 1266 address))) 1267 referenced++; 1268 } 1269 if (!writable) { 1270 result = SCAN_PAGE_RO; 1271 } else if (cc->is_khugepaged && 1272 (!referenced || 1273 (unmapped && referenced < HPAGE_PMD_NR / 2))) { 1274 result = SCAN_LACK_REFERENCED_PAGE; 1275 } else { 1276 result = SCAN_SUCCEED; 1277 } 1278 out_unmap: 1279 pte_unmap_unlock(pte, ptl); 1280 if (result == SCAN_SUCCEED) { 1281 result = collapse_huge_page(mm, address, referenced, 1282 unmapped, cc); 1283 /* collapse_huge_page will return with the mmap_lock released */ 1284 *mmap_locked = false; 1285 } 1286 out: 1287 trace_mm_khugepaged_scan_pmd(mm, page, writable, referenced, 1288 none_or_zero, result, unmapped); 1289 return result; 1290 } 1291 1292 static void collect_mm_slot(struct khugepaged_mm_slot *mm_slot) 1293 { 1294 struct mm_slot *slot = &mm_slot->slot; 1295 struct mm_struct *mm = slot->mm; 1296 1297 lockdep_assert_held(&khugepaged_mm_lock); 1298 1299 if (hpage_collapse_test_exit(mm)) { 1300 /* free mm_slot */ 1301 hash_del(&slot->hash); 1302 list_del(&slot->mm_node); 1303 1304 /* 1305 * Not strictly needed because the mm exited already. 1306 * 1307 * clear_bit(MMF_VM_HUGEPAGE, &mm->flags); 1308 */ 1309 1310 /* khugepaged_mm_lock actually not necessary for the below */ 1311 mm_slot_free(mm_slot_cache, mm_slot); 1312 mmdrop(mm); 1313 } 1314 } 1315 1316 #ifdef CONFIG_SHMEM 1317 /* 1318 * Notify khugepaged that given addr of the mm is pte-mapped THP. Then 1319 * khugepaged should try to collapse the page table. 1320 * 1321 * Note that following race exists: 1322 * (1) khugepaged calls khugepaged_collapse_pte_mapped_thps() for mm_struct A, 1323 * emptying the A's ->pte_mapped_thp[] array. 1324 * (2) MADV_COLLAPSE collapses some file extent with target mm_struct B, and 1325 * retract_page_tables() finds a VMA in mm_struct A mapping the same extent 1326 * (at virtual address X) and adds an entry (for X) into mm_struct A's 1327 * ->pte-mapped_thp[] array. 1328 * (3) khugepaged calls khugepaged_collapse_scan_file() for mm_struct A at X, 1329 * sees a pte-mapped THP (SCAN_PTE_MAPPED_HUGEPAGE) and adds an entry 1330 * (for X) into mm_struct A's ->pte-mapped_thp[] array. 1331 * Thus, it's possible the same address is added multiple times for the same 1332 * mm_struct. Should this happen, we'll simply attempt 1333 * collapse_pte_mapped_thp() multiple times for the same address, under the same 1334 * exclusive mmap_lock, and assuming the first call is successful, subsequent 1335 * attempts will return quickly (without grabbing any additional locks) when 1336 * a huge pmd is found in find_pmd_or_thp_or_none(). Since this is a cheap 1337 * check, and since this is a rare occurrence, the cost of preventing this 1338 * "multiple-add" is thought to be more expensive than just handling it, should 1339 * it occur. 1340 */ 1341 static bool khugepaged_add_pte_mapped_thp(struct mm_struct *mm, 1342 unsigned long addr) 1343 { 1344 struct khugepaged_mm_slot *mm_slot; 1345 struct mm_slot *slot; 1346 bool ret = false; 1347 1348 VM_BUG_ON(addr & ~HPAGE_PMD_MASK); 1349 1350 spin_lock(&khugepaged_mm_lock); 1351 slot = mm_slot_lookup(mm_slots_hash, mm); 1352 mm_slot = mm_slot_entry(slot, struct khugepaged_mm_slot, slot); 1353 if (likely(mm_slot && mm_slot->nr_pte_mapped_thp < MAX_PTE_MAPPED_THP)) { 1354 mm_slot->pte_mapped_thp[mm_slot->nr_pte_mapped_thp++] = addr; 1355 ret = true; 1356 } 1357 spin_unlock(&khugepaged_mm_lock); 1358 return ret; 1359 } 1360 1361 /* hpage must be locked, and mmap_lock must be held in write */ 1362 static int set_huge_pmd(struct vm_area_struct *vma, unsigned long addr, 1363 pmd_t *pmdp, struct page *hpage) 1364 { 1365 struct vm_fault vmf = { 1366 .vma = vma, 1367 .address = addr, 1368 .flags = 0, 1369 .pmd = pmdp, 1370 }; 1371 1372 VM_BUG_ON(!PageTransHuge(hpage)); 1373 mmap_assert_write_locked(vma->vm_mm); 1374 1375 if (do_set_pmd(&vmf, hpage)) 1376 return SCAN_FAIL; 1377 1378 get_page(hpage); 1379 return SCAN_SUCCEED; 1380 } 1381 1382 static void collapse_and_free_pmd(struct mm_struct *mm, struct vm_area_struct *vma, 1383 unsigned long addr, pmd_t *pmdp) 1384 { 1385 spinlock_t *ptl; 1386 pmd_t pmd; 1387 1388 mmap_assert_write_locked(mm); 1389 ptl = pmd_lock(vma->vm_mm, pmdp); 1390 pmd = pmdp_collapse_flush(vma, addr, pmdp); 1391 spin_unlock(ptl); 1392 mm_dec_nr_ptes(mm); 1393 page_table_check_pte_clear_range(mm, addr, pmd); 1394 pte_free(mm, pmd_pgtable(pmd)); 1395 } 1396 1397 /** 1398 * collapse_pte_mapped_thp - Try to collapse a pte-mapped THP for mm at 1399 * address haddr. 1400 * 1401 * @mm: process address space where collapse happens 1402 * @addr: THP collapse address 1403 * @install_pmd: If a huge PMD should be installed 1404 * 1405 * This function checks whether all the PTEs in the PMD are pointing to the 1406 * right THP. If so, retract the page table so the THP can refault in with 1407 * as pmd-mapped. Possibly install a huge PMD mapping the THP. 1408 */ 1409 int collapse_pte_mapped_thp(struct mm_struct *mm, unsigned long addr, 1410 bool install_pmd) 1411 { 1412 unsigned long haddr = addr & HPAGE_PMD_MASK; 1413 struct vm_area_struct *vma = vma_lookup(mm, haddr); 1414 struct page *hpage; 1415 pte_t *start_pte, *pte; 1416 pmd_t *pmd; 1417 spinlock_t *ptl; 1418 int count = 0, result = SCAN_FAIL; 1419 int i; 1420 1421 mmap_assert_write_locked(mm); 1422 1423 /* Fast check before locking page if already PMD-mapped */ 1424 result = find_pmd_or_thp_or_none(mm, haddr, &pmd); 1425 if (result == SCAN_PMD_MAPPED) 1426 return result; 1427 1428 if (!vma || !vma->vm_file || 1429 !range_in_vma(vma, haddr, haddr + HPAGE_PMD_SIZE)) 1430 return SCAN_VMA_CHECK; 1431 1432 /* 1433 * If we are here, we've succeeded in replacing all the native pages 1434 * in the page cache with a single hugepage. If a mm were to fault-in 1435 * this memory (mapped by a suitably aligned VMA), we'd get the hugepage 1436 * and map it by a PMD, regardless of sysfs THP settings. As such, let's 1437 * analogously elide sysfs THP settings here. 1438 */ 1439 if (!hugepage_vma_check(vma, vma->vm_flags, false, false, false)) 1440 return SCAN_VMA_CHECK; 1441 1442 /* Keep pmd pgtable for uffd-wp; see comment in retract_page_tables() */ 1443 if (userfaultfd_wp(vma)) 1444 return SCAN_PTE_UFFD_WP; 1445 1446 hpage = find_lock_page(vma->vm_file->f_mapping, 1447 linear_page_index(vma, haddr)); 1448 if (!hpage) 1449 return SCAN_PAGE_NULL; 1450 1451 if (!PageHead(hpage)) { 1452 result = SCAN_FAIL; 1453 goto drop_hpage; 1454 } 1455 1456 if (compound_order(hpage) != HPAGE_PMD_ORDER) { 1457 result = SCAN_PAGE_COMPOUND; 1458 goto drop_hpage; 1459 } 1460 1461 switch (result) { 1462 case SCAN_SUCCEED: 1463 break; 1464 case SCAN_PMD_NONE: 1465 /* 1466 * In MADV_COLLAPSE path, possible race with khugepaged where 1467 * all pte entries have been removed and pmd cleared. If so, 1468 * skip all the pte checks and just update the pmd mapping. 1469 */ 1470 goto maybe_install_pmd; 1471 default: 1472 goto drop_hpage; 1473 } 1474 1475 start_pte = pte_offset_map_lock(mm, pmd, haddr, &ptl); 1476 result = SCAN_FAIL; 1477 1478 /* step 1: check all mapped PTEs are to the right huge page */ 1479 for (i = 0, addr = haddr, pte = start_pte; 1480 i < HPAGE_PMD_NR; i++, addr += PAGE_SIZE, pte++) { 1481 struct page *page; 1482 1483 /* empty pte, skip */ 1484 if (pte_none(*pte)) 1485 continue; 1486 1487 /* page swapped out, abort */ 1488 if (!pte_present(*pte)) { 1489 result = SCAN_PTE_NON_PRESENT; 1490 goto abort; 1491 } 1492 1493 page = vm_normal_page(vma, addr, *pte); 1494 if (WARN_ON_ONCE(page && is_zone_device_page(page))) 1495 page = NULL; 1496 /* 1497 * Note that uprobe, debugger, or MAP_PRIVATE may change the 1498 * page table, but the new page will not be a subpage of hpage. 1499 */ 1500 if (hpage + i != page) 1501 goto abort; 1502 count++; 1503 } 1504 1505 /* step 2: adjust rmap */ 1506 for (i = 0, addr = haddr, pte = start_pte; 1507 i < HPAGE_PMD_NR; i++, addr += PAGE_SIZE, pte++) { 1508 struct page *page; 1509 1510 if (pte_none(*pte)) 1511 continue; 1512 page = vm_normal_page(vma, addr, *pte); 1513 if (WARN_ON_ONCE(page && is_zone_device_page(page))) 1514 goto abort; 1515 page_remove_rmap(page, vma, false); 1516 } 1517 1518 pte_unmap_unlock(start_pte, ptl); 1519 1520 /* step 3: set proper refcount and mm_counters. */ 1521 if (count) { 1522 page_ref_sub(hpage, count); 1523 add_mm_counter(vma->vm_mm, mm_counter_file(hpage), -count); 1524 } 1525 1526 /* step 4: remove pte entries */ 1527 collapse_and_free_pmd(mm, vma, haddr, pmd); 1528 1529 maybe_install_pmd: 1530 /* step 5: install pmd entry */ 1531 result = install_pmd 1532 ? set_huge_pmd(vma, haddr, pmd, hpage) 1533 : SCAN_SUCCEED; 1534 1535 drop_hpage: 1536 unlock_page(hpage); 1537 put_page(hpage); 1538 return result; 1539 1540 abort: 1541 pte_unmap_unlock(start_pte, ptl); 1542 goto drop_hpage; 1543 } 1544 1545 static void khugepaged_collapse_pte_mapped_thps(struct khugepaged_mm_slot *mm_slot) 1546 { 1547 struct mm_slot *slot = &mm_slot->slot; 1548 struct mm_struct *mm = slot->mm; 1549 int i; 1550 1551 if (likely(mm_slot->nr_pte_mapped_thp == 0)) 1552 return; 1553 1554 if (!mmap_write_trylock(mm)) 1555 return; 1556 1557 if (unlikely(hpage_collapse_test_exit(mm))) 1558 goto out; 1559 1560 for (i = 0; i < mm_slot->nr_pte_mapped_thp; i++) 1561 collapse_pte_mapped_thp(mm, mm_slot->pte_mapped_thp[i], false); 1562 1563 out: 1564 mm_slot->nr_pte_mapped_thp = 0; 1565 mmap_write_unlock(mm); 1566 } 1567 1568 static int retract_page_tables(struct address_space *mapping, pgoff_t pgoff, 1569 struct mm_struct *target_mm, 1570 unsigned long target_addr, struct page *hpage, 1571 struct collapse_control *cc) 1572 { 1573 struct vm_area_struct *vma; 1574 int target_result = SCAN_FAIL; 1575 1576 i_mmap_lock_write(mapping); 1577 vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff, pgoff) { 1578 int result = SCAN_FAIL; 1579 struct mm_struct *mm = NULL; 1580 unsigned long addr = 0; 1581 pmd_t *pmd; 1582 bool is_target = false; 1583 1584 /* 1585 * Check vma->anon_vma to exclude MAP_PRIVATE mappings that 1586 * got written to. These VMAs are likely not worth investing 1587 * mmap_write_lock(mm) as PMD-mapping is likely to be split 1588 * later. 1589 * 1590 * Note that vma->anon_vma check is racy: it can be set up after 1591 * the check but before we took mmap_lock by the fault path. 1592 * But page lock would prevent establishing any new ptes of the 1593 * page, so we are safe. 1594 * 1595 * An alternative would be drop the check, but check that page 1596 * table is clear before calling pmdp_collapse_flush() under 1597 * ptl. It has higher chance to recover THP for the VMA, but 1598 * has higher cost too. 1599 */ 1600 if (vma->anon_vma) { 1601 result = SCAN_PAGE_ANON; 1602 goto next; 1603 } 1604 addr = vma->vm_start + ((pgoff - vma->vm_pgoff) << PAGE_SHIFT); 1605 if (addr & ~HPAGE_PMD_MASK || 1606 vma->vm_end < addr + HPAGE_PMD_SIZE) { 1607 result = SCAN_VMA_CHECK; 1608 goto next; 1609 } 1610 mm = vma->vm_mm; 1611 is_target = mm == target_mm && addr == target_addr; 1612 result = find_pmd_or_thp_or_none(mm, addr, &pmd); 1613 if (result != SCAN_SUCCEED) 1614 goto next; 1615 /* 1616 * We need exclusive mmap_lock to retract page table. 1617 * 1618 * We use trylock due to lock inversion: we need to acquire 1619 * mmap_lock while holding page lock. Fault path does it in 1620 * reverse order. Trylock is a way to avoid deadlock. 1621 * 1622 * Also, it's not MADV_COLLAPSE's job to collapse other 1623 * mappings - let khugepaged take care of them later. 1624 */ 1625 result = SCAN_PTE_MAPPED_HUGEPAGE; 1626 if ((cc->is_khugepaged || is_target) && 1627 mmap_write_trylock(mm)) { 1628 /* 1629 * When a vma is registered with uffd-wp, we can't 1630 * recycle the pmd pgtable because there can be pte 1631 * markers installed. Skip it only, so the rest mm/vma 1632 * can still have the same file mapped hugely, however 1633 * it'll always mapped in small page size for uffd-wp 1634 * registered ranges. 1635 */ 1636 if (hpage_collapse_test_exit(mm)) { 1637 result = SCAN_ANY_PROCESS; 1638 goto unlock_next; 1639 } 1640 if (userfaultfd_wp(vma)) { 1641 result = SCAN_PTE_UFFD_WP; 1642 goto unlock_next; 1643 } 1644 collapse_and_free_pmd(mm, vma, addr, pmd); 1645 if (!cc->is_khugepaged && is_target) 1646 result = set_huge_pmd(vma, addr, pmd, hpage); 1647 else 1648 result = SCAN_SUCCEED; 1649 1650 unlock_next: 1651 mmap_write_unlock(mm); 1652 goto next; 1653 } 1654 /* 1655 * Calling context will handle target mm/addr. Otherwise, let 1656 * khugepaged try again later. 1657 */ 1658 if (!is_target) { 1659 khugepaged_add_pte_mapped_thp(mm, addr); 1660 continue; 1661 } 1662 next: 1663 if (is_target) 1664 target_result = result; 1665 } 1666 i_mmap_unlock_write(mapping); 1667 return target_result; 1668 } 1669 1670 /** 1671 * collapse_file - collapse filemap/tmpfs/shmem pages into huge one. 1672 * 1673 * @mm: process address space where collapse happens 1674 * @addr: virtual collapse start address 1675 * @file: file that collapse on 1676 * @start: collapse start address 1677 * @cc: collapse context and scratchpad 1678 * 1679 * Basic scheme is simple, details are more complex: 1680 * - allocate and lock a new huge page; 1681 * - scan page cache replacing old pages with the new one 1682 * + swap/gup in pages if necessary; 1683 * + fill in gaps; 1684 * + keep old pages around in case rollback is required; 1685 * - if replacing succeeds: 1686 * + copy data over; 1687 * + free old pages; 1688 * + unlock huge page; 1689 * - if replacing failed; 1690 * + put all pages back and unfreeze them; 1691 * + restore gaps in the page cache; 1692 * + unlock and free huge page; 1693 */ 1694 static int collapse_file(struct mm_struct *mm, unsigned long addr, 1695 struct file *file, pgoff_t start, 1696 struct collapse_control *cc) 1697 { 1698 struct address_space *mapping = file->f_mapping; 1699 struct page *hpage; 1700 pgoff_t index, end = start + HPAGE_PMD_NR; 1701 LIST_HEAD(pagelist); 1702 XA_STATE_ORDER(xas, &mapping->i_pages, start, HPAGE_PMD_ORDER); 1703 int nr_none = 0, result = SCAN_SUCCEED; 1704 bool is_shmem = shmem_file(file); 1705 int nr; 1706 1707 VM_BUG_ON(!IS_ENABLED(CONFIG_READ_ONLY_THP_FOR_FS) && !is_shmem); 1708 VM_BUG_ON(start & (HPAGE_PMD_NR - 1)); 1709 1710 result = alloc_charge_hpage(&hpage, mm, cc); 1711 if (result != SCAN_SUCCEED) 1712 goto out; 1713 1714 /* 1715 * Ensure we have slots for all the pages in the range. This is 1716 * almost certainly a no-op because most of the pages must be present 1717 */ 1718 do { 1719 xas_lock_irq(&xas); 1720 xas_create_range(&xas); 1721 if (!xas_error(&xas)) 1722 break; 1723 xas_unlock_irq(&xas); 1724 if (!xas_nomem(&xas, GFP_KERNEL)) { 1725 result = SCAN_FAIL; 1726 goto out; 1727 } 1728 } while (1); 1729 1730 __SetPageLocked(hpage); 1731 if (is_shmem) 1732 __SetPageSwapBacked(hpage); 1733 hpage->index = start; 1734 hpage->mapping = mapping; 1735 1736 /* 1737 * At this point the hpage is locked and not up-to-date. 1738 * It's safe to insert it into the page cache, because nobody would 1739 * be able to map it or use it in another way until we unlock it. 1740 */ 1741 1742 xas_set(&xas, start); 1743 for (index = start; index < end; index++) { 1744 struct page *page = xas_next(&xas); 1745 1746 VM_BUG_ON(index != xas.xa_index); 1747 if (is_shmem) { 1748 if (!page) { 1749 /* 1750 * Stop if extent has been truncated or 1751 * hole-punched, and is now completely 1752 * empty. 1753 */ 1754 if (index == start) { 1755 if (!xas_next_entry(&xas, end - 1)) { 1756 result = SCAN_TRUNCATED; 1757 goto xa_locked; 1758 } 1759 xas_set(&xas, index); 1760 } 1761 if (!shmem_charge(mapping->host, 1)) { 1762 result = SCAN_FAIL; 1763 goto xa_locked; 1764 } 1765 xas_store(&xas, hpage); 1766 nr_none++; 1767 continue; 1768 } 1769 1770 if (xa_is_value(page) || !PageUptodate(page)) { 1771 struct folio *folio; 1772 1773 xas_unlock_irq(&xas); 1774 /* swap in or instantiate fallocated page */ 1775 if (shmem_get_folio(mapping->host, index, 1776 &folio, SGP_NOALLOC)) { 1777 result = SCAN_FAIL; 1778 goto xa_unlocked; 1779 } 1780 page = folio_file_page(folio, index); 1781 } else if (trylock_page(page)) { 1782 get_page(page); 1783 xas_unlock_irq(&xas); 1784 } else { 1785 result = SCAN_PAGE_LOCK; 1786 goto xa_locked; 1787 } 1788 } else { /* !is_shmem */ 1789 if (!page || xa_is_value(page)) { 1790 xas_unlock_irq(&xas); 1791 page_cache_sync_readahead(mapping, &file->f_ra, 1792 file, index, 1793 end - index); 1794 /* drain pagevecs to help isolate_lru_page() */ 1795 lru_add_drain(); 1796 page = find_lock_page(mapping, index); 1797 if (unlikely(page == NULL)) { 1798 result = SCAN_FAIL; 1799 goto xa_unlocked; 1800 } 1801 } else if (PageDirty(page)) { 1802 /* 1803 * khugepaged only works on read-only fd, 1804 * so this page is dirty because it hasn't 1805 * been flushed since first write. There 1806 * won't be new dirty pages. 1807 * 1808 * Trigger async flush here and hope the 1809 * writeback is done when khugepaged 1810 * revisits this page. 1811 * 1812 * This is a one-off situation. We are not 1813 * forcing writeback in loop. 1814 */ 1815 xas_unlock_irq(&xas); 1816 filemap_flush(mapping); 1817 result = SCAN_FAIL; 1818 goto xa_unlocked; 1819 } else if (PageWriteback(page)) { 1820 xas_unlock_irq(&xas); 1821 result = SCAN_FAIL; 1822 goto xa_unlocked; 1823 } else if (trylock_page(page)) { 1824 get_page(page); 1825 xas_unlock_irq(&xas); 1826 } else { 1827 result = SCAN_PAGE_LOCK; 1828 goto xa_locked; 1829 } 1830 } 1831 1832 /* 1833 * The page must be locked, so we can drop the i_pages lock 1834 * without racing with truncate. 1835 */ 1836 VM_BUG_ON_PAGE(!PageLocked(page), page); 1837 1838 /* make sure the page is up to date */ 1839 if (unlikely(!PageUptodate(page))) { 1840 result = SCAN_FAIL; 1841 goto out_unlock; 1842 } 1843 1844 /* 1845 * If file was truncated then extended, or hole-punched, before 1846 * we locked the first page, then a THP might be there already. 1847 * This will be discovered on the first iteration. 1848 */ 1849 if (PageTransCompound(page)) { 1850 struct page *head = compound_head(page); 1851 1852 result = compound_order(head) == HPAGE_PMD_ORDER && 1853 head->index == start 1854 /* Maybe PMD-mapped */ 1855 ? SCAN_PTE_MAPPED_HUGEPAGE 1856 : SCAN_PAGE_COMPOUND; 1857 goto out_unlock; 1858 } 1859 1860 if (page_mapping(page) != mapping) { 1861 result = SCAN_TRUNCATED; 1862 goto out_unlock; 1863 } 1864 1865 if (!is_shmem && (PageDirty(page) || 1866 PageWriteback(page))) { 1867 /* 1868 * khugepaged only works on read-only fd, so this 1869 * page is dirty because it hasn't been flushed 1870 * since first write. 1871 */ 1872 result = SCAN_FAIL; 1873 goto out_unlock; 1874 } 1875 1876 if (isolate_lru_page(page)) { 1877 result = SCAN_DEL_PAGE_LRU; 1878 goto out_unlock; 1879 } 1880 1881 if (page_has_private(page) && 1882 !try_to_release_page(page, GFP_KERNEL)) { 1883 result = SCAN_PAGE_HAS_PRIVATE; 1884 putback_lru_page(page); 1885 goto out_unlock; 1886 } 1887 1888 if (page_mapped(page)) 1889 try_to_unmap(page_folio(page), 1890 TTU_IGNORE_MLOCK | TTU_BATCH_FLUSH); 1891 1892 xas_lock_irq(&xas); 1893 xas_set(&xas, index); 1894 1895 VM_BUG_ON_PAGE(page != xas_load(&xas), page); 1896 1897 /* 1898 * The page is expected to have page_count() == 3: 1899 * - we hold a pin on it; 1900 * - one reference from page cache; 1901 * - one from isolate_lru_page; 1902 */ 1903 if (!page_ref_freeze(page, 3)) { 1904 result = SCAN_PAGE_COUNT; 1905 xas_unlock_irq(&xas); 1906 putback_lru_page(page); 1907 goto out_unlock; 1908 } 1909 1910 /* 1911 * Add the page to the list to be able to undo the collapse if 1912 * something go wrong. 1913 */ 1914 list_add_tail(&page->lru, &pagelist); 1915 1916 /* Finally, replace with the new page. */ 1917 xas_store(&xas, hpage); 1918 continue; 1919 out_unlock: 1920 unlock_page(page); 1921 put_page(page); 1922 goto xa_unlocked; 1923 } 1924 nr = thp_nr_pages(hpage); 1925 1926 if (is_shmem) 1927 __mod_lruvec_page_state(hpage, NR_SHMEM_THPS, nr); 1928 else { 1929 __mod_lruvec_page_state(hpage, NR_FILE_THPS, nr); 1930 filemap_nr_thps_inc(mapping); 1931 /* 1932 * Paired with smp_mb() in do_dentry_open() to ensure 1933 * i_writecount is up to date and the update to nr_thps is 1934 * visible. Ensures the page cache will be truncated if the 1935 * file is opened writable. 1936 */ 1937 smp_mb(); 1938 if (inode_is_open_for_write(mapping->host)) { 1939 result = SCAN_FAIL; 1940 __mod_lruvec_page_state(hpage, NR_FILE_THPS, -nr); 1941 filemap_nr_thps_dec(mapping); 1942 goto xa_locked; 1943 } 1944 } 1945 1946 if (nr_none) { 1947 __mod_lruvec_page_state(hpage, NR_FILE_PAGES, nr_none); 1948 /* nr_none is always 0 for non-shmem. */ 1949 __mod_lruvec_page_state(hpage, NR_SHMEM, nr_none); 1950 } 1951 1952 /* Join all the small entries into a single multi-index entry */ 1953 xas_set_order(&xas, start, HPAGE_PMD_ORDER); 1954 xas_store(&xas, hpage); 1955 xa_locked: 1956 xas_unlock_irq(&xas); 1957 xa_unlocked: 1958 1959 /* 1960 * If collapse is successful, flush must be done now before copying. 1961 * If collapse is unsuccessful, does flush actually need to be done? 1962 * Do it anyway, to clear the state. 1963 */ 1964 try_to_unmap_flush(); 1965 1966 if (result == SCAN_SUCCEED) { 1967 struct page *page, *tmp; 1968 1969 /* 1970 * Replacing old pages with new one has succeeded, now we 1971 * need to copy the content and free the old pages. 1972 */ 1973 index = start; 1974 list_for_each_entry_safe(page, tmp, &pagelist, lru) { 1975 while (index < page->index) { 1976 clear_highpage(hpage + (index % HPAGE_PMD_NR)); 1977 index++; 1978 } 1979 copy_highpage(hpage + (page->index % HPAGE_PMD_NR), 1980 page); 1981 list_del(&page->lru); 1982 page->mapping = NULL; 1983 page_ref_unfreeze(page, 1); 1984 ClearPageActive(page); 1985 ClearPageUnevictable(page); 1986 unlock_page(page); 1987 put_page(page); 1988 index++; 1989 } 1990 while (index < end) { 1991 clear_highpage(hpage + (index % HPAGE_PMD_NR)); 1992 index++; 1993 } 1994 1995 SetPageUptodate(hpage); 1996 page_ref_add(hpage, HPAGE_PMD_NR - 1); 1997 if (is_shmem) 1998 set_page_dirty(hpage); 1999 lru_cache_add(hpage); 2000 2001 /* 2002 * Remove pte page tables, so we can re-fault the page as huge. 2003 */ 2004 result = retract_page_tables(mapping, start, mm, addr, hpage, 2005 cc); 2006 unlock_page(hpage); 2007 hpage = NULL; 2008 } else { 2009 struct page *page; 2010 2011 /* Something went wrong: roll back page cache changes */ 2012 xas_lock_irq(&xas); 2013 if (nr_none) { 2014 mapping->nrpages -= nr_none; 2015 shmem_uncharge(mapping->host, nr_none); 2016 } 2017 2018 xas_set(&xas, start); 2019 xas_for_each(&xas, page, end - 1) { 2020 page = list_first_entry_or_null(&pagelist, 2021 struct page, lru); 2022 if (!page || xas.xa_index < page->index) { 2023 if (!nr_none) 2024 break; 2025 nr_none--; 2026 /* Put holes back where they were */ 2027 xas_store(&xas, NULL); 2028 continue; 2029 } 2030 2031 VM_BUG_ON_PAGE(page->index != xas.xa_index, page); 2032 2033 /* Unfreeze the page. */ 2034 list_del(&page->lru); 2035 page_ref_unfreeze(page, 2); 2036 xas_store(&xas, page); 2037 xas_pause(&xas); 2038 xas_unlock_irq(&xas); 2039 unlock_page(page); 2040 putback_lru_page(page); 2041 xas_lock_irq(&xas); 2042 } 2043 VM_BUG_ON(nr_none); 2044 xas_unlock_irq(&xas); 2045 2046 hpage->mapping = NULL; 2047 } 2048 2049 if (hpage) 2050 unlock_page(hpage); 2051 out: 2052 VM_BUG_ON(!list_empty(&pagelist)); 2053 if (hpage) { 2054 mem_cgroup_uncharge(page_folio(hpage)); 2055 put_page(hpage); 2056 } 2057 /* TODO: tracepoints */ 2058 return result; 2059 } 2060 2061 static int hpage_collapse_scan_file(struct mm_struct *mm, unsigned long addr, 2062 struct file *file, pgoff_t start, 2063 struct collapse_control *cc) 2064 { 2065 struct page *page = NULL; 2066 struct address_space *mapping = file->f_mapping; 2067 XA_STATE(xas, &mapping->i_pages, start); 2068 int present, swap; 2069 int node = NUMA_NO_NODE; 2070 int result = SCAN_SUCCEED; 2071 2072 present = 0; 2073 swap = 0; 2074 memset(cc->node_load, 0, sizeof(cc->node_load)); 2075 nodes_clear(cc->alloc_nmask); 2076 rcu_read_lock(); 2077 xas_for_each(&xas, page, start + HPAGE_PMD_NR - 1) { 2078 if (xas_retry(&xas, page)) 2079 continue; 2080 2081 if (xa_is_value(page)) { 2082 ++swap; 2083 if (cc->is_khugepaged && 2084 swap > khugepaged_max_ptes_swap) { 2085 result = SCAN_EXCEED_SWAP_PTE; 2086 count_vm_event(THP_SCAN_EXCEED_SWAP_PTE); 2087 break; 2088 } 2089 continue; 2090 } 2091 2092 /* 2093 * TODO: khugepaged should compact smaller compound pages 2094 * into a PMD sized page 2095 */ 2096 if (PageTransCompound(page)) { 2097 struct page *head = compound_head(page); 2098 2099 result = compound_order(head) == HPAGE_PMD_ORDER && 2100 head->index == start 2101 /* Maybe PMD-mapped */ 2102 ? SCAN_PTE_MAPPED_HUGEPAGE 2103 : SCAN_PAGE_COMPOUND; 2104 /* 2105 * For SCAN_PTE_MAPPED_HUGEPAGE, further processing 2106 * by the caller won't touch the page cache, and so 2107 * it's safe to skip LRU and refcount checks before 2108 * returning. 2109 */ 2110 break; 2111 } 2112 2113 node = page_to_nid(page); 2114 if (hpage_collapse_scan_abort(node, cc)) { 2115 result = SCAN_SCAN_ABORT; 2116 break; 2117 } 2118 cc->node_load[node]++; 2119 2120 if (!PageLRU(page)) { 2121 result = SCAN_PAGE_LRU; 2122 break; 2123 } 2124 2125 if (page_count(page) != 2126 1 + page_mapcount(page) + page_has_private(page)) { 2127 result = SCAN_PAGE_COUNT; 2128 break; 2129 } 2130 2131 /* 2132 * We probably should check if the page is referenced here, but 2133 * nobody would transfer pte_young() to PageReferenced() for us. 2134 * And rmap walk here is just too costly... 2135 */ 2136 2137 present++; 2138 2139 if (need_resched()) { 2140 xas_pause(&xas); 2141 cond_resched_rcu(); 2142 } 2143 } 2144 rcu_read_unlock(); 2145 2146 if (result == SCAN_SUCCEED) { 2147 if (cc->is_khugepaged && 2148 present < HPAGE_PMD_NR - khugepaged_max_ptes_none) { 2149 result = SCAN_EXCEED_NONE_PTE; 2150 count_vm_event(THP_SCAN_EXCEED_NONE_PTE); 2151 } else { 2152 result = collapse_file(mm, addr, file, start, cc); 2153 } 2154 } 2155 2156 trace_mm_khugepaged_scan_file(mm, page, file, present, swap, result); 2157 return result; 2158 } 2159 #else 2160 static int hpage_collapse_scan_file(struct mm_struct *mm, unsigned long addr, 2161 struct file *file, pgoff_t start, 2162 struct collapse_control *cc) 2163 { 2164 BUILD_BUG(); 2165 } 2166 2167 static void khugepaged_collapse_pte_mapped_thps(struct khugepaged_mm_slot *mm_slot) 2168 { 2169 } 2170 2171 static bool khugepaged_add_pte_mapped_thp(struct mm_struct *mm, 2172 unsigned long addr) 2173 { 2174 return false; 2175 } 2176 #endif 2177 2178 static unsigned int khugepaged_scan_mm_slot(unsigned int pages, int *result, 2179 struct collapse_control *cc) 2180 __releases(&khugepaged_mm_lock) 2181 __acquires(&khugepaged_mm_lock) 2182 { 2183 struct vma_iterator vmi; 2184 struct khugepaged_mm_slot *mm_slot; 2185 struct mm_slot *slot; 2186 struct mm_struct *mm; 2187 struct vm_area_struct *vma; 2188 int progress = 0; 2189 2190 VM_BUG_ON(!pages); 2191 lockdep_assert_held(&khugepaged_mm_lock); 2192 *result = SCAN_FAIL; 2193 2194 if (khugepaged_scan.mm_slot) { 2195 mm_slot = khugepaged_scan.mm_slot; 2196 slot = &mm_slot->slot; 2197 } else { 2198 slot = list_entry(khugepaged_scan.mm_head.next, 2199 struct mm_slot, mm_node); 2200 mm_slot = mm_slot_entry(slot, struct khugepaged_mm_slot, slot); 2201 khugepaged_scan.address = 0; 2202 khugepaged_scan.mm_slot = mm_slot; 2203 } 2204 spin_unlock(&khugepaged_mm_lock); 2205 khugepaged_collapse_pte_mapped_thps(mm_slot); 2206 2207 mm = slot->mm; 2208 /* 2209 * Don't wait for semaphore (to avoid long wait times). Just move to 2210 * the next mm on the list. 2211 */ 2212 vma = NULL; 2213 if (unlikely(!mmap_read_trylock(mm))) 2214 goto breakouterloop_mmap_lock; 2215 2216 progress++; 2217 if (unlikely(hpage_collapse_test_exit(mm))) 2218 goto breakouterloop; 2219 2220 vma_iter_init(&vmi, mm, khugepaged_scan.address); 2221 for_each_vma(vmi, vma) { 2222 unsigned long hstart, hend; 2223 2224 cond_resched(); 2225 if (unlikely(hpage_collapse_test_exit(mm))) { 2226 progress++; 2227 break; 2228 } 2229 if (!hugepage_vma_check(vma, vma->vm_flags, false, false, true)) { 2230 skip: 2231 progress++; 2232 continue; 2233 } 2234 hstart = round_up(vma->vm_start, HPAGE_PMD_SIZE); 2235 hend = round_down(vma->vm_end, HPAGE_PMD_SIZE); 2236 if (khugepaged_scan.address > hend) 2237 goto skip; 2238 if (khugepaged_scan.address < hstart) 2239 khugepaged_scan.address = hstart; 2240 VM_BUG_ON(khugepaged_scan.address & ~HPAGE_PMD_MASK); 2241 2242 while (khugepaged_scan.address < hend) { 2243 bool mmap_locked = true; 2244 2245 cond_resched(); 2246 if (unlikely(hpage_collapse_test_exit(mm))) 2247 goto breakouterloop; 2248 2249 VM_BUG_ON(khugepaged_scan.address < hstart || 2250 khugepaged_scan.address + HPAGE_PMD_SIZE > 2251 hend); 2252 if (IS_ENABLED(CONFIG_SHMEM) && vma->vm_file) { 2253 struct file *file = get_file(vma->vm_file); 2254 pgoff_t pgoff = linear_page_index(vma, 2255 khugepaged_scan.address); 2256 2257 mmap_read_unlock(mm); 2258 *result = hpage_collapse_scan_file(mm, 2259 khugepaged_scan.address, 2260 file, pgoff, cc); 2261 mmap_locked = false; 2262 fput(file); 2263 } else { 2264 *result = hpage_collapse_scan_pmd(mm, vma, 2265 khugepaged_scan.address, 2266 &mmap_locked, 2267 cc); 2268 } 2269 switch (*result) { 2270 case SCAN_PTE_MAPPED_HUGEPAGE: { 2271 pmd_t *pmd; 2272 2273 *result = find_pmd_or_thp_or_none(mm, 2274 khugepaged_scan.address, 2275 &pmd); 2276 if (*result != SCAN_SUCCEED) 2277 break; 2278 if (!khugepaged_add_pte_mapped_thp(mm, 2279 khugepaged_scan.address)) 2280 break; 2281 } fallthrough; 2282 case SCAN_SUCCEED: 2283 ++khugepaged_pages_collapsed; 2284 break; 2285 default: 2286 break; 2287 } 2288 2289 /* move to next address */ 2290 khugepaged_scan.address += HPAGE_PMD_SIZE; 2291 progress += HPAGE_PMD_NR; 2292 if (!mmap_locked) 2293 /* 2294 * We released mmap_lock so break loop. Note 2295 * that we drop mmap_lock before all hugepage 2296 * allocations, so if allocation fails, we are 2297 * guaranteed to break here and report the 2298 * correct result back to caller. 2299 */ 2300 goto breakouterloop_mmap_lock; 2301 if (progress >= pages) 2302 goto breakouterloop; 2303 } 2304 } 2305 breakouterloop: 2306 mmap_read_unlock(mm); /* exit_mmap will destroy ptes after this */ 2307 breakouterloop_mmap_lock: 2308 2309 spin_lock(&khugepaged_mm_lock); 2310 VM_BUG_ON(khugepaged_scan.mm_slot != mm_slot); 2311 /* 2312 * Release the current mm_slot if this mm is about to die, or 2313 * if we scanned all vmas of this mm. 2314 */ 2315 if (hpage_collapse_test_exit(mm) || !vma) { 2316 /* 2317 * Make sure that if mm_users is reaching zero while 2318 * khugepaged runs here, khugepaged_exit will find 2319 * mm_slot not pointing to the exiting mm. 2320 */ 2321 if (slot->mm_node.next != &khugepaged_scan.mm_head) { 2322 slot = list_entry(slot->mm_node.next, 2323 struct mm_slot, mm_node); 2324 khugepaged_scan.mm_slot = 2325 mm_slot_entry(slot, struct khugepaged_mm_slot, slot); 2326 khugepaged_scan.address = 0; 2327 } else { 2328 khugepaged_scan.mm_slot = NULL; 2329 khugepaged_full_scans++; 2330 } 2331 2332 collect_mm_slot(mm_slot); 2333 } 2334 2335 return progress; 2336 } 2337 2338 static int khugepaged_has_work(void) 2339 { 2340 return !list_empty(&khugepaged_scan.mm_head) && 2341 hugepage_flags_enabled(); 2342 } 2343 2344 static int khugepaged_wait_event(void) 2345 { 2346 return !list_empty(&khugepaged_scan.mm_head) || 2347 kthread_should_stop(); 2348 } 2349 2350 static void khugepaged_do_scan(struct collapse_control *cc) 2351 { 2352 unsigned int progress = 0, pass_through_head = 0; 2353 unsigned int pages = READ_ONCE(khugepaged_pages_to_scan); 2354 bool wait = true; 2355 int result = SCAN_SUCCEED; 2356 2357 lru_add_drain_all(); 2358 2359 while (true) { 2360 cond_resched(); 2361 2362 if (unlikely(kthread_should_stop() || try_to_freeze())) 2363 break; 2364 2365 spin_lock(&khugepaged_mm_lock); 2366 if (!khugepaged_scan.mm_slot) 2367 pass_through_head++; 2368 if (khugepaged_has_work() && 2369 pass_through_head < 2) 2370 progress += khugepaged_scan_mm_slot(pages - progress, 2371 &result, cc); 2372 else 2373 progress = pages; 2374 spin_unlock(&khugepaged_mm_lock); 2375 2376 if (progress >= pages) 2377 break; 2378 2379 if (result == SCAN_ALLOC_HUGE_PAGE_FAIL) { 2380 /* 2381 * If fail to allocate the first time, try to sleep for 2382 * a while. When hit again, cancel the scan. 2383 */ 2384 if (!wait) 2385 break; 2386 wait = false; 2387 khugepaged_alloc_sleep(); 2388 } 2389 } 2390 } 2391 2392 static bool khugepaged_should_wakeup(void) 2393 { 2394 return kthread_should_stop() || 2395 time_after_eq(jiffies, khugepaged_sleep_expire); 2396 } 2397 2398 static void khugepaged_wait_work(void) 2399 { 2400 if (khugepaged_has_work()) { 2401 const unsigned long scan_sleep_jiffies = 2402 msecs_to_jiffies(khugepaged_scan_sleep_millisecs); 2403 2404 if (!scan_sleep_jiffies) 2405 return; 2406 2407 khugepaged_sleep_expire = jiffies + scan_sleep_jiffies; 2408 wait_event_freezable_timeout(khugepaged_wait, 2409 khugepaged_should_wakeup(), 2410 scan_sleep_jiffies); 2411 return; 2412 } 2413 2414 if (hugepage_flags_enabled()) 2415 wait_event_freezable(khugepaged_wait, khugepaged_wait_event()); 2416 } 2417 2418 static int khugepaged(void *none) 2419 { 2420 struct khugepaged_mm_slot *mm_slot; 2421 2422 set_freezable(); 2423 set_user_nice(current, MAX_NICE); 2424 2425 while (!kthread_should_stop()) { 2426 khugepaged_do_scan(&khugepaged_collapse_control); 2427 khugepaged_wait_work(); 2428 } 2429 2430 spin_lock(&khugepaged_mm_lock); 2431 mm_slot = khugepaged_scan.mm_slot; 2432 khugepaged_scan.mm_slot = NULL; 2433 if (mm_slot) 2434 collect_mm_slot(mm_slot); 2435 spin_unlock(&khugepaged_mm_lock); 2436 return 0; 2437 } 2438 2439 static void set_recommended_min_free_kbytes(void) 2440 { 2441 struct zone *zone; 2442 int nr_zones = 0; 2443 unsigned long recommended_min; 2444 2445 if (!hugepage_flags_enabled()) { 2446 calculate_min_free_kbytes(); 2447 goto update_wmarks; 2448 } 2449 2450 for_each_populated_zone(zone) { 2451 /* 2452 * We don't need to worry about fragmentation of 2453 * ZONE_MOVABLE since it only has movable pages. 2454 */ 2455 if (zone_idx(zone) > gfp_zone(GFP_USER)) 2456 continue; 2457 2458 nr_zones++; 2459 } 2460 2461 /* Ensure 2 pageblocks are free to assist fragmentation avoidance */ 2462 recommended_min = pageblock_nr_pages * nr_zones * 2; 2463 2464 /* 2465 * Make sure that on average at least two pageblocks are almost free 2466 * of another type, one for a migratetype to fall back to and a 2467 * second to avoid subsequent fallbacks of other types There are 3 2468 * MIGRATE_TYPES we care about. 2469 */ 2470 recommended_min += pageblock_nr_pages * nr_zones * 2471 MIGRATE_PCPTYPES * MIGRATE_PCPTYPES; 2472 2473 /* don't ever allow to reserve more than 5% of the lowmem */ 2474 recommended_min = min(recommended_min, 2475 (unsigned long) nr_free_buffer_pages() / 20); 2476 recommended_min <<= (PAGE_SHIFT-10); 2477 2478 if (recommended_min > min_free_kbytes) { 2479 if (user_min_free_kbytes >= 0) 2480 pr_info("raising min_free_kbytes from %d to %lu to help transparent hugepage allocations\n", 2481 min_free_kbytes, recommended_min); 2482 2483 min_free_kbytes = recommended_min; 2484 } 2485 2486 update_wmarks: 2487 setup_per_zone_wmarks(); 2488 } 2489 2490 int start_stop_khugepaged(void) 2491 { 2492 int err = 0; 2493 2494 mutex_lock(&khugepaged_mutex); 2495 if (hugepage_flags_enabled()) { 2496 if (!khugepaged_thread) 2497 khugepaged_thread = kthread_run(khugepaged, NULL, 2498 "khugepaged"); 2499 if (IS_ERR(khugepaged_thread)) { 2500 pr_err("khugepaged: kthread_run(khugepaged) failed\n"); 2501 err = PTR_ERR(khugepaged_thread); 2502 khugepaged_thread = NULL; 2503 goto fail; 2504 } 2505 2506 if (!list_empty(&khugepaged_scan.mm_head)) 2507 wake_up_interruptible(&khugepaged_wait); 2508 } else if (khugepaged_thread) { 2509 kthread_stop(khugepaged_thread); 2510 khugepaged_thread = NULL; 2511 } 2512 set_recommended_min_free_kbytes(); 2513 fail: 2514 mutex_unlock(&khugepaged_mutex); 2515 return err; 2516 } 2517 2518 void khugepaged_min_free_kbytes_update(void) 2519 { 2520 mutex_lock(&khugepaged_mutex); 2521 if (hugepage_flags_enabled() && khugepaged_thread) 2522 set_recommended_min_free_kbytes(); 2523 mutex_unlock(&khugepaged_mutex); 2524 } 2525 2526 static int madvise_collapse_errno(enum scan_result r) 2527 { 2528 /* 2529 * MADV_COLLAPSE breaks from existing madvise(2) conventions to provide 2530 * actionable feedback to caller, so they may take an appropriate 2531 * fallback measure depending on the nature of the failure. 2532 */ 2533 switch (r) { 2534 case SCAN_ALLOC_HUGE_PAGE_FAIL: 2535 return -ENOMEM; 2536 case SCAN_CGROUP_CHARGE_FAIL: 2537 return -EBUSY; 2538 /* Resource temporary unavailable - trying again might succeed */ 2539 case SCAN_PAGE_LOCK: 2540 case SCAN_PAGE_LRU: 2541 case SCAN_DEL_PAGE_LRU: 2542 return -EAGAIN; 2543 /* 2544 * Other: Trying again likely not to succeed / error intrinsic to 2545 * specified memory range. khugepaged likely won't be able to collapse 2546 * either. 2547 */ 2548 default: 2549 return -EINVAL; 2550 } 2551 } 2552 2553 int madvise_collapse(struct vm_area_struct *vma, struct vm_area_struct **prev, 2554 unsigned long start, unsigned long end) 2555 { 2556 struct collapse_control *cc; 2557 struct mm_struct *mm = vma->vm_mm; 2558 unsigned long hstart, hend, addr; 2559 int thps = 0, last_fail = SCAN_FAIL; 2560 bool mmap_locked = true; 2561 2562 BUG_ON(vma->vm_start > start); 2563 BUG_ON(vma->vm_end < end); 2564 2565 *prev = vma; 2566 2567 if (!hugepage_vma_check(vma, vma->vm_flags, false, false, false)) 2568 return -EINVAL; 2569 2570 cc = kmalloc(sizeof(*cc), GFP_KERNEL); 2571 if (!cc) 2572 return -ENOMEM; 2573 cc->is_khugepaged = false; 2574 2575 mmgrab(mm); 2576 lru_add_drain_all(); 2577 2578 hstart = (start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK; 2579 hend = end & HPAGE_PMD_MASK; 2580 2581 for (addr = hstart; addr < hend; addr += HPAGE_PMD_SIZE) { 2582 int result = SCAN_FAIL; 2583 2584 if (!mmap_locked) { 2585 cond_resched(); 2586 mmap_read_lock(mm); 2587 mmap_locked = true; 2588 result = hugepage_vma_revalidate(mm, addr, false, &vma, 2589 cc); 2590 if (result != SCAN_SUCCEED) { 2591 last_fail = result; 2592 goto out_nolock; 2593 } 2594 2595 hend = vma->vm_end & HPAGE_PMD_MASK; 2596 } 2597 mmap_assert_locked(mm); 2598 memset(cc->node_load, 0, sizeof(cc->node_load)); 2599 nodes_clear(cc->alloc_nmask); 2600 if (IS_ENABLED(CONFIG_SHMEM) && vma->vm_file) { 2601 struct file *file = get_file(vma->vm_file); 2602 pgoff_t pgoff = linear_page_index(vma, addr); 2603 2604 mmap_read_unlock(mm); 2605 mmap_locked = false; 2606 result = hpage_collapse_scan_file(mm, addr, file, pgoff, 2607 cc); 2608 fput(file); 2609 } else { 2610 result = hpage_collapse_scan_pmd(mm, vma, addr, 2611 &mmap_locked, cc); 2612 } 2613 if (!mmap_locked) 2614 *prev = NULL; /* Tell caller we dropped mmap_lock */ 2615 2616 handle_result: 2617 switch (result) { 2618 case SCAN_SUCCEED: 2619 case SCAN_PMD_MAPPED: 2620 ++thps; 2621 break; 2622 case SCAN_PTE_MAPPED_HUGEPAGE: 2623 BUG_ON(mmap_locked); 2624 BUG_ON(*prev); 2625 mmap_write_lock(mm); 2626 result = collapse_pte_mapped_thp(mm, addr, true); 2627 mmap_write_unlock(mm); 2628 goto handle_result; 2629 /* Whitelisted set of results where continuing OK */ 2630 case SCAN_PMD_NULL: 2631 case SCAN_PTE_NON_PRESENT: 2632 case SCAN_PTE_UFFD_WP: 2633 case SCAN_PAGE_RO: 2634 case SCAN_LACK_REFERENCED_PAGE: 2635 case SCAN_PAGE_NULL: 2636 case SCAN_PAGE_COUNT: 2637 case SCAN_PAGE_LOCK: 2638 case SCAN_PAGE_COMPOUND: 2639 case SCAN_PAGE_LRU: 2640 case SCAN_DEL_PAGE_LRU: 2641 last_fail = result; 2642 break; 2643 default: 2644 last_fail = result; 2645 /* Other error, exit */ 2646 goto out_maybelock; 2647 } 2648 } 2649 2650 out_maybelock: 2651 /* Caller expects us to hold mmap_lock on return */ 2652 if (!mmap_locked) 2653 mmap_read_lock(mm); 2654 out_nolock: 2655 mmap_assert_locked(mm); 2656 mmdrop(mm); 2657 kfree(cc); 2658 2659 return thps == ((hend - hstart) >> HPAGE_PMD_SHIFT) ? 0 2660 : madvise_collapse_errno(last_fail); 2661 } 2662