1 // SPDX-License-Identifier: GPL-2.0 2 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 3 4 #include <linux/mm.h> 5 #include <linux/sched.h> 6 #include <linux/sched/mm.h> 7 #include <linux/sched/coredump.h> 8 #include <linux/mmu_notifier.h> 9 #include <linux/rmap.h> 10 #include <linux/swap.h> 11 #include <linux/mm_inline.h> 12 #include <linux/kthread.h> 13 #include <linux/khugepaged.h> 14 #include <linux/freezer.h> 15 #include <linux/mman.h> 16 #include <linux/hashtable.h> 17 #include <linux/userfaultfd_k.h> 18 #include <linux/page_idle.h> 19 #include <linux/page_table_check.h> 20 #include <linux/swapops.h> 21 #include <linux/shmem_fs.h> 22 23 #include <asm/tlb.h> 24 #include <asm/pgalloc.h> 25 #include "internal.h" 26 #include "mm_slot.h" 27 28 enum scan_result { 29 SCAN_FAIL, 30 SCAN_SUCCEED, 31 SCAN_PMD_NULL, 32 SCAN_PMD_NONE, 33 SCAN_PMD_MAPPED, 34 SCAN_EXCEED_NONE_PTE, 35 SCAN_EXCEED_SWAP_PTE, 36 SCAN_EXCEED_SHARED_PTE, 37 SCAN_PTE_NON_PRESENT, 38 SCAN_PTE_UFFD_WP, 39 SCAN_PTE_MAPPED_HUGEPAGE, 40 SCAN_PAGE_RO, 41 SCAN_LACK_REFERENCED_PAGE, 42 SCAN_PAGE_NULL, 43 SCAN_SCAN_ABORT, 44 SCAN_PAGE_COUNT, 45 SCAN_PAGE_LRU, 46 SCAN_PAGE_LOCK, 47 SCAN_PAGE_ANON, 48 SCAN_PAGE_COMPOUND, 49 SCAN_ANY_PROCESS, 50 SCAN_VMA_NULL, 51 SCAN_VMA_CHECK, 52 SCAN_ADDRESS_RANGE, 53 SCAN_DEL_PAGE_LRU, 54 SCAN_ALLOC_HUGE_PAGE_FAIL, 55 SCAN_CGROUP_CHARGE_FAIL, 56 SCAN_TRUNCATED, 57 SCAN_PAGE_HAS_PRIVATE, 58 }; 59 60 #define CREATE_TRACE_POINTS 61 #include <trace/events/huge_memory.h> 62 63 static struct task_struct *khugepaged_thread __read_mostly; 64 static DEFINE_MUTEX(khugepaged_mutex); 65 66 /* default scan 8*512 pte (or vmas) every 30 second */ 67 static unsigned int khugepaged_pages_to_scan __read_mostly; 68 static unsigned int khugepaged_pages_collapsed; 69 static unsigned int khugepaged_full_scans; 70 static unsigned int khugepaged_scan_sleep_millisecs __read_mostly = 10000; 71 /* during fragmentation poll the hugepage allocator once every minute */ 72 static unsigned int khugepaged_alloc_sleep_millisecs __read_mostly = 60000; 73 static unsigned long khugepaged_sleep_expire; 74 static DEFINE_SPINLOCK(khugepaged_mm_lock); 75 static DECLARE_WAIT_QUEUE_HEAD(khugepaged_wait); 76 /* 77 * default collapse hugepages if there is at least one pte mapped like 78 * it would have happened if the vma was large enough during page 79 * fault. 80 * 81 * Note that these are only respected if collapse was initiated by khugepaged. 82 */ 83 static unsigned int khugepaged_max_ptes_none __read_mostly; 84 static unsigned int khugepaged_max_ptes_swap __read_mostly; 85 static unsigned int khugepaged_max_ptes_shared __read_mostly; 86 87 #define MM_SLOTS_HASH_BITS 10 88 static __read_mostly DEFINE_HASHTABLE(mm_slots_hash, MM_SLOTS_HASH_BITS); 89 90 static struct kmem_cache *mm_slot_cache __read_mostly; 91 92 #define MAX_PTE_MAPPED_THP 8 93 94 struct collapse_control { 95 bool is_khugepaged; 96 97 /* Num pages scanned per node */ 98 u32 node_load[MAX_NUMNODES]; 99 100 /* nodemask for allocation fallback */ 101 nodemask_t alloc_nmask; 102 }; 103 104 /** 105 * struct khugepaged_mm_slot - khugepaged information per mm that is being scanned 106 * @slot: hash lookup from mm to mm_slot 107 * @nr_pte_mapped_thp: number of pte mapped THP 108 * @pte_mapped_thp: address array corresponding pte mapped THP 109 */ 110 struct khugepaged_mm_slot { 111 struct mm_slot slot; 112 113 /* pte-mapped THP in this mm */ 114 int nr_pte_mapped_thp; 115 unsigned long pte_mapped_thp[MAX_PTE_MAPPED_THP]; 116 }; 117 118 /** 119 * struct khugepaged_scan - cursor for scanning 120 * @mm_head: the head of the mm list to scan 121 * @mm_slot: the current mm_slot we are scanning 122 * @address: the next address inside that to be scanned 123 * 124 * There is only the one khugepaged_scan instance of this cursor structure. 125 */ 126 struct khugepaged_scan { 127 struct list_head mm_head; 128 struct khugepaged_mm_slot *mm_slot; 129 unsigned long address; 130 }; 131 132 static struct khugepaged_scan khugepaged_scan = { 133 .mm_head = LIST_HEAD_INIT(khugepaged_scan.mm_head), 134 }; 135 136 #ifdef CONFIG_SYSFS 137 static ssize_t scan_sleep_millisecs_show(struct kobject *kobj, 138 struct kobj_attribute *attr, 139 char *buf) 140 { 141 return sysfs_emit(buf, "%u\n", khugepaged_scan_sleep_millisecs); 142 } 143 144 static ssize_t scan_sleep_millisecs_store(struct kobject *kobj, 145 struct kobj_attribute *attr, 146 const char *buf, size_t count) 147 { 148 unsigned int msecs; 149 int err; 150 151 err = kstrtouint(buf, 10, &msecs); 152 if (err) 153 return -EINVAL; 154 155 khugepaged_scan_sleep_millisecs = msecs; 156 khugepaged_sleep_expire = 0; 157 wake_up_interruptible(&khugepaged_wait); 158 159 return count; 160 } 161 static struct kobj_attribute scan_sleep_millisecs_attr = 162 __ATTR_RW(scan_sleep_millisecs); 163 164 static ssize_t alloc_sleep_millisecs_show(struct kobject *kobj, 165 struct kobj_attribute *attr, 166 char *buf) 167 { 168 return sysfs_emit(buf, "%u\n", khugepaged_alloc_sleep_millisecs); 169 } 170 171 static ssize_t alloc_sleep_millisecs_store(struct kobject *kobj, 172 struct kobj_attribute *attr, 173 const char *buf, size_t count) 174 { 175 unsigned int msecs; 176 int err; 177 178 err = kstrtouint(buf, 10, &msecs); 179 if (err) 180 return -EINVAL; 181 182 khugepaged_alloc_sleep_millisecs = msecs; 183 khugepaged_sleep_expire = 0; 184 wake_up_interruptible(&khugepaged_wait); 185 186 return count; 187 } 188 static struct kobj_attribute alloc_sleep_millisecs_attr = 189 __ATTR_RW(alloc_sleep_millisecs); 190 191 static ssize_t pages_to_scan_show(struct kobject *kobj, 192 struct kobj_attribute *attr, 193 char *buf) 194 { 195 return sysfs_emit(buf, "%u\n", khugepaged_pages_to_scan); 196 } 197 static ssize_t pages_to_scan_store(struct kobject *kobj, 198 struct kobj_attribute *attr, 199 const char *buf, size_t count) 200 { 201 unsigned int pages; 202 int err; 203 204 err = kstrtouint(buf, 10, &pages); 205 if (err || !pages) 206 return -EINVAL; 207 208 khugepaged_pages_to_scan = pages; 209 210 return count; 211 } 212 static struct kobj_attribute pages_to_scan_attr = 213 __ATTR_RW(pages_to_scan); 214 215 static ssize_t pages_collapsed_show(struct kobject *kobj, 216 struct kobj_attribute *attr, 217 char *buf) 218 { 219 return sysfs_emit(buf, "%u\n", khugepaged_pages_collapsed); 220 } 221 static struct kobj_attribute pages_collapsed_attr = 222 __ATTR_RO(pages_collapsed); 223 224 static ssize_t full_scans_show(struct kobject *kobj, 225 struct kobj_attribute *attr, 226 char *buf) 227 { 228 return sysfs_emit(buf, "%u\n", khugepaged_full_scans); 229 } 230 static struct kobj_attribute full_scans_attr = 231 __ATTR_RO(full_scans); 232 233 static ssize_t defrag_show(struct kobject *kobj, 234 struct kobj_attribute *attr, char *buf) 235 { 236 return single_hugepage_flag_show(kobj, attr, buf, 237 TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG); 238 } 239 static ssize_t defrag_store(struct kobject *kobj, 240 struct kobj_attribute *attr, 241 const char *buf, size_t count) 242 { 243 return single_hugepage_flag_store(kobj, attr, buf, count, 244 TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG); 245 } 246 static struct kobj_attribute khugepaged_defrag_attr = 247 __ATTR_RW(defrag); 248 249 /* 250 * max_ptes_none controls if khugepaged should collapse hugepages over 251 * any unmapped ptes in turn potentially increasing the memory 252 * footprint of the vmas. When max_ptes_none is 0 khugepaged will not 253 * reduce the available free memory in the system as it 254 * runs. Increasing max_ptes_none will instead potentially reduce the 255 * free memory in the system during the khugepaged scan. 256 */ 257 static ssize_t max_ptes_none_show(struct kobject *kobj, 258 struct kobj_attribute *attr, 259 char *buf) 260 { 261 return sysfs_emit(buf, "%u\n", khugepaged_max_ptes_none); 262 } 263 static ssize_t max_ptes_none_store(struct kobject *kobj, 264 struct kobj_attribute *attr, 265 const char *buf, size_t count) 266 { 267 int err; 268 unsigned long max_ptes_none; 269 270 err = kstrtoul(buf, 10, &max_ptes_none); 271 if (err || max_ptes_none > HPAGE_PMD_NR - 1) 272 return -EINVAL; 273 274 khugepaged_max_ptes_none = max_ptes_none; 275 276 return count; 277 } 278 static struct kobj_attribute khugepaged_max_ptes_none_attr = 279 __ATTR_RW(max_ptes_none); 280 281 static ssize_t max_ptes_swap_show(struct kobject *kobj, 282 struct kobj_attribute *attr, 283 char *buf) 284 { 285 return sysfs_emit(buf, "%u\n", khugepaged_max_ptes_swap); 286 } 287 288 static ssize_t max_ptes_swap_store(struct kobject *kobj, 289 struct kobj_attribute *attr, 290 const char *buf, size_t count) 291 { 292 int err; 293 unsigned long max_ptes_swap; 294 295 err = kstrtoul(buf, 10, &max_ptes_swap); 296 if (err || max_ptes_swap > HPAGE_PMD_NR - 1) 297 return -EINVAL; 298 299 khugepaged_max_ptes_swap = max_ptes_swap; 300 301 return count; 302 } 303 304 static struct kobj_attribute khugepaged_max_ptes_swap_attr = 305 __ATTR_RW(max_ptes_swap); 306 307 static ssize_t max_ptes_shared_show(struct kobject *kobj, 308 struct kobj_attribute *attr, 309 char *buf) 310 { 311 return sysfs_emit(buf, "%u\n", khugepaged_max_ptes_shared); 312 } 313 314 static ssize_t max_ptes_shared_store(struct kobject *kobj, 315 struct kobj_attribute *attr, 316 const char *buf, size_t count) 317 { 318 int err; 319 unsigned long max_ptes_shared; 320 321 err = kstrtoul(buf, 10, &max_ptes_shared); 322 if (err || max_ptes_shared > HPAGE_PMD_NR - 1) 323 return -EINVAL; 324 325 khugepaged_max_ptes_shared = max_ptes_shared; 326 327 return count; 328 } 329 330 static struct kobj_attribute khugepaged_max_ptes_shared_attr = 331 __ATTR_RW(max_ptes_shared); 332 333 static struct attribute *khugepaged_attr[] = { 334 &khugepaged_defrag_attr.attr, 335 &khugepaged_max_ptes_none_attr.attr, 336 &khugepaged_max_ptes_swap_attr.attr, 337 &khugepaged_max_ptes_shared_attr.attr, 338 &pages_to_scan_attr.attr, 339 &pages_collapsed_attr.attr, 340 &full_scans_attr.attr, 341 &scan_sleep_millisecs_attr.attr, 342 &alloc_sleep_millisecs_attr.attr, 343 NULL, 344 }; 345 346 struct attribute_group khugepaged_attr_group = { 347 .attrs = khugepaged_attr, 348 .name = "khugepaged", 349 }; 350 #endif /* CONFIG_SYSFS */ 351 352 int hugepage_madvise(struct vm_area_struct *vma, 353 unsigned long *vm_flags, int advice) 354 { 355 switch (advice) { 356 case MADV_HUGEPAGE: 357 #ifdef CONFIG_S390 358 /* 359 * qemu blindly sets MADV_HUGEPAGE on all allocations, but s390 360 * can't handle this properly after s390_enable_sie, so we simply 361 * ignore the madvise to prevent qemu from causing a SIGSEGV. 362 */ 363 if (mm_has_pgste(vma->vm_mm)) 364 return 0; 365 #endif 366 *vm_flags &= ~VM_NOHUGEPAGE; 367 *vm_flags |= VM_HUGEPAGE; 368 /* 369 * If the vma become good for khugepaged to scan, 370 * register it here without waiting a page fault that 371 * may not happen any time soon. 372 */ 373 khugepaged_enter_vma(vma, *vm_flags); 374 break; 375 case MADV_NOHUGEPAGE: 376 *vm_flags &= ~VM_HUGEPAGE; 377 *vm_flags |= VM_NOHUGEPAGE; 378 /* 379 * Setting VM_NOHUGEPAGE will prevent khugepaged from scanning 380 * this vma even if we leave the mm registered in khugepaged if 381 * it got registered before VM_NOHUGEPAGE was set. 382 */ 383 break; 384 } 385 386 return 0; 387 } 388 389 int __init khugepaged_init(void) 390 { 391 mm_slot_cache = kmem_cache_create("khugepaged_mm_slot", 392 sizeof(struct khugepaged_mm_slot), 393 __alignof__(struct khugepaged_mm_slot), 394 0, NULL); 395 if (!mm_slot_cache) 396 return -ENOMEM; 397 398 khugepaged_pages_to_scan = HPAGE_PMD_NR * 8; 399 khugepaged_max_ptes_none = HPAGE_PMD_NR - 1; 400 khugepaged_max_ptes_swap = HPAGE_PMD_NR / 8; 401 khugepaged_max_ptes_shared = HPAGE_PMD_NR / 2; 402 403 return 0; 404 } 405 406 void __init khugepaged_destroy(void) 407 { 408 kmem_cache_destroy(mm_slot_cache); 409 } 410 411 static inline int hpage_collapse_test_exit(struct mm_struct *mm) 412 { 413 return atomic_read(&mm->mm_users) == 0; 414 } 415 416 void __khugepaged_enter(struct mm_struct *mm) 417 { 418 struct khugepaged_mm_slot *mm_slot; 419 struct mm_slot *slot; 420 int wakeup; 421 422 mm_slot = mm_slot_alloc(mm_slot_cache); 423 if (!mm_slot) 424 return; 425 426 slot = &mm_slot->slot; 427 428 /* __khugepaged_exit() must not run from under us */ 429 VM_BUG_ON_MM(hpage_collapse_test_exit(mm), mm); 430 if (unlikely(test_and_set_bit(MMF_VM_HUGEPAGE, &mm->flags))) { 431 mm_slot_free(mm_slot_cache, mm_slot); 432 return; 433 } 434 435 spin_lock(&khugepaged_mm_lock); 436 mm_slot_insert(mm_slots_hash, mm, slot); 437 /* 438 * Insert just behind the scanning cursor, to let the area settle 439 * down a little. 440 */ 441 wakeup = list_empty(&khugepaged_scan.mm_head); 442 list_add_tail(&slot->mm_node, &khugepaged_scan.mm_head); 443 spin_unlock(&khugepaged_mm_lock); 444 445 mmgrab(mm); 446 if (wakeup) 447 wake_up_interruptible(&khugepaged_wait); 448 } 449 450 void khugepaged_enter_vma(struct vm_area_struct *vma, 451 unsigned long vm_flags) 452 { 453 if (!test_bit(MMF_VM_HUGEPAGE, &vma->vm_mm->flags) && 454 hugepage_flags_enabled()) { 455 if (hugepage_vma_check(vma, vm_flags, false, false, true)) 456 __khugepaged_enter(vma->vm_mm); 457 } 458 } 459 460 void __khugepaged_exit(struct mm_struct *mm) 461 { 462 struct khugepaged_mm_slot *mm_slot; 463 struct mm_slot *slot; 464 int free = 0; 465 466 spin_lock(&khugepaged_mm_lock); 467 slot = mm_slot_lookup(mm_slots_hash, mm); 468 mm_slot = mm_slot_entry(slot, struct khugepaged_mm_slot, slot); 469 if (mm_slot && khugepaged_scan.mm_slot != mm_slot) { 470 hash_del(&slot->hash); 471 list_del(&slot->mm_node); 472 free = 1; 473 } 474 spin_unlock(&khugepaged_mm_lock); 475 476 if (free) { 477 clear_bit(MMF_VM_HUGEPAGE, &mm->flags); 478 mm_slot_free(mm_slot_cache, mm_slot); 479 mmdrop(mm); 480 } else if (mm_slot) { 481 /* 482 * This is required to serialize against 483 * hpage_collapse_test_exit() (which is guaranteed to run 484 * under mmap sem read mode). Stop here (after we return all 485 * pagetables will be destroyed) until khugepaged has finished 486 * working on the pagetables under the mmap_lock. 487 */ 488 mmap_write_lock(mm); 489 mmap_write_unlock(mm); 490 } 491 } 492 493 static void release_pte_page(struct page *page) 494 { 495 mod_node_page_state(page_pgdat(page), 496 NR_ISOLATED_ANON + page_is_file_lru(page), 497 -compound_nr(page)); 498 unlock_page(page); 499 putback_lru_page(page); 500 } 501 502 static void release_pte_pages(pte_t *pte, pte_t *_pte, 503 struct list_head *compound_pagelist) 504 { 505 struct page *page, *tmp; 506 507 while (--_pte >= pte) { 508 pte_t pteval = *_pte; 509 510 page = pte_page(pteval); 511 if (!pte_none(pteval) && !is_zero_pfn(pte_pfn(pteval)) && 512 !PageCompound(page)) 513 release_pte_page(page); 514 } 515 516 list_for_each_entry_safe(page, tmp, compound_pagelist, lru) { 517 list_del(&page->lru); 518 release_pte_page(page); 519 } 520 } 521 522 static bool is_refcount_suitable(struct page *page) 523 { 524 int expected_refcount; 525 526 expected_refcount = total_mapcount(page); 527 if (PageSwapCache(page)) 528 expected_refcount += compound_nr(page); 529 530 return page_count(page) == expected_refcount; 531 } 532 533 static int __collapse_huge_page_isolate(struct vm_area_struct *vma, 534 unsigned long address, 535 pte_t *pte, 536 struct collapse_control *cc, 537 struct list_head *compound_pagelist) 538 { 539 struct page *page = NULL; 540 pte_t *_pte; 541 int none_or_zero = 0, shared = 0, result = SCAN_FAIL, referenced = 0; 542 bool writable = false; 543 544 for (_pte = pte; _pte < pte + HPAGE_PMD_NR; 545 _pte++, address += PAGE_SIZE) { 546 pte_t pteval = *_pte; 547 if (pte_none(pteval) || (pte_present(pteval) && 548 is_zero_pfn(pte_pfn(pteval)))) { 549 ++none_or_zero; 550 if (!userfaultfd_armed(vma) && 551 (!cc->is_khugepaged || 552 none_or_zero <= khugepaged_max_ptes_none)) { 553 continue; 554 } else { 555 result = SCAN_EXCEED_NONE_PTE; 556 count_vm_event(THP_SCAN_EXCEED_NONE_PTE); 557 goto out; 558 } 559 } 560 if (!pte_present(pteval)) { 561 result = SCAN_PTE_NON_PRESENT; 562 goto out; 563 } 564 page = vm_normal_page(vma, address, pteval); 565 if (unlikely(!page) || unlikely(is_zone_device_page(page))) { 566 result = SCAN_PAGE_NULL; 567 goto out; 568 } 569 570 VM_BUG_ON_PAGE(!PageAnon(page), page); 571 572 if (page_mapcount(page) > 1) { 573 ++shared; 574 if (cc->is_khugepaged && 575 shared > khugepaged_max_ptes_shared) { 576 result = SCAN_EXCEED_SHARED_PTE; 577 count_vm_event(THP_SCAN_EXCEED_SHARED_PTE); 578 goto out; 579 } 580 } 581 582 if (PageCompound(page)) { 583 struct page *p; 584 page = compound_head(page); 585 586 /* 587 * Check if we have dealt with the compound page 588 * already 589 */ 590 list_for_each_entry(p, compound_pagelist, lru) { 591 if (page == p) 592 goto next; 593 } 594 } 595 596 /* 597 * We can do it before isolate_lru_page because the 598 * page can't be freed from under us. NOTE: PG_lock 599 * is needed to serialize against split_huge_page 600 * when invoked from the VM. 601 */ 602 if (!trylock_page(page)) { 603 result = SCAN_PAGE_LOCK; 604 goto out; 605 } 606 607 /* 608 * Check if the page has any GUP (or other external) pins. 609 * 610 * The page table that maps the page has been already unlinked 611 * from the page table tree and this process cannot get 612 * an additional pin on the page. 613 * 614 * New pins can come later if the page is shared across fork, 615 * but not from this process. The other process cannot write to 616 * the page, only trigger CoW. 617 */ 618 if (!is_refcount_suitable(page)) { 619 unlock_page(page); 620 result = SCAN_PAGE_COUNT; 621 goto out; 622 } 623 624 /* 625 * Isolate the page to avoid collapsing an hugepage 626 * currently in use by the VM. 627 */ 628 if (isolate_lru_page(page)) { 629 unlock_page(page); 630 result = SCAN_DEL_PAGE_LRU; 631 goto out; 632 } 633 mod_node_page_state(page_pgdat(page), 634 NR_ISOLATED_ANON + page_is_file_lru(page), 635 compound_nr(page)); 636 VM_BUG_ON_PAGE(!PageLocked(page), page); 637 VM_BUG_ON_PAGE(PageLRU(page), page); 638 639 if (PageCompound(page)) 640 list_add_tail(&page->lru, compound_pagelist); 641 next: 642 /* 643 * If collapse was initiated by khugepaged, check that there is 644 * enough young pte to justify collapsing the page 645 */ 646 if (cc->is_khugepaged && 647 (pte_young(pteval) || page_is_young(page) || 648 PageReferenced(page) || mmu_notifier_test_young(vma->vm_mm, 649 address))) 650 referenced++; 651 652 if (pte_write(pteval)) 653 writable = true; 654 } 655 656 if (unlikely(!writable)) { 657 result = SCAN_PAGE_RO; 658 } else if (unlikely(cc->is_khugepaged && !referenced)) { 659 result = SCAN_LACK_REFERENCED_PAGE; 660 } else { 661 result = SCAN_SUCCEED; 662 trace_mm_collapse_huge_page_isolate(page, none_or_zero, 663 referenced, writable, result); 664 return result; 665 } 666 out: 667 release_pte_pages(pte, _pte, compound_pagelist); 668 trace_mm_collapse_huge_page_isolate(page, none_or_zero, 669 referenced, writable, result); 670 return result; 671 } 672 673 static void __collapse_huge_page_copy(pte_t *pte, struct page *page, 674 struct vm_area_struct *vma, 675 unsigned long address, 676 spinlock_t *ptl, 677 struct list_head *compound_pagelist) 678 { 679 struct page *src_page, *tmp; 680 pte_t *_pte; 681 for (_pte = pte; _pte < pte + HPAGE_PMD_NR; 682 _pte++, page++, address += PAGE_SIZE) { 683 pte_t pteval = *_pte; 684 685 if (pte_none(pteval) || is_zero_pfn(pte_pfn(pteval))) { 686 clear_user_highpage(page, address); 687 add_mm_counter(vma->vm_mm, MM_ANONPAGES, 1); 688 if (is_zero_pfn(pte_pfn(pteval))) { 689 /* 690 * ptl mostly unnecessary. 691 */ 692 spin_lock(ptl); 693 ptep_clear(vma->vm_mm, address, _pte); 694 spin_unlock(ptl); 695 } 696 } else { 697 src_page = pte_page(pteval); 698 copy_user_highpage(page, src_page, address, vma); 699 if (!PageCompound(src_page)) 700 release_pte_page(src_page); 701 /* 702 * ptl mostly unnecessary, but preempt has to 703 * be disabled to update the per-cpu stats 704 * inside page_remove_rmap(). 705 */ 706 spin_lock(ptl); 707 ptep_clear(vma->vm_mm, address, _pte); 708 page_remove_rmap(src_page, vma, false); 709 spin_unlock(ptl); 710 free_page_and_swap_cache(src_page); 711 } 712 } 713 714 list_for_each_entry_safe(src_page, tmp, compound_pagelist, lru) { 715 list_del(&src_page->lru); 716 mod_node_page_state(page_pgdat(src_page), 717 NR_ISOLATED_ANON + page_is_file_lru(src_page), 718 -compound_nr(src_page)); 719 unlock_page(src_page); 720 free_swap_cache(src_page); 721 putback_lru_page(src_page); 722 } 723 } 724 725 static void khugepaged_alloc_sleep(void) 726 { 727 DEFINE_WAIT(wait); 728 729 add_wait_queue(&khugepaged_wait, &wait); 730 __set_current_state(TASK_INTERRUPTIBLE|TASK_FREEZABLE); 731 schedule_timeout(msecs_to_jiffies(khugepaged_alloc_sleep_millisecs)); 732 remove_wait_queue(&khugepaged_wait, &wait); 733 } 734 735 struct collapse_control khugepaged_collapse_control = { 736 .is_khugepaged = true, 737 }; 738 739 static bool hpage_collapse_scan_abort(int nid, struct collapse_control *cc) 740 { 741 int i; 742 743 /* 744 * If node_reclaim_mode is disabled, then no extra effort is made to 745 * allocate memory locally. 746 */ 747 if (!node_reclaim_enabled()) 748 return false; 749 750 /* If there is a count for this node already, it must be acceptable */ 751 if (cc->node_load[nid]) 752 return false; 753 754 for (i = 0; i < MAX_NUMNODES; i++) { 755 if (!cc->node_load[i]) 756 continue; 757 if (node_distance(nid, i) > node_reclaim_distance) 758 return true; 759 } 760 return false; 761 } 762 763 #define khugepaged_defrag() \ 764 (transparent_hugepage_flags & \ 765 (1<<TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG)) 766 767 /* Defrag for khugepaged will enter direct reclaim/compaction if necessary */ 768 static inline gfp_t alloc_hugepage_khugepaged_gfpmask(void) 769 { 770 return khugepaged_defrag() ? GFP_TRANSHUGE : GFP_TRANSHUGE_LIGHT; 771 } 772 773 #ifdef CONFIG_NUMA 774 static int hpage_collapse_find_target_node(struct collapse_control *cc) 775 { 776 int nid, target_node = 0, max_value = 0; 777 778 /* find first node with max normal pages hit */ 779 for (nid = 0; nid < MAX_NUMNODES; nid++) 780 if (cc->node_load[nid] > max_value) { 781 max_value = cc->node_load[nid]; 782 target_node = nid; 783 } 784 785 for_each_online_node(nid) { 786 if (max_value == cc->node_load[nid]) 787 node_set(nid, cc->alloc_nmask); 788 } 789 790 return target_node; 791 } 792 #else 793 static int hpage_collapse_find_target_node(struct collapse_control *cc) 794 { 795 return 0; 796 } 797 #endif 798 799 static bool hpage_collapse_alloc_page(struct page **hpage, gfp_t gfp, int node, 800 nodemask_t *nmask) 801 { 802 *hpage = __alloc_pages(gfp, HPAGE_PMD_ORDER, node, nmask); 803 if (unlikely(!*hpage)) { 804 count_vm_event(THP_COLLAPSE_ALLOC_FAILED); 805 return false; 806 } 807 808 prep_transhuge_page(*hpage); 809 count_vm_event(THP_COLLAPSE_ALLOC); 810 return true; 811 } 812 813 /* 814 * If mmap_lock temporarily dropped, revalidate vma 815 * before taking mmap_lock. 816 * Returns enum scan_result value. 817 */ 818 819 static int hugepage_vma_revalidate(struct mm_struct *mm, unsigned long address, 820 bool expect_anon, 821 struct vm_area_struct **vmap, 822 struct collapse_control *cc) 823 { 824 struct vm_area_struct *vma; 825 826 if (unlikely(hpage_collapse_test_exit(mm))) 827 return SCAN_ANY_PROCESS; 828 829 *vmap = vma = find_vma(mm, address); 830 if (!vma) 831 return SCAN_VMA_NULL; 832 833 if (!transhuge_vma_suitable(vma, address)) 834 return SCAN_ADDRESS_RANGE; 835 if (!hugepage_vma_check(vma, vma->vm_flags, false, false, 836 cc->is_khugepaged)) 837 return SCAN_VMA_CHECK; 838 /* 839 * Anon VMA expected, the address may be unmapped then 840 * remapped to file after khugepaged reaquired the mmap_lock. 841 * 842 * hugepage_vma_check may return true for qualified file 843 * vmas. 844 */ 845 if (expect_anon && (!(*vmap)->anon_vma || !vma_is_anonymous(*vmap))) 846 return SCAN_PAGE_ANON; 847 return SCAN_SUCCEED; 848 } 849 850 static int find_pmd_or_thp_or_none(struct mm_struct *mm, 851 unsigned long address, 852 pmd_t **pmd) 853 { 854 pmd_t pmde; 855 856 *pmd = mm_find_pmd(mm, address); 857 if (!*pmd) 858 return SCAN_PMD_NULL; 859 860 pmde = pmd_read_atomic(*pmd); 861 862 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 863 /* See comments in pmd_none_or_trans_huge_or_clear_bad() */ 864 barrier(); 865 #endif 866 if (pmd_none(pmde)) 867 return SCAN_PMD_NONE; 868 if (pmd_trans_huge(pmde)) 869 return SCAN_PMD_MAPPED; 870 if (pmd_bad(pmde)) 871 return SCAN_PMD_NULL; 872 return SCAN_SUCCEED; 873 } 874 875 static int check_pmd_still_valid(struct mm_struct *mm, 876 unsigned long address, 877 pmd_t *pmd) 878 { 879 pmd_t *new_pmd; 880 int result = find_pmd_or_thp_or_none(mm, address, &new_pmd); 881 882 if (result != SCAN_SUCCEED) 883 return result; 884 if (new_pmd != pmd) 885 return SCAN_FAIL; 886 return SCAN_SUCCEED; 887 } 888 889 /* 890 * Bring missing pages in from swap, to complete THP collapse. 891 * Only done if hpage_collapse_scan_pmd believes it is worthwhile. 892 * 893 * Called and returns without pte mapped or spinlocks held. 894 * Note that if false is returned, mmap_lock will be released. 895 */ 896 897 static int __collapse_huge_page_swapin(struct mm_struct *mm, 898 struct vm_area_struct *vma, 899 unsigned long haddr, pmd_t *pmd, 900 int referenced) 901 { 902 int swapped_in = 0; 903 vm_fault_t ret = 0; 904 unsigned long address, end = haddr + (HPAGE_PMD_NR * PAGE_SIZE); 905 906 for (address = haddr; address < end; address += PAGE_SIZE) { 907 struct vm_fault vmf = { 908 .vma = vma, 909 .address = address, 910 .pgoff = linear_page_index(vma, haddr), 911 .flags = FAULT_FLAG_ALLOW_RETRY, 912 .pmd = pmd, 913 }; 914 915 vmf.pte = pte_offset_map(pmd, address); 916 vmf.orig_pte = *vmf.pte; 917 if (!is_swap_pte(vmf.orig_pte)) { 918 pte_unmap(vmf.pte); 919 continue; 920 } 921 ret = do_swap_page(&vmf); 922 923 /* 924 * do_swap_page returns VM_FAULT_RETRY with released mmap_lock. 925 * Note we treat VM_FAULT_RETRY as VM_FAULT_ERROR here because 926 * we do not retry here and swap entry will remain in pagetable 927 * resulting in later failure. 928 */ 929 if (ret & VM_FAULT_RETRY) { 930 trace_mm_collapse_huge_page_swapin(mm, swapped_in, referenced, 0); 931 /* Likely, but not guaranteed, that page lock failed */ 932 return SCAN_PAGE_LOCK; 933 } 934 if (ret & VM_FAULT_ERROR) { 935 mmap_read_unlock(mm); 936 trace_mm_collapse_huge_page_swapin(mm, swapped_in, referenced, 0); 937 return SCAN_FAIL; 938 } 939 swapped_in++; 940 } 941 942 /* Drain LRU add pagevec to remove extra pin on the swapped in pages */ 943 if (swapped_in) 944 lru_add_drain(); 945 946 trace_mm_collapse_huge_page_swapin(mm, swapped_in, referenced, 1); 947 return SCAN_SUCCEED; 948 } 949 950 static int alloc_charge_hpage(struct page **hpage, struct mm_struct *mm, 951 struct collapse_control *cc) 952 { 953 gfp_t gfp = (cc->is_khugepaged ? alloc_hugepage_khugepaged_gfpmask() : 954 GFP_TRANSHUGE); 955 int node = hpage_collapse_find_target_node(cc); 956 957 if (!hpage_collapse_alloc_page(hpage, gfp, node, &cc->alloc_nmask)) 958 return SCAN_ALLOC_HUGE_PAGE_FAIL; 959 if (unlikely(mem_cgroup_charge(page_folio(*hpage), mm, gfp))) 960 return SCAN_CGROUP_CHARGE_FAIL; 961 count_memcg_page_event(*hpage, THP_COLLAPSE_ALLOC); 962 return SCAN_SUCCEED; 963 } 964 965 static int collapse_huge_page(struct mm_struct *mm, unsigned long address, 966 int referenced, int unmapped, 967 struct collapse_control *cc) 968 { 969 LIST_HEAD(compound_pagelist); 970 pmd_t *pmd, _pmd; 971 pte_t *pte; 972 pgtable_t pgtable; 973 struct page *hpage; 974 spinlock_t *pmd_ptl, *pte_ptl; 975 int result = SCAN_FAIL; 976 struct vm_area_struct *vma; 977 struct mmu_notifier_range range; 978 979 VM_BUG_ON(address & ~HPAGE_PMD_MASK); 980 981 /* 982 * Before allocating the hugepage, release the mmap_lock read lock. 983 * The allocation can take potentially a long time if it involves 984 * sync compaction, and we do not need to hold the mmap_lock during 985 * that. We will recheck the vma after taking it again in write mode. 986 */ 987 mmap_read_unlock(mm); 988 989 result = alloc_charge_hpage(&hpage, mm, cc); 990 if (result != SCAN_SUCCEED) 991 goto out_nolock; 992 993 mmap_read_lock(mm); 994 result = hugepage_vma_revalidate(mm, address, true, &vma, cc); 995 if (result != SCAN_SUCCEED) { 996 mmap_read_unlock(mm); 997 goto out_nolock; 998 } 999 1000 result = find_pmd_or_thp_or_none(mm, address, &pmd); 1001 if (result != SCAN_SUCCEED) { 1002 mmap_read_unlock(mm); 1003 goto out_nolock; 1004 } 1005 1006 if (unmapped) { 1007 /* 1008 * __collapse_huge_page_swapin will return with mmap_lock 1009 * released when it fails. So we jump out_nolock directly in 1010 * that case. Continuing to collapse causes inconsistency. 1011 */ 1012 result = __collapse_huge_page_swapin(mm, vma, address, pmd, 1013 referenced); 1014 if (result != SCAN_SUCCEED) 1015 goto out_nolock; 1016 } 1017 1018 mmap_read_unlock(mm); 1019 /* 1020 * Prevent all access to pagetables with the exception of 1021 * gup_fast later handled by the ptep_clear_flush and the VM 1022 * handled by the anon_vma lock + PG_lock. 1023 */ 1024 mmap_write_lock(mm); 1025 result = hugepage_vma_revalidate(mm, address, true, &vma, cc); 1026 if (result != SCAN_SUCCEED) 1027 goto out_up_write; 1028 /* check if the pmd is still valid */ 1029 result = check_pmd_still_valid(mm, address, pmd); 1030 if (result != SCAN_SUCCEED) 1031 goto out_up_write; 1032 1033 anon_vma_lock_write(vma->anon_vma); 1034 1035 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, NULL, mm, 1036 address, address + HPAGE_PMD_SIZE); 1037 mmu_notifier_invalidate_range_start(&range); 1038 1039 pte = pte_offset_map(pmd, address); 1040 pte_ptl = pte_lockptr(mm, pmd); 1041 1042 pmd_ptl = pmd_lock(mm, pmd); /* probably unnecessary */ 1043 /* 1044 * This removes any huge TLB entry from the CPU so we won't allow 1045 * huge and small TLB entries for the same virtual address to 1046 * avoid the risk of CPU bugs in that area. 1047 * 1048 * Parallel fast GUP is fine since fast GUP will back off when 1049 * it detects PMD is changed. 1050 */ 1051 _pmd = pmdp_collapse_flush(vma, address, pmd); 1052 spin_unlock(pmd_ptl); 1053 mmu_notifier_invalidate_range_end(&range); 1054 tlb_remove_table_sync_one(); 1055 1056 spin_lock(pte_ptl); 1057 result = __collapse_huge_page_isolate(vma, address, pte, cc, 1058 &compound_pagelist); 1059 spin_unlock(pte_ptl); 1060 1061 if (unlikely(result != SCAN_SUCCEED)) { 1062 pte_unmap(pte); 1063 spin_lock(pmd_ptl); 1064 BUG_ON(!pmd_none(*pmd)); 1065 /* 1066 * We can only use set_pmd_at when establishing 1067 * hugepmds and never for establishing regular pmds that 1068 * points to regular pagetables. Use pmd_populate for that 1069 */ 1070 pmd_populate(mm, pmd, pmd_pgtable(_pmd)); 1071 spin_unlock(pmd_ptl); 1072 anon_vma_unlock_write(vma->anon_vma); 1073 goto out_up_write; 1074 } 1075 1076 /* 1077 * All pages are isolated and locked so anon_vma rmap 1078 * can't run anymore. 1079 */ 1080 anon_vma_unlock_write(vma->anon_vma); 1081 1082 __collapse_huge_page_copy(pte, hpage, vma, address, pte_ptl, 1083 &compound_pagelist); 1084 pte_unmap(pte); 1085 /* 1086 * spin_lock() below is not the equivalent of smp_wmb(), but 1087 * the smp_wmb() inside __SetPageUptodate() can be reused to 1088 * avoid the copy_huge_page writes to become visible after 1089 * the set_pmd_at() write. 1090 */ 1091 __SetPageUptodate(hpage); 1092 pgtable = pmd_pgtable(_pmd); 1093 1094 _pmd = mk_huge_pmd(hpage, vma->vm_page_prot); 1095 _pmd = maybe_pmd_mkwrite(pmd_mkdirty(_pmd), vma); 1096 1097 spin_lock(pmd_ptl); 1098 BUG_ON(!pmd_none(*pmd)); 1099 page_add_new_anon_rmap(hpage, vma, address); 1100 lru_cache_add_inactive_or_unevictable(hpage, vma); 1101 pgtable_trans_huge_deposit(mm, pmd, pgtable); 1102 set_pmd_at(mm, address, pmd, _pmd); 1103 update_mmu_cache_pmd(vma, address, pmd); 1104 spin_unlock(pmd_ptl); 1105 1106 hpage = NULL; 1107 1108 result = SCAN_SUCCEED; 1109 out_up_write: 1110 mmap_write_unlock(mm); 1111 out_nolock: 1112 if (hpage) { 1113 mem_cgroup_uncharge(page_folio(hpage)); 1114 put_page(hpage); 1115 } 1116 trace_mm_collapse_huge_page(mm, result == SCAN_SUCCEED, result); 1117 return result; 1118 } 1119 1120 static int hpage_collapse_scan_pmd(struct mm_struct *mm, 1121 struct vm_area_struct *vma, 1122 unsigned long address, bool *mmap_locked, 1123 struct collapse_control *cc) 1124 { 1125 pmd_t *pmd; 1126 pte_t *pte, *_pte; 1127 int result = SCAN_FAIL, referenced = 0; 1128 int none_or_zero = 0, shared = 0; 1129 struct page *page = NULL; 1130 unsigned long _address; 1131 spinlock_t *ptl; 1132 int node = NUMA_NO_NODE, unmapped = 0; 1133 bool writable = false; 1134 1135 VM_BUG_ON(address & ~HPAGE_PMD_MASK); 1136 1137 result = find_pmd_or_thp_or_none(mm, address, &pmd); 1138 if (result != SCAN_SUCCEED) 1139 goto out; 1140 1141 memset(cc->node_load, 0, sizeof(cc->node_load)); 1142 nodes_clear(cc->alloc_nmask); 1143 pte = pte_offset_map_lock(mm, pmd, address, &ptl); 1144 for (_address = address, _pte = pte; _pte < pte + HPAGE_PMD_NR; 1145 _pte++, _address += PAGE_SIZE) { 1146 pte_t pteval = *_pte; 1147 if (is_swap_pte(pteval)) { 1148 ++unmapped; 1149 if (!cc->is_khugepaged || 1150 unmapped <= khugepaged_max_ptes_swap) { 1151 /* 1152 * Always be strict with uffd-wp 1153 * enabled swap entries. Please see 1154 * comment below for pte_uffd_wp(). 1155 */ 1156 if (pte_swp_uffd_wp(pteval)) { 1157 result = SCAN_PTE_UFFD_WP; 1158 goto out_unmap; 1159 } 1160 continue; 1161 } else { 1162 result = SCAN_EXCEED_SWAP_PTE; 1163 count_vm_event(THP_SCAN_EXCEED_SWAP_PTE); 1164 goto out_unmap; 1165 } 1166 } 1167 if (pte_none(pteval) || is_zero_pfn(pte_pfn(pteval))) { 1168 ++none_or_zero; 1169 if (!userfaultfd_armed(vma) && 1170 (!cc->is_khugepaged || 1171 none_or_zero <= khugepaged_max_ptes_none)) { 1172 continue; 1173 } else { 1174 result = SCAN_EXCEED_NONE_PTE; 1175 count_vm_event(THP_SCAN_EXCEED_NONE_PTE); 1176 goto out_unmap; 1177 } 1178 } 1179 if (pte_uffd_wp(pteval)) { 1180 /* 1181 * Don't collapse the page if any of the small 1182 * PTEs are armed with uffd write protection. 1183 * Here we can also mark the new huge pmd as 1184 * write protected if any of the small ones is 1185 * marked but that could bring unknown 1186 * userfault messages that falls outside of 1187 * the registered range. So, just be simple. 1188 */ 1189 result = SCAN_PTE_UFFD_WP; 1190 goto out_unmap; 1191 } 1192 if (pte_write(pteval)) 1193 writable = true; 1194 1195 page = vm_normal_page(vma, _address, pteval); 1196 if (unlikely(!page) || unlikely(is_zone_device_page(page))) { 1197 result = SCAN_PAGE_NULL; 1198 goto out_unmap; 1199 } 1200 1201 if (page_mapcount(page) > 1) { 1202 ++shared; 1203 if (cc->is_khugepaged && 1204 shared > khugepaged_max_ptes_shared) { 1205 result = SCAN_EXCEED_SHARED_PTE; 1206 count_vm_event(THP_SCAN_EXCEED_SHARED_PTE); 1207 goto out_unmap; 1208 } 1209 } 1210 1211 page = compound_head(page); 1212 1213 /* 1214 * Record which node the original page is from and save this 1215 * information to cc->node_load[]. 1216 * Khugepaged will allocate hugepage from the node has the max 1217 * hit record. 1218 */ 1219 node = page_to_nid(page); 1220 if (hpage_collapse_scan_abort(node, cc)) { 1221 result = SCAN_SCAN_ABORT; 1222 goto out_unmap; 1223 } 1224 cc->node_load[node]++; 1225 if (!PageLRU(page)) { 1226 result = SCAN_PAGE_LRU; 1227 goto out_unmap; 1228 } 1229 if (PageLocked(page)) { 1230 result = SCAN_PAGE_LOCK; 1231 goto out_unmap; 1232 } 1233 if (!PageAnon(page)) { 1234 result = SCAN_PAGE_ANON; 1235 goto out_unmap; 1236 } 1237 1238 /* 1239 * Check if the page has any GUP (or other external) pins. 1240 * 1241 * Here the check is racy it may see total_mapcount > refcount 1242 * in some cases. 1243 * For example, one process with one forked child process. 1244 * The parent has the PMD split due to MADV_DONTNEED, then 1245 * the child is trying unmap the whole PMD, but khugepaged 1246 * may be scanning the parent between the child has 1247 * PageDoubleMap flag cleared and dec the mapcount. So 1248 * khugepaged may see total_mapcount > refcount. 1249 * 1250 * But such case is ephemeral we could always retry collapse 1251 * later. However it may report false positive if the page 1252 * has excessive GUP pins (i.e. 512). Anyway the same check 1253 * will be done again later the risk seems low. 1254 */ 1255 if (!is_refcount_suitable(page)) { 1256 result = SCAN_PAGE_COUNT; 1257 goto out_unmap; 1258 } 1259 1260 /* 1261 * If collapse was initiated by khugepaged, check that there is 1262 * enough young pte to justify collapsing the page 1263 */ 1264 if (cc->is_khugepaged && 1265 (pte_young(pteval) || page_is_young(page) || 1266 PageReferenced(page) || mmu_notifier_test_young(vma->vm_mm, 1267 address))) 1268 referenced++; 1269 } 1270 if (!writable) { 1271 result = SCAN_PAGE_RO; 1272 } else if (cc->is_khugepaged && 1273 (!referenced || 1274 (unmapped && referenced < HPAGE_PMD_NR / 2))) { 1275 result = SCAN_LACK_REFERENCED_PAGE; 1276 } else { 1277 result = SCAN_SUCCEED; 1278 } 1279 out_unmap: 1280 pte_unmap_unlock(pte, ptl); 1281 if (result == SCAN_SUCCEED) { 1282 result = collapse_huge_page(mm, address, referenced, 1283 unmapped, cc); 1284 /* collapse_huge_page will return with the mmap_lock released */ 1285 *mmap_locked = false; 1286 } 1287 out: 1288 trace_mm_khugepaged_scan_pmd(mm, page, writable, referenced, 1289 none_or_zero, result, unmapped); 1290 return result; 1291 } 1292 1293 static void collect_mm_slot(struct khugepaged_mm_slot *mm_slot) 1294 { 1295 struct mm_slot *slot = &mm_slot->slot; 1296 struct mm_struct *mm = slot->mm; 1297 1298 lockdep_assert_held(&khugepaged_mm_lock); 1299 1300 if (hpage_collapse_test_exit(mm)) { 1301 /* free mm_slot */ 1302 hash_del(&slot->hash); 1303 list_del(&slot->mm_node); 1304 1305 /* 1306 * Not strictly needed because the mm exited already. 1307 * 1308 * clear_bit(MMF_VM_HUGEPAGE, &mm->flags); 1309 */ 1310 1311 /* khugepaged_mm_lock actually not necessary for the below */ 1312 mm_slot_free(mm_slot_cache, mm_slot); 1313 mmdrop(mm); 1314 } 1315 } 1316 1317 #ifdef CONFIG_SHMEM 1318 /* 1319 * Notify khugepaged that given addr of the mm is pte-mapped THP. Then 1320 * khugepaged should try to collapse the page table. 1321 * 1322 * Note that following race exists: 1323 * (1) khugepaged calls khugepaged_collapse_pte_mapped_thps() for mm_struct A, 1324 * emptying the A's ->pte_mapped_thp[] array. 1325 * (2) MADV_COLLAPSE collapses some file extent with target mm_struct B, and 1326 * retract_page_tables() finds a VMA in mm_struct A mapping the same extent 1327 * (at virtual address X) and adds an entry (for X) into mm_struct A's 1328 * ->pte-mapped_thp[] array. 1329 * (3) khugepaged calls khugepaged_collapse_scan_file() for mm_struct A at X, 1330 * sees a pte-mapped THP (SCAN_PTE_MAPPED_HUGEPAGE) and adds an entry 1331 * (for X) into mm_struct A's ->pte-mapped_thp[] array. 1332 * Thus, it's possible the same address is added multiple times for the same 1333 * mm_struct. Should this happen, we'll simply attempt 1334 * collapse_pte_mapped_thp() multiple times for the same address, under the same 1335 * exclusive mmap_lock, and assuming the first call is successful, subsequent 1336 * attempts will return quickly (without grabbing any additional locks) when 1337 * a huge pmd is found in find_pmd_or_thp_or_none(). Since this is a cheap 1338 * check, and since this is a rare occurrence, the cost of preventing this 1339 * "multiple-add" is thought to be more expensive than just handling it, should 1340 * it occur. 1341 */ 1342 static bool khugepaged_add_pte_mapped_thp(struct mm_struct *mm, 1343 unsigned long addr) 1344 { 1345 struct khugepaged_mm_slot *mm_slot; 1346 struct mm_slot *slot; 1347 bool ret = false; 1348 1349 VM_BUG_ON(addr & ~HPAGE_PMD_MASK); 1350 1351 spin_lock(&khugepaged_mm_lock); 1352 slot = mm_slot_lookup(mm_slots_hash, mm); 1353 mm_slot = mm_slot_entry(slot, struct khugepaged_mm_slot, slot); 1354 if (likely(mm_slot && mm_slot->nr_pte_mapped_thp < MAX_PTE_MAPPED_THP)) { 1355 mm_slot->pte_mapped_thp[mm_slot->nr_pte_mapped_thp++] = addr; 1356 ret = true; 1357 } 1358 spin_unlock(&khugepaged_mm_lock); 1359 return ret; 1360 } 1361 1362 /* hpage must be locked, and mmap_lock must be held in write */ 1363 static int set_huge_pmd(struct vm_area_struct *vma, unsigned long addr, 1364 pmd_t *pmdp, struct page *hpage) 1365 { 1366 struct vm_fault vmf = { 1367 .vma = vma, 1368 .address = addr, 1369 .flags = 0, 1370 .pmd = pmdp, 1371 }; 1372 1373 VM_BUG_ON(!PageTransHuge(hpage)); 1374 mmap_assert_write_locked(vma->vm_mm); 1375 1376 if (do_set_pmd(&vmf, hpage)) 1377 return SCAN_FAIL; 1378 1379 get_page(hpage); 1380 return SCAN_SUCCEED; 1381 } 1382 1383 /* 1384 * A note about locking: 1385 * Trying to take the page table spinlocks would be useless here because those 1386 * are only used to synchronize: 1387 * 1388 * - modifying terminal entries (ones that point to a data page, not to another 1389 * page table) 1390 * - installing *new* non-terminal entries 1391 * 1392 * Instead, we need roughly the same kind of protection as free_pgtables() or 1393 * mm_take_all_locks() (but only for a single VMA): 1394 * The mmap lock together with this VMA's rmap locks covers all paths towards 1395 * the page table entries we're messing with here, except for hardware page 1396 * table walks and lockless_pages_from_mm(). 1397 */ 1398 static void collapse_and_free_pmd(struct mm_struct *mm, struct vm_area_struct *vma, 1399 unsigned long addr, pmd_t *pmdp) 1400 { 1401 pmd_t pmd; 1402 struct mmu_notifier_range range; 1403 1404 mmap_assert_write_locked(mm); 1405 if (vma->vm_file) 1406 lockdep_assert_held_write(&vma->vm_file->f_mapping->i_mmap_rwsem); 1407 /* 1408 * All anon_vmas attached to the VMA have the same root and are 1409 * therefore locked by the same lock. 1410 */ 1411 if (vma->anon_vma) 1412 lockdep_assert_held_write(&vma->anon_vma->root->rwsem); 1413 1414 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, NULL, mm, addr, 1415 addr + HPAGE_PMD_SIZE); 1416 mmu_notifier_invalidate_range_start(&range); 1417 pmd = pmdp_collapse_flush(vma, addr, pmdp); 1418 tlb_remove_table_sync_one(); 1419 mmu_notifier_invalidate_range_end(&range); 1420 mm_dec_nr_ptes(mm); 1421 page_table_check_pte_clear_range(mm, addr, pmd); 1422 pte_free(mm, pmd_pgtable(pmd)); 1423 } 1424 1425 /** 1426 * collapse_pte_mapped_thp - Try to collapse a pte-mapped THP for mm at 1427 * address haddr. 1428 * 1429 * @mm: process address space where collapse happens 1430 * @addr: THP collapse address 1431 * @install_pmd: If a huge PMD should be installed 1432 * 1433 * This function checks whether all the PTEs in the PMD are pointing to the 1434 * right THP. If so, retract the page table so the THP can refault in with 1435 * as pmd-mapped. Possibly install a huge PMD mapping the THP. 1436 */ 1437 int collapse_pte_mapped_thp(struct mm_struct *mm, unsigned long addr, 1438 bool install_pmd) 1439 { 1440 unsigned long haddr = addr & HPAGE_PMD_MASK; 1441 struct vm_area_struct *vma = vma_lookup(mm, haddr); 1442 struct page *hpage; 1443 pte_t *start_pte, *pte; 1444 pmd_t *pmd; 1445 spinlock_t *ptl; 1446 int count = 0, result = SCAN_FAIL; 1447 int i; 1448 1449 mmap_assert_write_locked(mm); 1450 1451 /* Fast check before locking page if already PMD-mapped */ 1452 result = find_pmd_or_thp_or_none(mm, haddr, &pmd); 1453 if (result == SCAN_PMD_MAPPED) 1454 return result; 1455 1456 if (!vma || !vma->vm_file || 1457 !range_in_vma(vma, haddr, haddr + HPAGE_PMD_SIZE)) 1458 return SCAN_VMA_CHECK; 1459 1460 /* 1461 * If we are here, we've succeeded in replacing all the native pages 1462 * in the page cache with a single hugepage. If a mm were to fault-in 1463 * this memory (mapped by a suitably aligned VMA), we'd get the hugepage 1464 * and map it by a PMD, regardless of sysfs THP settings. As such, let's 1465 * analogously elide sysfs THP settings here. 1466 */ 1467 if (!hugepage_vma_check(vma, vma->vm_flags, false, false, false)) 1468 return SCAN_VMA_CHECK; 1469 1470 /* 1471 * Symmetry with retract_page_tables(): Exclude MAP_PRIVATE mappings 1472 * that got written to. Without this, we'd have to also lock the 1473 * anon_vma if one exists. 1474 */ 1475 if (vma->anon_vma) 1476 return SCAN_VMA_CHECK; 1477 1478 /* Keep pmd pgtable for uffd-wp; see comment in retract_page_tables() */ 1479 if (userfaultfd_wp(vma)) 1480 return SCAN_PTE_UFFD_WP; 1481 1482 hpage = find_lock_page(vma->vm_file->f_mapping, 1483 linear_page_index(vma, haddr)); 1484 if (!hpage) 1485 return SCAN_PAGE_NULL; 1486 1487 if (!PageHead(hpage)) { 1488 result = SCAN_FAIL; 1489 goto drop_hpage; 1490 } 1491 1492 if (compound_order(hpage) != HPAGE_PMD_ORDER) { 1493 result = SCAN_PAGE_COMPOUND; 1494 goto drop_hpage; 1495 } 1496 1497 switch (result) { 1498 case SCAN_SUCCEED: 1499 break; 1500 case SCAN_PMD_NONE: 1501 /* 1502 * In MADV_COLLAPSE path, possible race with khugepaged where 1503 * all pte entries have been removed and pmd cleared. If so, 1504 * skip all the pte checks and just update the pmd mapping. 1505 */ 1506 goto maybe_install_pmd; 1507 default: 1508 goto drop_hpage; 1509 } 1510 1511 /* 1512 * We need to lock the mapping so that from here on, only GUP-fast and 1513 * hardware page walks can access the parts of the page tables that 1514 * we're operating on. 1515 * See collapse_and_free_pmd(). 1516 */ 1517 i_mmap_lock_write(vma->vm_file->f_mapping); 1518 1519 /* 1520 * This spinlock should be unnecessary: Nobody else should be accessing 1521 * the page tables under spinlock protection here, only 1522 * lockless_pages_from_mm() and the hardware page walker can access page 1523 * tables while all the high-level locks are held in write mode. 1524 */ 1525 start_pte = pte_offset_map_lock(mm, pmd, haddr, &ptl); 1526 result = SCAN_FAIL; 1527 1528 /* step 1: check all mapped PTEs are to the right huge page */ 1529 for (i = 0, addr = haddr, pte = start_pte; 1530 i < HPAGE_PMD_NR; i++, addr += PAGE_SIZE, pte++) { 1531 struct page *page; 1532 1533 /* empty pte, skip */ 1534 if (pte_none(*pte)) 1535 continue; 1536 1537 /* page swapped out, abort */ 1538 if (!pte_present(*pte)) { 1539 result = SCAN_PTE_NON_PRESENT; 1540 goto abort; 1541 } 1542 1543 page = vm_normal_page(vma, addr, *pte); 1544 if (WARN_ON_ONCE(page && is_zone_device_page(page))) 1545 page = NULL; 1546 /* 1547 * Note that uprobe, debugger, or MAP_PRIVATE may change the 1548 * page table, but the new page will not be a subpage of hpage. 1549 */ 1550 if (hpage + i != page) 1551 goto abort; 1552 count++; 1553 } 1554 1555 /* step 2: adjust rmap */ 1556 for (i = 0, addr = haddr, pte = start_pte; 1557 i < HPAGE_PMD_NR; i++, addr += PAGE_SIZE, pte++) { 1558 struct page *page; 1559 1560 if (pte_none(*pte)) 1561 continue; 1562 page = vm_normal_page(vma, addr, *pte); 1563 if (WARN_ON_ONCE(page && is_zone_device_page(page))) 1564 goto abort; 1565 page_remove_rmap(page, vma, false); 1566 } 1567 1568 pte_unmap_unlock(start_pte, ptl); 1569 1570 /* step 3: set proper refcount and mm_counters. */ 1571 if (count) { 1572 page_ref_sub(hpage, count); 1573 add_mm_counter(vma->vm_mm, mm_counter_file(hpage), -count); 1574 } 1575 1576 /* step 4: remove pte entries */ 1577 collapse_and_free_pmd(mm, vma, haddr, pmd); 1578 1579 i_mmap_unlock_write(vma->vm_file->f_mapping); 1580 1581 maybe_install_pmd: 1582 /* step 5: install pmd entry */ 1583 result = install_pmd 1584 ? set_huge_pmd(vma, haddr, pmd, hpage) 1585 : SCAN_SUCCEED; 1586 1587 drop_hpage: 1588 unlock_page(hpage); 1589 put_page(hpage); 1590 return result; 1591 1592 abort: 1593 pte_unmap_unlock(start_pte, ptl); 1594 i_mmap_unlock_write(vma->vm_file->f_mapping); 1595 goto drop_hpage; 1596 } 1597 1598 static void khugepaged_collapse_pte_mapped_thps(struct khugepaged_mm_slot *mm_slot) 1599 { 1600 struct mm_slot *slot = &mm_slot->slot; 1601 struct mm_struct *mm = slot->mm; 1602 int i; 1603 1604 if (likely(mm_slot->nr_pte_mapped_thp == 0)) 1605 return; 1606 1607 if (!mmap_write_trylock(mm)) 1608 return; 1609 1610 if (unlikely(hpage_collapse_test_exit(mm))) 1611 goto out; 1612 1613 for (i = 0; i < mm_slot->nr_pte_mapped_thp; i++) 1614 collapse_pte_mapped_thp(mm, mm_slot->pte_mapped_thp[i], false); 1615 1616 out: 1617 mm_slot->nr_pte_mapped_thp = 0; 1618 mmap_write_unlock(mm); 1619 } 1620 1621 static int retract_page_tables(struct address_space *mapping, pgoff_t pgoff, 1622 struct mm_struct *target_mm, 1623 unsigned long target_addr, struct page *hpage, 1624 struct collapse_control *cc) 1625 { 1626 struct vm_area_struct *vma; 1627 int target_result = SCAN_FAIL; 1628 1629 i_mmap_lock_write(mapping); 1630 vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff, pgoff) { 1631 int result = SCAN_FAIL; 1632 struct mm_struct *mm = NULL; 1633 unsigned long addr = 0; 1634 pmd_t *pmd; 1635 bool is_target = false; 1636 1637 /* 1638 * Check vma->anon_vma to exclude MAP_PRIVATE mappings that 1639 * got written to. These VMAs are likely not worth investing 1640 * mmap_write_lock(mm) as PMD-mapping is likely to be split 1641 * later. 1642 * 1643 * Note that vma->anon_vma check is racy: it can be set up after 1644 * the check but before we took mmap_lock by the fault path. 1645 * But page lock would prevent establishing any new ptes of the 1646 * page, so we are safe. 1647 * 1648 * An alternative would be drop the check, but check that page 1649 * table is clear before calling pmdp_collapse_flush() under 1650 * ptl. It has higher chance to recover THP for the VMA, but 1651 * has higher cost too. It would also probably require locking 1652 * the anon_vma. 1653 */ 1654 if (vma->anon_vma) { 1655 result = SCAN_PAGE_ANON; 1656 goto next; 1657 } 1658 addr = vma->vm_start + ((pgoff - vma->vm_pgoff) << PAGE_SHIFT); 1659 if (addr & ~HPAGE_PMD_MASK || 1660 vma->vm_end < addr + HPAGE_PMD_SIZE) { 1661 result = SCAN_VMA_CHECK; 1662 goto next; 1663 } 1664 mm = vma->vm_mm; 1665 is_target = mm == target_mm && addr == target_addr; 1666 result = find_pmd_or_thp_or_none(mm, addr, &pmd); 1667 if (result != SCAN_SUCCEED) 1668 goto next; 1669 /* 1670 * We need exclusive mmap_lock to retract page table. 1671 * 1672 * We use trylock due to lock inversion: we need to acquire 1673 * mmap_lock while holding page lock. Fault path does it in 1674 * reverse order. Trylock is a way to avoid deadlock. 1675 * 1676 * Also, it's not MADV_COLLAPSE's job to collapse other 1677 * mappings - let khugepaged take care of them later. 1678 */ 1679 result = SCAN_PTE_MAPPED_HUGEPAGE; 1680 if ((cc->is_khugepaged || is_target) && 1681 mmap_write_trylock(mm)) { 1682 /* 1683 * When a vma is registered with uffd-wp, we can't 1684 * recycle the pmd pgtable because there can be pte 1685 * markers installed. Skip it only, so the rest mm/vma 1686 * can still have the same file mapped hugely, however 1687 * it'll always mapped in small page size for uffd-wp 1688 * registered ranges. 1689 */ 1690 if (hpage_collapse_test_exit(mm)) { 1691 result = SCAN_ANY_PROCESS; 1692 goto unlock_next; 1693 } 1694 if (userfaultfd_wp(vma)) { 1695 result = SCAN_PTE_UFFD_WP; 1696 goto unlock_next; 1697 } 1698 collapse_and_free_pmd(mm, vma, addr, pmd); 1699 if (!cc->is_khugepaged && is_target) 1700 result = set_huge_pmd(vma, addr, pmd, hpage); 1701 else 1702 result = SCAN_SUCCEED; 1703 1704 unlock_next: 1705 mmap_write_unlock(mm); 1706 goto next; 1707 } 1708 /* 1709 * Calling context will handle target mm/addr. Otherwise, let 1710 * khugepaged try again later. 1711 */ 1712 if (!is_target) { 1713 khugepaged_add_pte_mapped_thp(mm, addr); 1714 continue; 1715 } 1716 next: 1717 if (is_target) 1718 target_result = result; 1719 } 1720 i_mmap_unlock_write(mapping); 1721 return target_result; 1722 } 1723 1724 /** 1725 * collapse_file - collapse filemap/tmpfs/shmem pages into huge one. 1726 * 1727 * @mm: process address space where collapse happens 1728 * @addr: virtual collapse start address 1729 * @file: file that collapse on 1730 * @start: collapse start address 1731 * @cc: collapse context and scratchpad 1732 * 1733 * Basic scheme is simple, details are more complex: 1734 * - allocate and lock a new huge page; 1735 * - scan page cache replacing old pages with the new one 1736 * + swap/gup in pages if necessary; 1737 * + fill in gaps; 1738 * + keep old pages around in case rollback is required; 1739 * - if replacing succeeds: 1740 * + copy data over; 1741 * + free old pages; 1742 * + unlock huge page; 1743 * - if replacing failed; 1744 * + put all pages back and unfreeze them; 1745 * + restore gaps in the page cache; 1746 * + unlock and free huge page; 1747 */ 1748 static int collapse_file(struct mm_struct *mm, unsigned long addr, 1749 struct file *file, pgoff_t start, 1750 struct collapse_control *cc) 1751 { 1752 struct address_space *mapping = file->f_mapping; 1753 struct page *hpage; 1754 pgoff_t index, end = start + HPAGE_PMD_NR; 1755 LIST_HEAD(pagelist); 1756 XA_STATE_ORDER(xas, &mapping->i_pages, start, HPAGE_PMD_ORDER); 1757 int nr_none = 0, result = SCAN_SUCCEED; 1758 bool is_shmem = shmem_file(file); 1759 int nr; 1760 1761 VM_BUG_ON(!IS_ENABLED(CONFIG_READ_ONLY_THP_FOR_FS) && !is_shmem); 1762 VM_BUG_ON(start & (HPAGE_PMD_NR - 1)); 1763 1764 result = alloc_charge_hpage(&hpage, mm, cc); 1765 if (result != SCAN_SUCCEED) 1766 goto out; 1767 1768 /* 1769 * Ensure we have slots for all the pages in the range. This is 1770 * almost certainly a no-op because most of the pages must be present 1771 */ 1772 do { 1773 xas_lock_irq(&xas); 1774 xas_create_range(&xas); 1775 if (!xas_error(&xas)) 1776 break; 1777 xas_unlock_irq(&xas); 1778 if (!xas_nomem(&xas, GFP_KERNEL)) { 1779 result = SCAN_FAIL; 1780 goto out; 1781 } 1782 } while (1); 1783 1784 __SetPageLocked(hpage); 1785 if (is_shmem) 1786 __SetPageSwapBacked(hpage); 1787 hpage->index = start; 1788 hpage->mapping = mapping; 1789 1790 /* 1791 * At this point the hpage is locked and not up-to-date. 1792 * It's safe to insert it into the page cache, because nobody would 1793 * be able to map it or use it in another way until we unlock it. 1794 */ 1795 1796 xas_set(&xas, start); 1797 for (index = start; index < end; index++) { 1798 struct page *page = xas_next(&xas); 1799 1800 VM_BUG_ON(index != xas.xa_index); 1801 if (is_shmem) { 1802 if (!page) { 1803 /* 1804 * Stop if extent has been truncated or 1805 * hole-punched, and is now completely 1806 * empty. 1807 */ 1808 if (index == start) { 1809 if (!xas_next_entry(&xas, end - 1)) { 1810 result = SCAN_TRUNCATED; 1811 goto xa_locked; 1812 } 1813 xas_set(&xas, index); 1814 } 1815 if (!shmem_charge(mapping->host, 1)) { 1816 result = SCAN_FAIL; 1817 goto xa_locked; 1818 } 1819 xas_store(&xas, hpage); 1820 nr_none++; 1821 continue; 1822 } 1823 1824 if (xa_is_value(page) || !PageUptodate(page)) { 1825 struct folio *folio; 1826 1827 xas_unlock_irq(&xas); 1828 /* swap in or instantiate fallocated page */ 1829 if (shmem_get_folio(mapping->host, index, 1830 &folio, SGP_NOALLOC)) { 1831 result = SCAN_FAIL; 1832 goto xa_unlocked; 1833 } 1834 page = folio_file_page(folio, index); 1835 } else if (trylock_page(page)) { 1836 get_page(page); 1837 xas_unlock_irq(&xas); 1838 } else { 1839 result = SCAN_PAGE_LOCK; 1840 goto xa_locked; 1841 } 1842 } else { /* !is_shmem */ 1843 if (!page || xa_is_value(page)) { 1844 xas_unlock_irq(&xas); 1845 page_cache_sync_readahead(mapping, &file->f_ra, 1846 file, index, 1847 end - index); 1848 /* drain pagevecs to help isolate_lru_page() */ 1849 lru_add_drain(); 1850 page = find_lock_page(mapping, index); 1851 if (unlikely(page == NULL)) { 1852 result = SCAN_FAIL; 1853 goto xa_unlocked; 1854 } 1855 } else if (PageDirty(page)) { 1856 /* 1857 * khugepaged only works on read-only fd, 1858 * so this page is dirty because it hasn't 1859 * been flushed since first write. There 1860 * won't be new dirty pages. 1861 * 1862 * Trigger async flush here and hope the 1863 * writeback is done when khugepaged 1864 * revisits this page. 1865 * 1866 * This is a one-off situation. We are not 1867 * forcing writeback in loop. 1868 */ 1869 xas_unlock_irq(&xas); 1870 filemap_flush(mapping); 1871 result = SCAN_FAIL; 1872 goto xa_unlocked; 1873 } else if (PageWriteback(page)) { 1874 xas_unlock_irq(&xas); 1875 result = SCAN_FAIL; 1876 goto xa_unlocked; 1877 } else if (trylock_page(page)) { 1878 get_page(page); 1879 xas_unlock_irq(&xas); 1880 } else { 1881 result = SCAN_PAGE_LOCK; 1882 goto xa_locked; 1883 } 1884 } 1885 1886 /* 1887 * The page must be locked, so we can drop the i_pages lock 1888 * without racing with truncate. 1889 */ 1890 VM_BUG_ON_PAGE(!PageLocked(page), page); 1891 1892 /* make sure the page is up to date */ 1893 if (unlikely(!PageUptodate(page))) { 1894 result = SCAN_FAIL; 1895 goto out_unlock; 1896 } 1897 1898 /* 1899 * If file was truncated then extended, or hole-punched, before 1900 * we locked the first page, then a THP might be there already. 1901 * This will be discovered on the first iteration. 1902 */ 1903 if (PageTransCompound(page)) { 1904 struct page *head = compound_head(page); 1905 1906 result = compound_order(head) == HPAGE_PMD_ORDER && 1907 head->index == start 1908 /* Maybe PMD-mapped */ 1909 ? SCAN_PTE_MAPPED_HUGEPAGE 1910 : SCAN_PAGE_COMPOUND; 1911 goto out_unlock; 1912 } 1913 1914 if (page_mapping(page) != mapping) { 1915 result = SCAN_TRUNCATED; 1916 goto out_unlock; 1917 } 1918 1919 if (!is_shmem && (PageDirty(page) || 1920 PageWriteback(page))) { 1921 /* 1922 * khugepaged only works on read-only fd, so this 1923 * page is dirty because it hasn't been flushed 1924 * since first write. 1925 */ 1926 result = SCAN_FAIL; 1927 goto out_unlock; 1928 } 1929 1930 if (isolate_lru_page(page)) { 1931 result = SCAN_DEL_PAGE_LRU; 1932 goto out_unlock; 1933 } 1934 1935 if (page_has_private(page) && 1936 !try_to_release_page(page, GFP_KERNEL)) { 1937 result = SCAN_PAGE_HAS_PRIVATE; 1938 putback_lru_page(page); 1939 goto out_unlock; 1940 } 1941 1942 if (page_mapped(page)) 1943 try_to_unmap(page_folio(page), 1944 TTU_IGNORE_MLOCK | TTU_BATCH_FLUSH); 1945 1946 xas_lock_irq(&xas); 1947 xas_set(&xas, index); 1948 1949 VM_BUG_ON_PAGE(page != xas_load(&xas), page); 1950 1951 /* 1952 * The page is expected to have page_count() == 3: 1953 * - we hold a pin on it; 1954 * - one reference from page cache; 1955 * - one from isolate_lru_page; 1956 */ 1957 if (!page_ref_freeze(page, 3)) { 1958 result = SCAN_PAGE_COUNT; 1959 xas_unlock_irq(&xas); 1960 putback_lru_page(page); 1961 goto out_unlock; 1962 } 1963 1964 /* 1965 * Add the page to the list to be able to undo the collapse if 1966 * something go wrong. 1967 */ 1968 list_add_tail(&page->lru, &pagelist); 1969 1970 /* Finally, replace with the new page. */ 1971 xas_store(&xas, hpage); 1972 continue; 1973 out_unlock: 1974 unlock_page(page); 1975 put_page(page); 1976 goto xa_unlocked; 1977 } 1978 nr = thp_nr_pages(hpage); 1979 1980 if (is_shmem) 1981 __mod_lruvec_page_state(hpage, NR_SHMEM_THPS, nr); 1982 else { 1983 __mod_lruvec_page_state(hpage, NR_FILE_THPS, nr); 1984 filemap_nr_thps_inc(mapping); 1985 /* 1986 * Paired with smp_mb() in do_dentry_open() to ensure 1987 * i_writecount is up to date and the update to nr_thps is 1988 * visible. Ensures the page cache will be truncated if the 1989 * file is opened writable. 1990 */ 1991 smp_mb(); 1992 if (inode_is_open_for_write(mapping->host)) { 1993 result = SCAN_FAIL; 1994 __mod_lruvec_page_state(hpage, NR_FILE_THPS, -nr); 1995 filemap_nr_thps_dec(mapping); 1996 goto xa_locked; 1997 } 1998 } 1999 2000 if (nr_none) { 2001 __mod_lruvec_page_state(hpage, NR_FILE_PAGES, nr_none); 2002 /* nr_none is always 0 for non-shmem. */ 2003 __mod_lruvec_page_state(hpage, NR_SHMEM, nr_none); 2004 } 2005 2006 /* Join all the small entries into a single multi-index entry */ 2007 xas_set_order(&xas, start, HPAGE_PMD_ORDER); 2008 xas_store(&xas, hpage); 2009 xa_locked: 2010 xas_unlock_irq(&xas); 2011 xa_unlocked: 2012 2013 /* 2014 * If collapse is successful, flush must be done now before copying. 2015 * If collapse is unsuccessful, does flush actually need to be done? 2016 * Do it anyway, to clear the state. 2017 */ 2018 try_to_unmap_flush(); 2019 2020 if (result == SCAN_SUCCEED) { 2021 struct page *page, *tmp; 2022 2023 /* 2024 * Replacing old pages with new one has succeeded, now we 2025 * need to copy the content and free the old pages. 2026 */ 2027 index = start; 2028 list_for_each_entry_safe(page, tmp, &pagelist, lru) { 2029 while (index < page->index) { 2030 clear_highpage(hpage + (index % HPAGE_PMD_NR)); 2031 index++; 2032 } 2033 copy_highpage(hpage + (page->index % HPAGE_PMD_NR), 2034 page); 2035 list_del(&page->lru); 2036 page->mapping = NULL; 2037 page_ref_unfreeze(page, 1); 2038 ClearPageActive(page); 2039 ClearPageUnevictable(page); 2040 unlock_page(page); 2041 put_page(page); 2042 index++; 2043 } 2044 while (index < end) { 2045 clear_highpage(hpage + (index % HPAGE_PMD_NR)); 2046 index++; 2047 } 2048 2049 SetPageUptodate(hpage); 2050 page_ref_add(hpage, HPAGE_PMD_NR - 1); 2051 if (is_shmem) 2052 set_page_dirty(hpage); 2053 lru_cache_add(hpage); 2054 2055 /* 2056 * Remove pte page tables, so we can re-fault the page as huge. 2057 */ 2058 result = retract_page_tables(mapping, start, mm, addr, hpage, 2059 cc); 2060 unlock_page(hpage); 2061 hpage = NULL; 2062 } else { 2063 struct page *page; 2064 2065 /* Something went wrong: roll back page cache changes */ 2066 xas_lock_irq(&xas); 2067 if (nr_none) { 2068 mapping->nrpages -= nr_none; 2069 shmem_uncharge(mapping->host, nr_none); 2070 } 2071 2072 xas_set(&xas, start); 2073 xas_for_each(&xas, page, end - 1) { 2074 page = list_first_entry_or_null(&pagelist, 2075 struct page, lru); 2076 if (!page || xas.xa_index < page->index) { 2077 if (!nr_none) 2078 break; 2079 nr_none--; 2080 /* Put holes back where they were */ 2081 xas_store(&xas, NULL); 2082 continue; 2083 } 2084 2085 VM_BUG_ON_PAGE(page->index != xas.xa_index, page); 2086 2087 /* Unfreeze the page. */ 2088 list_del(&page->lru); 2089 page_ref_unfreeze(page, 2); 2090 xas_store(&xas, page); 2091 xas_pause(&xas); 2092 xas_unlock_irq(&xas); 2093 unlock_page(page); 2094 putback_lru_page(page); 2095 xas_lock_irq(&xas); 2096 } 2097 VM_BUG_ON(nr_none); 2098 xas_unlock_irq(&xas); 2099 2100 hpage->mapping = NULL; 2101 } 2102 2103 if (hpage) 2104 unlock_page(hpage); 2105 out: 2106 VM_BUG_ON(!list_empty(&pagelist)); 2107 if (hpage) { 2108 mem_cgroup_uncharge(page_folio(hpage)); 2109 put_page(hpage); 2110 } 2111 /* TODO: tracepoints */ 2112 return result; 2113 } 2114 2115 static int hpage_collapse_scan_file(struct mm_struct *mm, unsigned long addr, 2116 struct file *file, pgoff_t start, 2117 struct collapse_control *cc) 2118 { 2119 struct page *page = NULL; 2120 struct address_space *mapping = file->f_mapping; 2121 XA_STATE(xas, &mapping->i_pages, start); 2122 int present, swap; 2123 int node = NUMA_NO_NODE; 2124 int result = SCAN_SUCCEED; 2125 2126 present = 0; 2127 swap = 0; 2128 memset(cc->node_load, 0, sizeof(cc->node_load)); 2129 nodes_clear(cc->alloc_nmask); 2130 rcu_read_lock(); 2131 xas_for_each(&xas, page, start + HPAGE_PMD_NR - 1) { 2132 if (xas_retry(&xas, page)) 2133 continue; 2134 2135 if (xa_is_value(page)) { 2136 ++swap; 2137 if (cc->is_khugepaged && 2138 swap > khugepaged_max_ptes_swap) { 2139 result = SCAN_EXCEED_SWAP_PTE; 2140 count_vm_event(THP_SCAN_EXCEED_SWAP_PTE); 2141 break; 2142 } 2143 continue; 2144 } 2145 2146 /* 2147 * TODO: khugepaged should compact smaller compound pages 2148 * into a PMD sized page 2149 */ 2150 if (PageTransCompound(page)) { 2151 struct page *head = compound_head(page); 2152 2153 result = compound_order(head) == HPAGE_PMD_ORDER && 2154 head->index == start 2155 /* Maybe PMD-mapped */ 2156 ? SCAN_PTE_MAPPED_HUGEPAGE 2157 : SCAN_PAGE_COMPOUND; 2158 /* 2159 * For SCAN_PTE_MAPPED_HUGEPAGE, further processing 2160 * by the caller won't touch the page cache, and so 2161 * it's safe to skip LRU and refcount checks before 2162 * returning. 2163 */ 2164 break; 2165 } 2166 2167 node = page_to_nid(page); 2168 if (hpage_collapse_scan_abort(node, cc)) { 2169 result = SCAN_SCAN_ABORT; 2170 break; 2171 } 2172 cc->node_load[node]++; 2173 2174 if (!PageLRU(page)) { 2175 result = SCAN_PAGE_LRU; 2176 break; 2177 } 2178 2179 if (page_count(page) != 2180 1 + page_mapcount(page) + page_has_private(page)) { 2181 result = SCAN_PAGE_COUNT; 2182 break; 2183 } 2184 2185 /* 2186 * We probably should check if the page is referenced here, but 2187 * nobody would transfer pte_young() to PageReferenced() for us. 2188 * And rmap walk here is just too costly... 2189 */ 2190 2191 present++; 2192 2193 if (need_resched()) { 2194 xas_pause(&xas); 2195 cond_resched_rcu(); 2196 } 2197 } 2198 rcu_read_unlock(); 2199 2200 if (result == SCAN_SUCCEED) { 2201 if (cc->is_khugepaged && 2202 present < HPAGE_PMD_NR - khugepaged_max_ptes_none) { 2203 result = SCAN_EXCEED_NONE_PTE; 2204 count_vm_event(THP_SCAN_EXCEED_NONE_PTE); 2205 } else { 2206 result = collapse_file(mm, addr, file, start, cc); 2207 } 2208 } 2209 2210 trace_mm_khugepaged_scan_file(mm, page, file, present, swap, result); 2211 return result; 2212 } 2213 #else 2214 static int hpage_collapse_scan_file(struct mm_struct *mm, unsigned long addr, 2215 struct file *file, pgoff_t start, 2216 struct collapse_control *cc) 2217 { 2218 BUILD_BUG(); 2219 } 2220 2221 static void khugepaged_collapse_pte_mapped_thps(struct khugepaged_mm_slot *mm_slot) 2222 { 2223 } 2224 2225 static bool khugepaged_add_pte_mapped_thp(struct mm_struct *mm, 2226 unsigned long addr) 2227 { 2228 return false; 2229 } 2230 #endif 2231 2232 static unsigned int khugepaged_scan_mm_slot(unsigned int pages, int *result, 2233 struct collapse_control *cc) 2234 __releases(&khugepaged_mm_lock) 2235 __acquires(&khugepaged_mm_lock) 2236 { 2237 struct vma_iterator vmi; 2238 struct khugepaged_mm_slot *mm_slot; 2239 struct mm_slot *slot; 2240 struct mm_struct *mm; 2241 struct vm_area_struct *vma; 2242 int progress = 0; 2243 2244 VM_BUG_ON(!pages); 2245 lockdep_assert_held(&khugepaged_mm_lock); 2246 *result = SCAN_FAIL; 2247 2248 if (khugepaged_scan.mm_slot) { 2249 mm_slot = khugepaged_scan.mm_slot; 2250 slot = &mm_slot->slot; 2251 } else { 2252 slot = list_entry(khugepaged_scan.mm_head.next, 2253 struct mm_slot, mm_node); 2254 mm_slot = mm_slot_entry(slot, struct khugepaged_mm_slot, slot); 2255 khugepaged_scan.address = 0; 2256 khugepaged_scan.mm_slot = mm_slot; 2257 } 2258 spin_unlock(&khugepaged_mm_lock); 2259 khugepaged_collapse_pte_mapped_thps(mm_slot); 2260 2261 mm = slot->mm; 2262 /* 2263 * Don't wait for semaphore (to avoid long wait times). Just move to 2264 * the next mm on the list. 2265 */ 2266 vma = NULL; 2267 if (unlikely(!mmap_read_trylock(mm))) 2268 goto breakouterloop_mmap_lock; 2269 2270 progress++; 2271 if (unlikely(hpage_collapse_test_exit(mm))) 2272 goto breakouterloop; 2273 2274 vma_iter_init(&vmi, mm, khugepaged_scan.address); 2275 for_each_vma(vmi, vma) { 2276 unsigned long hstart, hend; 2277 2278 cond_resched(); 2279 if (unlikely(hpage_collapse_test_exit(mm))) { 2280 progress++; 2281 break; 2282 } 2283 if (!hugepage_vma_check(vma, vma->vm_flags, false, false, true)) { 2284 skip: 2285 progress++; 2286 continue; 2287 } 2288 hstart = round_up(vma->vm_start, HPAGE_PMD_SIZE); 2289 hend = round_down(vma->vm_end, HPAGE_PMD_SIZE); 2290 if (khugepaged_scan.address > hend) 2291 goto skip; 2292 if (khugepaged_scan.address < hstart) 2293 khugepaged_scan.address = hstart; 2294 VM_BUG_ON(khugepaged_scan.address & ~HPAGE_PMD_MASK); 2295 2296 while (khugepaged_scan.address < hend) { 2297 bool mmap_locked = true; 2298 2299 cond_resched(); 2300 if (unlikely(hpage_collapse_test_exit(mm))) 2301 goto breakouterloop; 2302 2303 VM_BUG_ON(khugepaged_scan.address < hstart || 2304 khugepaged_scan.address + HPAGE_PMD_SIZE > 2305 hend); 2306 if (IS_ENABLED(CONFIG_SHMEM) && vma->vm_file) { 2307 struct file *file = get_file(vma->vm_file); 2308 pgoff_t pgoff = linear_page_index(vma, 2309 khugepaged_scan.address); 2310 2311 mmap_read_unlock(mm); 2312 *result = hpage_collapse_scan_file(mm, 2313 khugepaged_scan.address, 2314 file, pgoff, cc); 2315 mmap_locked = false; 2316 fput(file); 2317 } else { 2318 *result = hpage_collapse_scan_pmd(mm, vma, 2319 khugepaged_scan.address, 2320 &mmap_locked, 2321 cc); 2322 } 2323 switch (*result) { 2324 case SCAN_PTE_MAPPED_HUGEPAGE: { 2325 pmd_t *pmd; 2326 2327 *result = find_pmd_or_thp_or_none(mm, 2328 khugepaged_scan.address, 2329 &pmd); 2330 if (*result != SCAN_SUCCEED) 2331 break; 2332 if (!khugepaged_add_pte_mapped_thp(mm, 2333 khugepaged_scan.address)) 2334 break; 2335 } fallthrough; 2336 case SCAN_SUCCEED: 2337 ++khugepaged_pages_collapsed; 2338 break; 2339 default: 2340 break; 2341 } 2342 2343 /* move to next address */ 2344 khugepaged_scan.address += HPAGE_PMD_SIZE; 2345 progress += HPAGE_PMD_NR; 2346 if (!mmap_locked) 2347 /* 2348 * We released mmap_lock so break loop. Note 2349 * that we drop mmap_lock before all hugepage 2350 * allocations, so if allocation fails, we are 2351 * guaranteed to break here and report the 2352 * correct result back to caller. 2353 */ 2354 goto breakouterloop_mmap_lock; 2355 if (progress >= pages) 2356 goto breakouterloop; 2357 } 2358 } 2359 breakouterloop: 2360 mmap_read_unlock(mm); /* exit_mmap will destroy ptes after this */ 2361 breakouterloop_mmap_lock: 2362 2363 spin_lock(&khugepaged_mm_lock); 2364 VM_BUG_ON(khugepaged_scan.mm_slot != mm_slot); 2365 /* 2366 * Release the current mm_slot if this mm is about to die, or 2367 * if we scanned all vmas of this mm. 2368 */ 2369 if (hpage_collapse_test_exit(mm) || !vma) { 2370 /* 2371 * Make sure that if mm_users is reaching zero while 2372 * khugepaged runs here, khugepaged_exit will find 2373 * mm_slot not pointing to the exiting mm. 2374 */ 2375 if (slot->mm_node.next != &khugepaged_scan.mm_head) { 2376 slot = list_entry(slot->mm_node.next, 2377 struct mm_slot, mm_node); 2378 khugepaged_scan.mm_slot = 2379 mm_slot_entry(slot, struct khugepaged_mm_slot, slot); 2380 khugepaged_scan.address = 0; 2381 } else { 2382 khugepaged_scan.mm_slot = NULL; 2383 khugepaged_full_scans++; 2384 } 2385 2386 collect_mm_slot(mm_slot); 2387 } 2388 2389 return progress; 2390 } 2391 2392 static int khugepaged_has_work(void) 2393 { 2394 return !list_empty(&khugepaged_scan.mm_head) && 2395 hugepage_flags_enabled(); 2396 } 2397 2398 static int khugepaged_wait_event(void) 2399 { 2400 return !list_empty(&khugepaged_scan.mm_head) || 2401 kthread_should_stop(); 2402 } 2403 2404 static void khugepaged_do_scan(struct collapse_control *cc) 2405 { 2406 unsigned int progress = 0, pass_through_head = 0; 2407 unsigned int pages = READ_ONCE(khugepaged_pages_to_scan); 2408 bool wait = true; 2409 int result = SCAN_SUCCEED; 2410 2411 lru_add_drain_all(); 2412 2413 while (true) { 2414 cond_resched(); 2415 2416 if (unlikely(kthread_should_stop() || try_to_freeze())) 2417 break; 2418 2419 spin_lock(&khugepaged_mm_lock); 2420 if (!khugepaged_scan.mm_slot) 2421 pass_through_head++; 2422 if (khugepaged_has_work() && 2423 pass_through_head < 2) 2424 progress += khugepaged_scan_mm_slot(pages - progress, 2425 &result, cc); 2426 else 2427 progress = pages; 2428 spin_unlock(&khugepaged_mm_lock); 2429 2430 if (progress >= pages) 2431 break; 2432 2433 if (result == SCAN_ALLOC_HUGE_PAGE_FAIL) { 2434 /* 2435 * If fail to allocate the first time, try to sleep for 2436 * a while. When hit again, cancel the scan. 2437 */ 2438 if (!wait) 2439 break; 2440 wait = false; 2441 khugepaged_alloc_sleep(); 2442 } 2443 } 2444 } 2445 2446 static bool khugepaged_should_wakeup(void) 2447 { 2448 return kthread_should_stop() || 2449 time_after_eq(jiffies, khugepaged_sleep_expire); 2450 } 2451 2452 static void khugepaged_wait_work(void) 2453 { 2454 if (khugepaged_has_work()) { 2455 const unsigned long scan_sleep_jiffies = 2456 msecs_to_jiffies(khugepaged_scan_sleep_millisecs); 2457 2458 if (!scan_sleep_jiffies) 2459 return; 2460 2461 khugepaged_sleep_expire = jiffies + scan_sleep_jiffies; 2462 wait_event_freezable_timeout(khugepaged_wait, 2463 khugepaged_should_wakeup(), 2464 scan_sleep_jiffies); 2465 return; 2466 } 2467 2468 if (hugepage_flags_enabled()) 2469 wait_event_freezable(khugepaged_wait, khugepaged_wait_event()); 2470 } 2471 2472 static int khugepaged(void *none) 2473 { 2474 struct khugepaged_mm_slot *mm_slot; 2475 2476 set_freezable(); 2477 set_user_nice(current, MAX_NICE); 2478 2479 while (!kthread_should_stop()) { 2480 khugepaged_do_scan(&khugepaged_collapse_control); 2481 khugepaged_wait_work(); 2482 } 2483 2484 spin_lock(&khugepaged_mm_lock); 2485 mm_slot = khugepaged_scan.mm_slot; 2486 khugepaged_scan.mm_slot = NULL; 2487 if (mm_slot) 2488 collect_mm_slot(mm_slot); 2489 spin_unlock(&khugepaged_mm_lock); 2490 return 0; 2491 } 2492 2493 static void set_recommended_min_free_kbytes(void) 2494 { 2495 struct zone *zone; 2496 int nr_zones = 0; 2497 unsigned long recommended_min; 2498 2499 if (!hugepage_flags_enabled()) { 2500 calculate_min_free_kbytes(); 2501 goto update_wmarks; 2502 } 2503 2504 for_each_populated_zone(zone) { 2505 /* 2506 * We don't need to worry about fragmentation of 2507 * ZONE_MOVABLE since it only has movable pages. 2508 */ 2509 if (zone_idx(zone) > gfp_zone(GFP_USER)) 2510 continue; 2511 2512 nr_zones++; 2513 } 2514 2515 /* Ensure 2 pageblocks are free to assist fragmentation avoidance */ 2516 recommended_min = pageblock_nr_pages * nr_zones * 2; 2517 2518 /* 2519 * Make sure that on average at least two pageblocks are almost free 2520 * of another type, one for a migratetype to fall back to and a 2521 * second to avoid subsequent fallbacks of other types There are 3 2522 * MIGRATE_TYPES we care about. 2523 */ 2524 recommended_min += pageblock_nr_pages * nr_zones * 2525 MIGRATE_PCPTYPES * MIGRATE_PCPTYPES; 2526 2527 /* don't ever allow to reserve more than 5% of the lowmem */ 2528 recommended_min = min(recommended_min, 2529 (unsigned long) nr_free_buffer_pages() / 20); 2530 recommended_min <<= (PAGE_SHIFT-10); 2531 2532 if (recommended_min > min_free_kbytes) { 2533 if (user_min_free_kbytes >= 0) 2534 pr_info("raising min_free_kbytes from %d to %lu to help transparent hugepage allocations\n", 2535 min_free_kbytes, recommended_min); 2536 2537 min_free_kbytes = recommended_min; 2538 } 2539 2540 update_wmarks: 2541 setup_per_zone_wmarks(); 2542 } 2543 2544 int start_stop_khugepaged(void) 2545 { 2546 int err = 0; 2547 2548 mutex_lock(&khugepaged_mutex); 2549 if (hugepage_flags_enabled()) { 2550 if (!khugepaged_thread) 2551 khugepaged_thread = kthread_run(khugepaged, NULL, 2552 "khugepaged"); 2553 if (IS_ERR(khugepaged_thread)) { 2554 pr_err("khugepaged: kthread_run(khugepaged) failed\n"); 2555 err = PTR_ERR(khugepaged_thread); 2556 khugepaged_thread = NULL; 2557 goto fail; 2558 } 2559 2560 if (!list_empty(&khugepaged_scan.mm_head)) 2561 wake_up_interruptible(&khugepaged_wait); 2562 } else if (khugepaged_thread) { 2563 kthread_stop(khugepaged_thread); 2564 khugepaged_thread = NULL; 2565 } 2566 set_recommended_min_free_kbytes(); 2567 fail: 2568 mutex_unlock(&khugepaged_mutex); 2569 return err; 2570 } 2571 2572 void khugepaged_min_free_kbytes_update(void) 2573 { 2574 mutex_lock(&khugepaged_mutex); 2575 if (hugepage_flags_enabled() && khugepaged_thread) 2576 set_recommended_min_free_kbytes(); 2577 mutex_unlock(&khugepaged_mutex); 2578 } 2579 2580 static int madvise_collapse_errno(enum scan_result r) 2581 { 2582 /* 2583 * MADV_COLLAPSE breaks from existing madvise(2) conventions to provide 2584 * actionable feedback to caller, so they may take an appropriate 2585 * fallback measure depending on the nature of the failure. 2586 */ 2587 switch (r) { 2588 case SCAN_ALLOC_HUGE_PAGE_FAIL: 2589 return -ENOMEM; 2590 case SCAN_CGROUP_CHARGE_FAIL: 2591 return -EBUSY; 2592 /* Resource temporary unavailable - trying again might succeed */ 2593 case SCAN_PAGE_LOCK: 2594 case SCAN_PAGE_LRU: 2595 case SCAN_DEL_PAGE_LRU: 2596 return -EAGAIN; 2597 /* 2598 * Other: Trying again likely not to succeed / error intrinsic to 2599 * specified memory range. khugepaged likely won't be able to collapse 2600 * either. 2601 */ 2602 default: 2603 return -EINVAL; 2604 } 2605 } 2606 2607 int madvise_collapse(struct vm_area_struct *vma, struct vm_area_struct **prev, 2608 unsigned long start, unsigned long end) 2609 { 2610 struct collapse_control *cc; 2611 struct mm_struct *mm = vma->vm_mm; 2612 unsigned long hstart, hend, addr; 2613 int thps = 0, last_fail = SCAN_FAIL; 2614 bool mmap_locked = true; 2615 2616 BUG_ON(vma->vm_start > start); 2617 BUG_ON(vma->vm_end < end); 2618 2619 *prev = vma; 2620 2621 if (!hugepage_vma_check(vma, vma->vm_flags, false, false, false)) 2622 return -EINVAL; 2623 2624 cc = kmalloc(sizeof(*cc), GFP_KERNEL); 2625 if (!cc) 2626 return -ENOMEM; 2627 cc->is_khugepaged = false; 2628 2629 mmgrab(mm); 2630 lru_add_drain_all(); 2631 2632 hstart = (start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK; 2633 hend = end & HPAGE_PMD_MASK; 2634 2635 for (addr = hstart; addr < hend; addr += HPAGE_PMD_SIZE) { 2636 int result = SCAN_FAIL; 2637 2638 if (!mmap_locked) { 2639 cond_resched(); 2640 mmap_read_lock(mm); 2641 mmap_locked = true; 2642 result = hugepage_vma_revalidate(mm, addr, false, &vma, 2643 cc); 2644 if (result != SCAN_SUCCEED) { 2645 last_fail = result; 2646 goto out_nolock; 2647 } 2648 2649 hend = vma->vm_end & HPAGE_PMD_MASK; 2650 } 2651 mmap_assert_locked(mm); 2652 memset(cc->node_load, 0, sizeof(cc->node_load)); 2653 nodes_clear(cc->alloc_nmask); 2654 if (IS_ENABLED(CONFIG_SHMEM) && vma->vm_file) { 2655 struct file *file = get_file(vma->vm_file); 2656 pgoff_t pgoff = linear_page_index(vma, addr); 2657 2658 mmap_read_unlock(mm); 2659 mmap_locked = false; 2660 result = hpage_collapse_scan_file(mm, addr, file, pgoff, 2661 cc); 2662 fput(file); 2663 } else { 2664 result = hpage_collapse_scan_pmd(mm, vma, addr, 2665 &mmap_locked, cc); 2666 } 2667 if (!mmap_locked) 2668 *prev = NULL; /* Tell caller we dropped mmap_lock */ 2669 2670 handle_result: 2671 switch (result) { 2672 case SCAN_SUCCEED: 2673 case SCAN_PMD_MAPPED: 2674 ++thps; 2675 break; 2676 case SCAN_PTE_MAPPED_HUGEPAGE: 2677 BUG_ON(mmap_locked); 2678 BUG_ON(*prev); 2679 mmap_write_lock(mm); 2680 result = collapse_pte_mapped_thp(mm, addr, true); 2681 mmap_write_unlock(mm); 2682 goto handle_result; 2683 /* Whitelisted set of results where continuing OK */ 2684 case SCAN_PMD_NULL: 2685 case SCAN_PTE_NON_PRESENT: 2686 case SCAN_PTE_UFFD_WP: 2687 case SCAN_PAGE_RO: 2688 case SCAN_LACK_REFERENCED_PAGE: 2689 case SCAN_PAGE_NULL: 2690 case SCAN_PAGE_COUNT: 2691 case SCAN_PAGE_LOCK: 2692 case SCAN_PAGE_COMPOUND: 2693 case SCAN_PAGE_LRU: 2694 case SCAN_DEL_PAGE_LRU: 2695 last_fail = result; 2696 break; 2697 default: 2698 last_fail = result; 2699 /* Other error, exit */ 2700 goto out_maybelock; 2701 } 2702 } 2703 2704 out_maybelock: 2705 /* Caller expects us to hold mmap_lock on return */ 2706 if (!mmap_locked) 2707 mmap_read_lock(mm); 2708 out_nolock: 2709 mmap_assert_locked(mm); 2710 mmdrop(mm); 2711 kfree(cc); 2712 2713 return thps == ((hend - hstart) >> HPAGE_PMD_SHIFT) ? 0 2714 : madvise_collapse_errno(last_fail); 2715 } 2716