1 /* 2 * Copyright (C) 2009 Red Hat, Inc. 3 * 4 * This work is licensed under the terms of the GNU GPL, version 2. See 5 * the COPYING file in the top-level directory. 6 */ 7 8 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 9 10 #include <linux/mm.h> 11 #include <linux/sched.h> 12 #include <linux/highmem.h> 13 #include <linux/hugetlb.h> 14 #include <linux/mmu_notifier.h> 15 #include <linux/rmap.h> 16 #include <linux/swap.h> 17 #include <linux/shrinker.h> 18 #include <linux/mm_inline.h> 19 #include <linux/swapops.h> 20 #include <linux/dax.h> 21 #include <linux/kthread.h> 22 #include <linux/khugepaged.h> 23 #include <linux/freezer.h> 24 #include <linux/pfn_t.h> 25 #include <linux/mman.h> 26 #include <linux/memremap.h> 27 #include <linux/pagemap.h> 28 #include <linux/debugfs.h> 29 #include <linux/migrate.h> 30 #include <linux/hashtable.h> 31 #include <linux/userfaultfd_k.h> 32 #include <linux/page_idle.h> 33 34 #include <asm/tlb.h> 35 #include <asm/pgalloc.h> 36 #include "internal.h" 37 38 enum scan_result { 39 SCAN_FAIL, 40 SCAN_SUCCEED, 41 SCAN_PMD_NULL, 42 SCAN_EXCEED_NONE_PTE, 43 SCAN_PTE_NON_PRESENT, 44 SCAN_PAGE_RO, 45 SCAN_NO_REFERENCED_PAGE, 46 SCAN_PAGE_NULL, 47 SCAN_SCAN_ABORT, 48 SCAN_PAGE_COUNT, 49 SCAN_PAGE_LRU, 50 SCAN_PAGE_LOCK, 51 SCAN_PAGE_ANON, 52 SCAN_PAGE_COMPOUND, 53 SCAN_ANY_PROCESS, 54 SCAN_VMA_NULL, 55 SCAN_VMA_CHECK, 56 SCAN_ADDRESS_RANGE, 57 SCAN_SWAP_CACHE_PAGE, 58 SCAN_DEL_PAGE_LRU, 59 SCAN_ALLOC_HUGE_PAGE_FAIL, 60 SCAN_CGROUP_CHARGE_FAIL 61 }; 62 63 #define CREATE_TRACE_POINTS 64 #include <trace/events/huge_memory.h> 65 66 /* 67 * By default transparent hugepage support is disabled in order that avoid 68 * to risk increase the memory footprint of applications without a guaranteed 69 * benefit. When transparent hugepage support is enabled, is for all mappings, 70 * and khugepaged scans all mappings. 71 * Defrag is invoked by khugepaged hugepage allocations and by page faults 72 * for all hugepage allocations. 73 */ 74 unsigned long transparent_hugepage_flags __read_mostly = 75 #ifdef CONFIG_TRANSPARENT_HUGEPAGE_ALWAYS 76 (1<<TRANSPARENT_HUGEPAGE_FLAG)| 77 #endif 78 #ifdef CONFIG_TRANSPARENT_HUGEPAGE_MADVISE 79 (1<<TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG)| 80 #endif 81 (1<<TRANSPARENT_HUGEPAGE_DEFRAG_FLAG)| 82 (1<<TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG)| 83 (1<<TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG); 84 85 /* default scan 8*512 pte (or vmas) every 30 second */ 86 static unsigned int khugepaged_pages_to_scan __read_mostly = HPAGE_PMD_NR*8; 87 static unsigned int khugepaged_pages_collapsed; 88 static unsigned int khugepaged_full_scans; 89 static unsigned int khugepaged_scan_sleep_millisecs __read_mostly = 10000; 90 /* during fragmentation poll the hugepage allocator once every minute */ 91 static unsigned int khugepaged_alloc_sleep_millisecs __read_mostly = 60000; 92 static struct task_struct *khugepaged_thread __read_mostly; 93 static DEFINE_MUTEX(khugepaged_mutex); 94 static DEFINE_SPINLOCK(khugepaged_mm_lock); 95 static DECLARE_WAIT_QUEUE_HEAD(khugepaged_wait); 96 /* 97 * default collapse hugepages if there is at least one pte mapped like 98 * it would have happened if the vma was large enough during page 99 * fault. 100 */ 101 static unsigned int khugepaged_max_ptes_none __read_mostly = HPAGE_PMD_NR-1; 102 103 static int khugepaged(void *none); 104 static int khugepaged_slab_init(void); 105 static void khugepaged_slab_exit(void); 106 107 #define MM_SLOTS_HASH_BITS 10 108 static __read_mostly DEFINE_HASHTABLE(mm_slots_hash, MM_SLOTS_HASH_BITS); 109 110 static struct kmem_cache *mm_slot_cache __read_mostly; 111 112 /** 113 * struct mm_slot - hash lookup from mm to mm_slot 114 * @hash: hash collision list 115 * @mm_node: khugepaged scan list headed in khugepaged_scan.mm_head 116 * @mm: the mm that this information is valid for 117 */ 118 struct mm_slot { 119 struct hlist_node hash; 120 struct list_head mm_node; 121 struct mm_struct *mm; 122 }; 123 124 /** 125 * struct khugepaged_scan - cursor for scanning 126 * @mm_head: the head of the mm list to scan 127 * @mm_slot: the current mm_slot we are scanning 128 * @address: the next address inside that to be scanned 129 * 130 * There is only the one khugepaged_scan instance of this cursor structure. 131 */ 132 struct khugepaged_scan { 133 struct list_head mm_head; 134 struct mm_slot *mm_slot; 135 unsigned long address; 136 }; 137 static struct khugepaged_scan khugepaged_scan = { 138 .mm_head = LIST_HEAD_INIT(khugepaged_scan.mm_head), 139 }; 140 141 static DEFINE_SPINLOCK(split_queue_lock); 142 static LIST_HEAD(split_queue); 143 static unsigned long split_queue_len; 144 static struct shrinker deferred_split_shrinker; 145 146 static void set_recommended_min_free_kbytes(void) 147 { 148 struct zone *zone; 149 int nr_zones = 0; 150 unsigned long recommended_min; 151 152 for_each_populated_zone(zone) 153 nr_zones++; 154 155 /* Ensure 2 pageblocks are free to assist fragmentation avoidance */ 156 recommended_min = pageblock_nr_pages * nr_zones * 2; 157 158 /* 159 * Make sure that on average at least two pageblocks are almost free 160 * of another type, one for a migratetype to fall back to and a 161 * second to avoid subsequent fallbacks of other types There are 3 162 * MIGRATE_TYPES we care about. 163 */ 164 recommended_min += pageblock_nr_pages * nr_zones * 165 MIGRATE_PCPTYPES * MIGRATE_PCPTYPES; 166 167 /* don't ever allow to reserve more than 5% of the lowmem */ 168 recommended_min = min(recommended_min, 169 (unsigned long) nr_free_buffer_pages() / 20); 170 recommended_min <<= (PAGE_SHIFT-10); 171 172 if (recommended_min > min_free_kbytes) { 173 if (user_min_free_kbytes >= 0) 174 pr_info("raising min_free_kbytes from %d to %lu " 175 "to help transparent hugepage allocations\n", 176 min_free_kbytes, recommended_min); 177 178 min_free_kbytes = recommended_min; 179 } 180 setup_per_zone_wmarks(); 181 } 182 183 static int start_stop_khugepaged(void) 184 { 185 int err = 0; 186 if (khugepaged_enabled()) { 187 if (!khugepaged_thread) 188 khugepaged_thread = kthread_run(khugepaged, NULL, 189 "khugepaged"); 190 if (IS_ERR(khugepaged_thread)) { 191 pr_err("khugepaged: kthread_run(khugepaged) failed\n"); 192 err = PTR_ERR(khugepaged_thread); 193 khugepaged_thread = NULL; 194 goto fail; 195 } 196 197 if (!list_empty(&khugepaged_scan.mm_head)) 198 wake_up_interruptible(&khugepaged_wait); 199 200 set_recommended_min_free_kbytes(); 201 } else if (khugepaged_thread) { 202 kthread_stop(khugepaged_thread); 203 khugepaged_thread = NULL; 204 } 205 fail: 206 return err; 207 } 208 209 static atomic_t huge_zero_refcount; 210 struct page *huge_zero_page __read_mostly; 211 212 struct page *get_huge_zero_page(void) 213 { 214 struct page *zero_page; 215 retry: 216 if (likely(atomic_inc_not_zero(&huge_zero_refcount))) 217 return READ_ONCE(huge_zero_page); 218 219 zero_page = alloc_pages((GFP_TRANSHUGE | __GFP_ZERO) & ~__GFP_MOVABLE, 220 HPAGE_PMD_ORDER); 221 if (!zero_page) { 222 count_vm_event(THP_ZERO_PAGE_ALLOC_FAILED); 223 return NULL; 224 } 225 count_vm_event(THP_ZERO_PAGE_ALLOC); 226 preempt_disable(); 227 if (cmpxchg(&huge_zero_page, NULL, zero_page)) { 228 preempt_enable(); 229 __free_pages(zero_page, compound_order(zero_page)); 230 goto retry; 231 } 232 233 /* We take additional reference here. It will be put back by shrinker */ 234 atomic_set(&huge_zero_refcount, 2); 235 preempt_enable(); 236 return READ_ONCE(huge_zero_page); 237 } 238 239 static void put_huge_zero_page(void) 240 { 241 /* 242 * Counter should never go to zero here. Only shrinker can put 243 * last reference. 244 */ 245 BUG_ON(atomic_dec_and_test(&huge_zero_refcount)); 246 } 247 248 static unsigned long shrink_huge_zero_page_count(struct shrinker *shrink, 249 struct shrink_control *sc) 250 { 251 /* we can free zero page only if last reference remains */ 252 return atomic_read(&huge_zero_refcount) == 1 ? HPAGE_PMD_NR : 0; 253 } 254 255 static unsigned long shrink_huge_zero_page_scan(struct shrinker *shrink, 256 struct shrink_control *sc) 257 { 258 if (atomic_cmpxchg(&huge_zero_refcount, 1, 0) == 1) { 259 struct page *zero_page = xchg(&huge_zero_page, NULL); 260 BUG_ON(zero_page == NULL); 261 __free_pages(zero_page, compound_order(zero_page)); 262 return HPAGE_PMD_NR; 263 } 264 265 return 0; 266 } 267 268 static struct shrinker huge_zero_page_shrinker = { 269 .count_objects = shrink_huge_zero_page_count, 270 .scan_objects = shrink_huge_zero_page_scan, 271 .seeks = DEFAULT_SEEKS, 272 }; 273 274 #ifdef CONFIG_SYSFS 275 276 static ssize_t double_flag_show(struct kobject *kobj, 277 struct kobj_attribute *attr, char *buf, 278 enum transparent_hugepage_flag enabled, 279 enum transparent_hugepage_flag req_madv) 280 { 281 if (test_bit(enabled, &transparent_hugepage_flags)) { 282 VM_BUG_ON(test_bit(req_madv, &transparent_hugepage_flags)); 283 return sprintf(buf, "[always] madvise never\n"); 284 } else if (test_bit(req_madv, &transparent_hugepage_flags)) 285 return sprintf(buf, "always [madvise] never\n"); 286 else 287 return sprintf(buf, "always madvise [never]\n"); 288 } 289 static ssize_t double_flag_store(struct kobject *kobj, 290 struct kobj_attribute *attr, 291 const char *buf, size_t count, 292 enum transparent_hugepage_flag enabled, 293 enum transparent_hugepage_flag req_madv) 294 { 295 if (!memcmp("always", buf, 296 min(sizeof("always")-1, count))) { 297 set_bit(enabled, &transparent_hugepage_flags); 298 clear_bit(req_madv, &transparent_hugepage_flags); 299 } else if (!memcmp("madvise", buf, 300 min(sizeof("madvise")-1, count))) { 301 clear_bit(enabled, &transparent_hugepage_flags); 302 set_bit(req_madv, &transparent_hugepage_flags); 303 } else if (!memcmp("never", buf, 304 min(sizeof("never")-1, count))) { 305 clear_bit(enabled, &transparent_hugepage_flags); 306 clear_bit(req_madv, &transparent_hugepage_flags); 307 } else 308 return -EINVAL; 309 310 return count; 311 } 312 313 static ssize_t enabled_show(struct kobject *kobj, 314 struct kobj_attribute *attr, char *buf) 315 { 316 return double_flag_show(kobj, attr, buf, 317 TRANSPARENT_HUGEPAGE_FLAG, 318 TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG); 319 } 320 static ssize_t enabled_store(struct kobject *kobj, 321 struct kobj_attribute *attr, 322 const char *buf, size_t count) 323 { 324 ssize_t ret; 325 326 ret = double_flag_store(kobj, attr, buf, count, 327 TRANSPARENT_HUGEPAGE_FLAG, 328 TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG); 329 330 if (ret > 0) { 331 int err; 332 333 mutex_lock(&khugepaged_mutex); 334 err = start_stop_khugepaged(); 335 mutex_unlock(&khugepaged_mutex); 336 337 if (err) 338 ret = err; 339 } 340 341 return ret; 342 } 343 static struct kobj_attribute enabled_attr = 344 __ATTR(enabled, 0644, enabled_show, enabled_store); 345 346 static ssize_t single_flag_show(struct kobject *kobj, 347 struct kobj_attribute *attr, char *buf, 348 enum transparent_hugepage_flag flag) 349 { 350 return sprintf(buf, "%d\n", 351 !!test_bit(flag, &transparent_hugepage_flags)); 352 } 353 354 static ssize_t single_flag_store(struct kobject *kobj, 355 struct kobj_attribute *attr, 356 const char *buf, size_t count, 357 enum transparent_hugepage_flag flag) 358 { 359 unsigned long value; 360 int ret; 361 362 ret = kstrtoul(buf, 10, &value); 363 if (ret < 0) 364 return ret; 365 if (value > 1) 366 return -EINVAL; 367 368 if (value) 369 set_bit(flag, &transparent_hugepage_flags); 370 else 371 clear_bit(flag, &transparent_hugepage_flags); 372 373 return count; 374 } 375 376 /* 377 * Currently defrag only disables __GFP_NOWAIT for allocation. A blind 378 * __GFP_REPEAT is too aggressive, it's never worth swapping tons of 379 * memory just to allocate one more hugepage. 380 */ 381 static ssize_t defrag_show(struct kobject *kobj, 382 struct kobj_attribute *attr, char *buf) 383 { 384 return double_flag_show(kobj, attr, buf, 385 TRANSPARENT_HUGEPAGE_DEFRAG_FLAG, 386 TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG); 387 } 388 static ssize_t defrag_store(struct kobject *kobj, 389 struct kobj_attribute *attr, 390 const char *buf, size_t count) 391 { 392 return double_flag_store(kobj, attr, buf, count, 393 TRANSPARENT_HUGEPAGE_DEFRAG_FLAG, 394 TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG); 395 } 396 static struct kobj_attribute defrag_attr = 397 __ATTR(defrag, 0644, defrag_show, defrag_store); 398 399 static ssize_t use_zero_page_show(struct kobject *kobj, 400 struct kobj_attribute *attr, char *buf) 401 { 402 return single_flag_show(kobj, attr, buf, 403 TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG); 404 } 405 static ssize_t use_zero_page_store(struct kobject *kobj, 406 struct kobj_attribute *attr, const char *buf, size_t count) 407 { 408 return single_flag_store(kobj, attr, buf, count, 409 TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG); 410 } 411 static struct kobj_attribute use_zero_page_attr = 412 __ATTR(use_zero_page, 0644, use_zero_page_show, use_zero_page_store); 413 #ifdef CONFIG_DEBUG_VM 414 static ssize_t debug_cow_show(struct kobject *kobj, 415 struct kobj_attribute *attr, char *buf) 416 { 417 return single_flag_show(kobj, attr, buf, 418 TRANSPARENT_HUGEPAGE_DEBUG_COW_FLAG); 419 } 420 static ssize_t debug_cow_store(struct kobject *kobj, 421 struct kobj_attribute *attr, 422 const char *buf, size_t count) 423 { 424 return single_flag_store(kobj, attr, buf, count, 425 TRANSPARENT_HUGEPAGE_DEBUG_COW_FLAG); 426 } 427 static struct kobj_attribute debug_cow_attr = 428 __ATTR(debug_cow, 0644, debug_cow_show, debug_cow_store); 429 #endif /* CONFIG_DEBUG_VM */ 430 431 static struct attribute *hugepage_attr[] = { 432 &enabled_attr.attr, 433 &defrag_attr.attr, 434 &use_zero_page_attr.attr, 435 #ifdef CONFIG_DEBUG_VM 436 &debug_cow_attr.attr, 437 #endif 438 NULL, 439 }; 440 441 static struct attribute_group hugepage_attr_group = { 442 .attrs = hugepage_attr, 443 }; 444 445 static ssize_t scan_sleep_millisecs_show(struct kobject *kobj, 446 struct kobj_attribute *attr, 447 char *buf) 448 { 449 return sprintf(buf, "%u\n", khugepaged_scan_sleep_millisecs); 450 } 451 452 static ssize_t scan_sleep_millisecs_store(struct kobject *kobj, 453 struct kobj_attribute *attr, 454 const char *buf, size_t count) 455 { 456 unsigned long msecs; 457 int err; 458 459 err = kstrtoul(buf, 10, &msecs); 460 if (err || msecs > UINT_MAX) 461 return -EINVAL; 462 463 khugepaged_scan_sleep_millisecs = msecs; 464 wake_up_interruptible(&khugepaged_wait); 465 466 return count; 467 } 468 static struct kobj_attribute scan_sleep_millisecs_attr = 469 __ATTR(scan_sleep_millisecs, 0644, scan_sleep_millisecs_show, 470 scan_sleep_millisecs_store); 471 472 static ssize_t alloc_sleep_millisecs_show(struct kobject *kobj, 473 struct kobj_attribute *attr, 474 char *buf) 475 { 476 return sprintf(buf, "%u\n", khugepaged_alloc_sleep_millisecs); 477 } 478 479 static ssize_t alloc_sleep_millisecs_store(struct kobject *kobj, 480 struct kobj_attribute *attr, 481 const char *buf, size_t count) 482 { 483 unsigned long msecs; 484 int err; 485 486 err = kstrtoul(buf, 10, &msecs); 487 if (err || msecs > UINT_MAX) 488 return -EINVAL; 489 490 khugepaged_alloc_sleep_millisecs = msecs; 491 wake_up_interruptible(&khugepaged_wait); 492 493 return count; 494 } 495 static struct kobj_attribute alloc_sleep_millisecs_attr = 496 __ATTR(alloc_sleep_millisecs, 0644, alloc_sleep_millisecs_show, 497 alloc_sleep_millisecs_store); 498 499 static ssize_t pages_to_scan_show(struct kobject *kobj, 500 struct kobj_attribute *attr, 501 char *buf) 502 { 503 return sprintf(buf, "%u\n", khugepaged_pages_to_scan); 504 } 505 static ssize_t pages_to_scan_store(struct kobject *kobj, 506 struct kobj_attribute *attr, 507 const char *buf, size_t count) 508 { 509 int err; 510 unsigned long pages; 511 512 err = kstrtoul(buf, 10, &pages); 513 if (err || !pages || pages > UINT_MAX) 514 return -EINVAL; 515 516 khugepaged_pages_to_scan = pages; 517 518 return count; 519 } 520 static struct kobj_attribute pages_to_scan_attr = 521 __ATTR(pages_to_scan, 0644, pages_to_scan_show, 522 pages_to_scan_store); 523 524 static ssize_t pages_collapsed_show(struct kobject *kobj, 525 struct kobj_attribute *attr, 526 char *buf) 527 { 528 return sprintf(buf, "%u\n", khugepaged_pages_collapsed); 529 } 530 static struct kobj_attribute pages_collapsed_attr = 531 __ATTR_RO(pages_collapsed); 532 533 static ssize_t full_scans_show(struct kobject *kobj, 534 struct kobj_attribute *attr, 535 char *buf) 536 { 537 return sprintf(buf, "%u\n", khugepaged_full_scans); 538 } 539 static struct kobj_attribute full_scans_attr = 540 __ATTR_RO(full_scans); 541 542 static ssize_t khugepaged_defrag_show(struct kobject *kobj, 543 struct kobj_attribute *attr, char *buf) 544 { 545 return single_flag_show(kobj, attr, buf, 546 TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG); 547 } 548 static ssize_t khugepaged_defrag_store(struct kobject *kobj, 549 struct kobj_attribute *attr, 550 const char *buf, size_t count) 551 { 552 return single_flag_store(kobj, attr, buf, count, 553 TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG); 554 } 555 static struct kobj_attribute khugepaged_defrag_attr = 556 __ATTR(defrag, 0644, khugepaged_defrag_show, 557 khugepaged_defrag_store); 558 559 /* 560 * max_ptes_none controls if khugepaged should collapse hugepages over 561 * any unmapped ptes in turn potentially increasing the memory 562 * footprint of the vmas. When max_ptes_none is 0 khugepaged will not 563 * reduce the available free memory in the system as it 564 * runs. Increasing max_ptes_none will instead potentially reduce the 565 * free memory in the system during the khugepaged scan. 566 */ 567 static ssize_t khugepaged_max_ptes_none_show(struct kobject *kobj, 568 struct kobj_attribute *attr, 569 char *buf) 570 { 571 return sprintf(buf, "%u\n", khugepaged_max_ptes_none); 572 } 573 static ssize_t khugepaged_max_ptes_none_store(struct kobject *kobj, 574 struct kobj_attribute *attr, 575 const char *buf, size_t count) 576 { 577 int err; 578 unsigned long max_ptes_none; 579 580 err = kstrtoul(buf, 10, &max_ptes_none); 581 if (err || max_ptes_none > HPAGE_PMD_NR-1) 582 return -EINVAL; 583 584 khugepaged_max_ptes_none = max_ptes_none; 585 586 return count; 587 } 588 static struct kobj_attribute khugepaged_max_ptes_none_attr = 589 __ATTR(max_ptes_none, 0644, khugepaged_max_ptes_none_show, 590 khugepaged_max_ptes_none_store); 591 592 static struct attribute *khugepaged_attr[] = { 593 &khugepaged_defrag_attr.attr, 594 &khugepaged_max_ptes_none_attr.attr, 595 &pages_to_scan_attr.attr, 596 &pages_collapsed_attr.attr, 597 &full_scans_attr.attr, 598 &scan_sleep_millisecs_attr.attr, 599 &alloc_sleep_millisecs_attr.attr, 600 NULL, 601 }; 602 603 static struct attribute_group khugepaged_attr_group = { 604 .attrs = khugepaged_attr, 605 .name = "khugepaged", 606 }; 607 608 static int __init hugepage_init_sysfs(struct kobject **hugepage_kobj) 609 { 610 int err; 611 612 *hugepage_kobj = kobject_create_and_add("transparent_hugepage", mm_kobj); 613 if (unlikely(!*hugepage_kobj)) { 614 pr_err("failed to create transparent hugepage kobject\n"); 615 return -ENOMEM; 616 } 617 618 err = sysfs_create_group(*hugepage_kobj, &hugepage_attr_group); 619 if (err) { 620 pr_err("failed to register transparent hugepage group\n"); 621 goto delete_obj; 622 } 623 624 err = sysfs_create_group(*hugepage_kobj, &khugepaged_attr_group); 625 if (err) { 626 pr_err("failed to register transparent hugepage group\n"); 627 goto remove_hp_group; 628 } 629 630 return 0; 631 632 remove_hp_group: 633 sysfs_remove_group(*hugepage_kobj, &hugepage_attr_group); 634 delete_obj: 635 kobject_put(*hugepage_kobj); 636 return err; 637 } 638 639 static void __init hugepage_exit_sysfs(struct kobject *hugepage_kobj) 640 { 641 sysfs_remove_group(hugepage_kobj, &khugepaged_attr_group); 642 sysfs_remove_group(hugepage_kobj, &hugepage_attr_group); 643 kobject_put(hugepage_kobj); 644 } 645 #else 646 static inline int hugepage_init_sysfs(struct kobject **hugepage_kobj) 647 { 648 return 0; 649 } 650 651 static inline void hugepage_exit_sysfs(struct kobject *hugepage_kobj) 652 { 653 } 654 #endif /* CONFIG_SYSFS */ 655 656 static int __init hugepage_init(void) 657 { 658 int err; 659 struct kobject *hugepage_kobj; 660 661 if (!has_transparent_hugepage()) { 662 transparent_hugepage_flags = 0; 663 return -EINVAL; 664 } 665 666 err = hugepage_init_sysfs(&hugepage_kobj); 667 if (err) 668 goto err_sysfs; 669 670 err = khugepaged_slab_init(); 671 if (err) 672 goto err_slab; 673 674 err = register_shrinker(&huge_zero_page_shrinker); 675 if (err) 676 goto err_hzp_shrinker; 677 err = register_shrinker(&deferred_split_shrinker); 678 if (err) 679 goto err_split_shrinker; 680 681 /* 682 * By default disable transparent hugepages on smaller systems, 683 * where the extra memory used could hurt more than TLB overhead 684 * is likely to save. The admin can still enable it through /sys. 685 */ 686 if (totalram_pages < (512 << (20 - PAGE_SHIFT))) { 687 transparent_hugepage_flags = 0; 688 return 0; 689 } 690 691 err = start_stop_khugepaged(); 692 if (err) 693 goto err_khugepaged; 694 695 return 0; 696 err_khugepaged: 697 unregister_shrinker(&deferred_split_shrinker); 698 err_split_shrinker: 699 unregister_shrinker(&huge_zero_page_shrinker); 700 err_hzp_shrinker: 701 khugepaged_slab_exit(); 702 err_slab: 703 hugepage_exit_sysfs(hugepage_kobj); 704 err_sysfs: 705 return err; 706 } 707 subsys_initcall(hugepage_init); 708 709 static int __init setup_transparent_hugepage(char *str) 710 { 711 int ret = 0; 712 if (!str) 713 goto out; 714 if (!strcmp(str, "always")) { 715 set_bit(TRANSPARENT_HUGEPAGE_FLAG, 716 &transparent_hugepage_flags); 717 clear_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, 718 &transparent_hugepage_flags); 719 ret = 1; 720 } else if (!strcmp(str, "madvise")) { 721 clear_bit(TRANSPARENT_HUGEPAGE_FLAG, 722 &transparent_hugepage_flags); 723 set_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, 724 &transparent_hugepage_flags); 725 ret = 1; 726 } else if (!strcmp(str, "never")) { 727 clear_bit(TRANSPARENT_HUGEPAGE_FLAG, 728 &transparent_hugepage_flags); 729 clear_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, 730 &transparent_hugepage_flags); 731 ret = 1; 732 } 733 out: 734 if (!ret) 735 pr_warn("transparent_hugepage= cannot parse, ignored\n"); 736 return ret; 737 } 738 __setup("transparent_hugepage=", setup_transparent_hugepage); 739 740 pmd_t maybe_pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma) 741 { 742 if (likely(vma->vm_flags & VM_WRITE)) 743 pmd = pmd_mkwrite(pmd); 744 return pmd; 745 } 746 747 static inline pmd_t mk_huge_pmd(struct page *page, pgprot_t prot) 748 { 749 pmd_t entry; 750 entry = mk_pmd(page, prot); 751 entry = pmd_mkhuge(entry); 752 return entry; 753 } 754 755 static inline struct list_head *page_deferred_list(struct page *page) 756 { 757 /* 758 * ->lru in the tail pages is occupied by compound_head. 759 * Let's use ->mapping + ->index in the second tail page as list_head. 760 */ 761 return (struct list_head *)&page[2].mapping; 762 } 763 764 void prep_transhuge_page(struct page *page) 765 { 766 /* 767 * we use page->mapping and page->indexlru in second tail page 768 * as list_head: assuming THP order >= 2 769 */ 770 BUILD_BUG_ON(HPAGE_PMD_ORDER < 2); 771 772 INIT_LIST_HEAD(page_deferred_list(page)); 773 set_compound_page_dtor(page, TRANSHUGE_PAGE_DTOR); 774 } 775 776 static int __do_huge_pmd_anonymous_page(struct mm_struct *mm, 777 struct vm_area_struct *vma, 778 unsigned long address, pmd_t *pmd, 779 struct page *page, gfp_t gfp, 780 unsigned int flags) 781 { 782 struct mem_cgroup *memcg; 783 pgtable_t pgtable; 784 spinlock_t *ptl; 785 unsigned long haddr = address & HPAGE_PMD_MASK; 786 787 VM_BUG_ON_PAGE(!PageCompound(page), page); 788 789 if (mem_cgroup_try_charge(page, mm, gfp, &memcg, true)) { 790 put_page(page); 791 count_vm_event(THP_FAULT_FALLBACK); 792 return VM_FAULT_FALLBACK; 793 } 794 795 pgtable = pte_alloc_one(mm, haddr); 796 if (unlikely(!pgtable)) { 797 mem_cgroup_cancel_charge(page, memcg, true); 798 put_page(page); 799 return VM_FAULT_OOM; 800 } 801 802 clear_huge_page(page, haddr, HPAGE_PMD_NR); 803 /* 804 * The memory barrier inside __SetPageUptodate makes sure that 805 * clear_huge_page writes become visible before the set_pmd_at() 806 * write. 807 */ 808 __SetPageUptodate(page); 809 810 ptl = pmd_lock(mm, pmd); 811 if (unlikely(!pmd_none(*pmd))) { 812 spin_unlock(ptl); 813 mem_cgroup_cancel_charge(page, memcg, true); 814 put_page(page); 815 pte_free(mm, pgtable); 816 } else { 817 pmd_t entry; 818 819 /* Deliver the page fault to userland */ 820 if (userfaultfd_missing(vma)) { 821 int ret; 822 823 spin_unlock(ptl); 824 mem_cgroup_cancel_charge(page, memcg, true); 825 put_page(page); 826 pte_free(mm, pgtable); 827 ret = handle_userfault(vma, address, flags, 828 VM_UFFD_MISSING); 829 VM_BUG_ON(ret & VM_FAULT_FALLBACK); 830 return ret; 831 } 832 833 entry = mk_huge_pmd(page, vma->vm_page_prot); 834 entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma); 835 page_add_new_anon_rmap(page, vma, haddr, true); 836 mem_cgroup_commit_charge(page, memcg, false, true); 837 lru_cache_add_active_or_unevictable(page, vma); 838 pgtable_trans_huge_deposit(mm, pmd, pgtable); 839 set_pmd_at(mm, haddr, pmd, entry); 840 add_mm_counter(mm, MM_ANONPAGES, HPAGE_PMD_NR); 841 atomic_long_inc(&mm->nr_ptes); 842 spin_unlock(ptl); 843 count_vm_event(THP_FAULT_ALLOC); 844 } 845 846 return 0; 847 } 848 849 static inline gfp_t alloc_hugepage_gfpmask(int defrag, gfp_t extra_gfp) 850 { 851 return (GFP_TRANSHUGE & ~(defrag ? 0 : __GFP_RECLAIM)) | extra_gfp; 852 } 853 854 /* Caller must hold page table lock. */ 855 static bool set_huge_zero_page(pgtable_t pgtable, struct mm_struct *mm, 856 struct vm_area_struct *vma, unsigned long haddr, pmd_t *pmd, 857 struct page *zero_page) 858 { 859 pmd_t entry; 860 if (!pmd_none(*pmd)) 861 return false; 862 entry = mk_pmd(zero_page, vma->vm_page_prot); 863 entry = pmd_mkhuge(entry); 864 pgtable_trans_huge_deposit(mm, pmd, pgtable); 865 set_pmd_at(mm, haddr, pmd, entry); 866 atomic_long_inc(&mm->nr_ptes); 867 return true; 868 } 869 870 int do_huge_pmd_anonymous_page(struct mm_struct *mm, struct vm_area_struct *vma, 871 unsigned long address, pmd_t *pmd, 872 unsigned int flags) 873 { 874 gfp_t gfp; 875 struct page *page; 876 unsigned long haddr = address & HPAGE_PMD_MASK; 877 878 if (haddr < vma->vm_start || haddr + HPAGE_PMD_SIZE > vma->vm_end) 879 return VM_FAULT_FALLBACK; 880 if (unlikely(anon_vma_prepare(vma))) 881 return VM_FAULT_OOM; 882 if (unlikely(khugepaged_enter(vma, vma->vm_flags))) 883 return VM_FAULT_OOM; 884 if (!(flags & FAULT_FLAG_WRITE) && !mm_forbids_zeropage(mm) && 885 transparent_hugepage_use_zero_page()) { 886 spinlock_t *ptl; 887 pgtable_t pgtable; 888 struct page *zero_page; 889 bool set; 890 int ret; 891 pgtable = pte_alloc_one(mm, haddr); 892 if (unlikely(!pgtable)) 893 return VM_FAULT_OOM; 894 zero_page = get_huge_zero_page(); 895 if (unlikely(!zero_page)) { 896 pte_free(mm, pgtable); 897 count_vm_event(THP_FAULT_FALLBACK); 898 return VM_FAULT_FALLBACK; 899 } 900 ptl = pmd_lock(mm, pmd); 901 ret = 0; 902 set = false; 903 if (pmd_none(*pmd)) { 904 if (userfaultfd_missing(vma)) { 905 spin_unlock(ptl); 906 ret = handle_userfault(vma, address, flags, 907 VM_UFFD_MISSING); 908 VM_BUG_ON(ret & VM_FAULT_FALLBACK); 909 } else { 910 set_huge_zero_page(pgtable, mm, vma, 911 haddr, pmd, 912 zero_page); 913 spin_unlock(ptl); 914 set = true; 915 } 916 } else 917 spin_unlock(ptl); 918 if (!set) { 919 pte_free(mm, pgtable); 920 put_huge_zero_page(); 921 } 922 return ret; 923 } 924 gfp = alloc_hugepage_gfpmask(transparent_hugepage_defrag(vma), 0); 925 page = alloc_hugepage_vma(gfp, vma, haddr, HPAGE_PMD_ORDER); 926 if (unlikely(!page)) { 927 count_vm_event(THP_FAULT_FALLBACK); 928 return VM_FAULT_FALLBACK; 929 } 930 prep_transhuge_page(page); 931 return __do_huge_pmd_anonymous_page(mm, vma, address, pmd, page, gfp, 932 flags); 933 } 934 935 static void insert_pfn_pmd(struct vm_area_struct *vma, unsigned long addr, 936 pmd_t *pmd, pfn_t pfn, pgprot_t prot, bool write) 937 { 938 struct mm_struct *mm = vma->vm_mm; 939 pmd_t entry; 940 spinlock_t *ptl; 941 942 ptl = pmd_lock(mm, pmd); 943 entry = pmd_mkhuge(pfn_t_pmd(pfn, prot)); 944 if (pfn_t_devmap(pfn)) 945 entry = pmd_mkdevmap(entry); 946 if (write) { 947 entry = pmd_mkyoung(pmd_mkdirty(entry)); 948 entry = maybe_pmd_mkwrite(entry, vma); 949 } 950 set_pmd_at(mm, addr, pmd, entry); 951 update_mmu_cache_pmd(vma, addr, pmd); 952 spin_unlock(ptl); 953 } 954 955 int vmf_insert_pfn_pmd(struct vm_area_struct *vma, unsigned long addr, 956 pmd_t *pmd, pfn_t pfn, bool write) 957 { 958 pgprot_t pgprot = vma->vm_page_prot; 959 /* 960 * If we had pmd_special, we could avoid all these restrictions, 961 * but we need to be consistent with PTEs and architectures that 962 * can't support a 'special' bit. 963 */ 964 BUG_ON(!(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP))); 965 BUG_ON((vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) == 966 (VM_PFNMAP|VM_MIXEDMAP)); 967 BUG_ON((vma->vm_flags & VM_PFNMAP) && is_cow_mapping(vma->vm_flags)); 968 BUG_ON(!pfn_t_devmap(pfn)); 969 970 if (addr < vma->vm_start || addr >= vma->vm_end) 971 return VM_FAULT_SIGBUS; 972 if (track_pfn_insert(vma, &pgprot, pfn)) 973 return VM_FAULT_SIGBUS; 974 insert_pfn_pmd(vma, addr, pmd, pfn, pgprot, write); 975 return VM_FAULT_NOPAGE; 976 } 977 978 static void touch_pmd(struct vm_area_struct *vma, unsigned long addr, 979 pmd_t *pmd) 980 { 981 pmd_t _pmd; 982 983 /* 984 * We should set the dirty bit only for FOLL_WRITE but for now 985 * the dirty bit in the pmd is meaningless. And if the dirty 986 * bit will become meaningful and we'll only set it with 987 * FOLL_WRITE, an atomic set_bit will be required on the pmd to 988 * set the young bit, instead of the current set_pmd_at. 989 */ 990 _pmd = pmd_mkyoung(pmd_mkdirty(*pmd)); 991 if (pmdp_set_access_flags(vma, addr & HPAGE_PMD_MASK, 992 pmd, _pmd, 1)) 993 update_mmu_cache_pmd(vma, addr, pmd); 994 } 995 996 struct page *follow_devmap_pmd(struct vm_area_struct *vma, unsigned long addr, 997 pmd_t *pmd, int flags) 998 { 999 unsigned long pfn = pmd_pfn(*pmd); 1000 struct mm_struct *mm = vma->vm_mm; 1001 struct dev_pagemap *pgmap; 1002 struct page *page; 1003 1004 assert_spin_locked(pmd_lockptr(mm, pmd)); 1005 1006 if (flags & FOLL_WRITE && !pmd_write(*pmd)) 1007 return NULL; 1008 1009 if (pmd_present(*pmd) && pmd_devmap(*pmd)) 1010 /* pass */; 1011 else 1012 return NULL; 1013 1014 if (flags & FOLL_TOUCH) 1015 touch_pmd(vma, addr, pmd); 1016 1017 /* 1018 * device mapped pages can only be returned if the 1019 * caller will manage the page reference count. 1020 */ 1021 if (!(flags & FOLL_GET)) 1022 return ERR_PTR(-EEXIST); 1023 1024 pfn += (addr & ~PMD_MASK) >> PAGE_SHIFT; 1025 pgmap = get_dev_pagemap(pfn, NULL); 1026 if (!pgmap) 1027 return ERR_PTR(-EFAULT); 1028 page = pfn_to_page(pfn); 1029 get_page(page); 1030 put_dev_pagemap(pgmap); 1031 1032 return page; 1033 } 1034 1035 int copy_huge_pmd(struct mm_struct *dst_mm, struct mm_struct *src_mm, 1036 pmd_t *dst_pmd, pmd_t *src_pmd, unsigned long addr, 1037 struct vm_area_struct *vma) 1038 { 1039 spinlock_t *dst_ptl, *src_ptl; 1040 struct page *src_page; 1041 pmd_t pmd; 1042 pgtable_t pgtable; 1043 int ret; 1044 1045 ret = -ENOMEM; 1046 pgtable = pte_alloc_one(dst_mm, addr); 1047 if (unlikely(!pgtable)) 1048 goto out; 1049 1050 dst_ptl = pmd_lock(dst_mm, dst_pmd); 1051 src_ptl = pmd_lockptr(src_mm, src_pmd); 1052 spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING); 1053 1054 ret = -EAGAIN; 1055 pmd = *src_pmd; 1056 if (unlikely(!pmd_trans_huge(pmd) && !pmd_devmap(pmd))) { 1057 pte_free(dst_mm, pgtable); 1058 goto out_unlock; 1059 } 1060 /* 1061 * When page table lock is held, the huge zero pmd should not be 1062 * under splitting since we don't split the page itself, only pmd to 1063 * a page table. 1064 */ 1065 if (is_huge_zero_pmd(pmd)) { 1066 struct page *zero_page; 1067 /* 1068 * get_huge_zero_page() will never allocate a new page here, 1069 * since we already have a zero page to copy. It just takes a 1070 * reference. 1071 */ 1072 zero_page = get_huge_zero_page(); 1073 set_huge_zero_page(pgtable, dst_mm, vma, addr, dst_pmd, 1074 zero_page); 1075 ret = 0; 1076 goto out_unlock; 1077 } 1078 1079 if (pmd_trans_huge(pmd)) { 1080 /* thp accounting separate from pmd_devmap accounting */ 1081 src_page = pmd_page(pmd); 1082 VM_BUG_ON_PAGE(!PageHead(src_page), src_page); 1083 get_page(src_page); 1084 page_dup_rmap(src_page, true); 1085 add_mm_counter(dst_mm, MM_ANONPAGES, HPAGE_PMD_NR); 1086 atomic_long_inc(&dst_mm->nr_ptes); 1087 pgtable_trans_huge_deposit(dst_mm, dst_pmd, pgtable); 1088 } 1089 1090 pmdp_set_wrprotect(src_mm, addr, src_pmd); 1091 pmd = pmd_mkold(pmd_wrprotect(pmd)); 1092 set_pmd_at(dst_mm, addr, dst_pmd, pmd); 1093 1094 ret = 0; 1095 out_unlock: 1096 spin_unlock(src_ptl); 1097 spin_unlock(dst_ptl); 1098 out: 1099 return ret; 1100 } 1101 1102 void huge_pmd_set_accessed(struct mm_struct *mm, 1103 struct vm_area_struct *vma, 1104 unsigned long address, 1105 pmd_t *pmd, pmd_t orig_pmd, 1106 int dirty) 1107 { 1108 spinlock_t *ptl; 1109 pmd_t entry; 1110 unsigned long haddr; 1111 1112 ptl = pmd_lock(mm, pmd); 1113 if (unlikely(!pmd_same(*pmd, orig_pmd))) 1114 goto unlock; 1115 1116 entry = pmd_mkyoung(orig_pmd); 1117 haddr = address & HPAGE_PMD_MASK; 1118 if (pmdp_set_access_flags(vma, haddr, pmd, entry, dirty)) 1119 update_mmu_cache_pmd(vma, address, pmd); 1120 1121 unlock: 1122 spin_unlock(ptl); 1123 } 1124 1125 static int do_huge_pmd_wp_page_fallback(struct mm_struct *mm, 1126 struct vm_area_struct *vma, 1127 unsigned long address, 1128 pmd_t *pmd, pmd_t orig_pmd, 1129 struct page *page, 1130 unsigned long haddr) 1131 { 1132 struct mem_cgroup *memcg; 1133 spinlock_t *ptl; 1134 pgtable_t pgtable; 1135 pmd_t _pmd; 1136 int ret = 0, i; 1137 struct page **pages; 1138 unsigned long mmun_start; /* For mmu_notifiers */ 1139 unsigned long mmun_end; /* For mmu_notifiers */ 1140 1141 pages = kmalloc(sizeof(struct page *) * HPAGE_PMD_NR, 1142 GFP_KERNEL); 1143 if (unlikely(!pages)) { 1144 ret |= VM_FAULT_OOM; 1145 goto out; 1146 } 1147 1148 for (i = 0; i < HPAGE_PMD_NR; i++) { 1149 pages[i] = alloc_page_vma_node(GFP_HIGHUSER_MOVABLE | 1150 __GFP_OTHER_NODE, 1151 vma, address, page_to_nid(page)); 1152 if (unlikely(!pages[i] || 1153 mem_cgroup_try_charge(pages[i], mm, GFP_KERNEL, 1154 &memcg, false))) { 1155 if (pages[i]) 1156 put_page(pages[i]); 1157 while (--i >= 0) { 1158 memcg = (void *)page_private(pages[i]); 1159 set_page_private(pages[i], 0); 1160 mem_cgroup_cancel_charge(pages[i], memcg, 1161 false); 1162 put_page(pages[i]); 1163 } 1164 kfree(pages); 1165 ret |= VM_FAULT_OOM; 1166 goto out; 1167 } 1168 set_page_private(pages[i], (unsigned long)memcg); 1169 } 1170 1171 for (i = 0; i < HPAGE_PMD_NR; i++) { 1172 copy_user_highpage(pages[i], page + i, 1173 haddr + PAGE_SIZE * i, vma); 1174 __SetPageUptodate(pages[i]); 1175 cond_resched(); 1176 } 1177 1178 mmun_start = haddr; 1179 mmun_end = haddr + HPAGE_PMD_SIZE; 1180 mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end); 1181 1182 ptl = pmd_lock(mm, pmd); 1183 if (unlikely(!pmd_same(*pmd, orig_pmd))) 1184 goto out_free_pages; 1185 VM_BUG_ON_PAGE(!PageHead(page), page); 1186 1187 pmdp_huge_clear_flush_notify(vma, haddr, pmd); 1188 /* leave pmd empty until pte is filled */ 1189 1190 pgtable = pgtable_trans_huge_withdraw(mm, pmd); 1191 pmd_populate(mm, &_pmd, pgtable); 1192 1193 for (i = 0; i < HPAGE_PMD_NR; i++, haddr += PAGE_SIZE) { 1194 pte_t *pte, entry; 1195 entry = mk_pte(pages[i], vma->vm_page_prot); 1196 entry = maybe_mkwrite(pte_mkdirty(entry), vma); 1197 memcg = (void *)page_private(pages[i]); 1198 set_page_private(pages[i], 0); 1199 page_add_new_anon_rmap(pages[i], vma, haddr, false); 1200 mem_cgroup_commit_charge(pages[i], memcg, false, false); 1201 lru_cache_add_active_or_unevictable(pages[i], vma); 1202 pte = pte_offset_map(&_pmd, haddr); 1203 VM_BUG_ON(!pte_none(*pte)); 1204 set_pte_at(mm, haddr, pte, entry); 1205 pte_unmap(pte); 1206 } 1207 kfree(pages); 1208 1209 smp_wmb(); /* make pte visible before pmd */ 1210 pmd_populate(mm, pmd, pgtable); 1211 page_remove_rmap(page, true); 1212 spin_unlock(ptl); 1213 1214 mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end); 1215 1216 ret |= VM_FAULT_WRITE; 1217 put_page(page); 1218 1219 out: 1220 return ret; 1221 1222 out_free_pages: 1223 spin_unlock(ptl); 1224 mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end); 1225 for (i = 0; i < HPAGE_PMD_NR; i++) { 1226 memcg = (void *)page_private(pages[i]); 1227 set_page_private(pages[i], 0); 1228 mem_cgroup_cancel_charge(pages[i], memcg, false); 1229 put_page(pages[i]); 1230 } 1231 kfree(pages); 1232 goto out; 1233 } 1234 1235 int do_huge_pmd_wp_page(struct mm_struct *mm, struct vm_area_struct *vma, 1236 unsigned long address, pmd_t *pmd, pmd_t orig_pmd) 1237 { 1238 spinlock_t *ptl; 1239 int ret = 0; 1240 struct page *page = NULL, *new_page; 1241 struct mem_cgroup *memcg; 1242 unsigned long haddr; 1243 unsigned long mmun_start; /* For mmu_notifiers */ 1244 unsigned long mmun_end; /* For mmu_notifiers */ 1245 gfp_t huge_gfp; /* for allocation and charge */ 1246 1247 ptl = pmd_lockptr(mm, pmd); 1248 VM_BUG_ON_VMA(!vma->anon_vma, vma); 1249 haddr = address & HPAGE_PMD_MASK; 1250 if (is_huge_zero_pmd(orig_pmd)) 1251 goto alloc; 1252 spin_lock(ptl); 1253 if (unlikely(!pmd_same(*pmd, orig_pmd))) 1254 goto out_unlock; 1255 1256 page = pmd_page(orig_pmd); 1257 VM_BUG_ON_PAGE(!PageCompound(page) || !PageHead(page), page); 1258 /* 1259 * We can only reuse the page if nobody else maps the huge page or it's 1260 * part. We can do it by checking page_mapcount() on each sub-page, but 1261 * it's expensive. 1262 * The cheaper way is to check page_count() to be equal 1: every 1263 * mapcount takes page reference reference, so this way we can 1264 * guarantee, that the PMD is the only mapping. 1265 * This can give false negative if somebody pinned the page, but that's 1266 * fine. 1267 */ 1268 if (page_mapcount(page) == 1 && page_count(page) == 1) { 1269 pmd_t entry; 1270 entry = pmd_mkyoung(orig_pmd); 1271 entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma); 1272 if (pmdp_set_access_flags(vma, haddr, pmd, entry, 1)) 1273 update_mmu_cache_pmd(vma, address, pmd); 1274 ret |= VM_FAULT_WRITE; 1275 goto out_unlock; 1276 } 1277 get_page(page); 1278 spin_unlock(ptl); 1279 alloc: 1280 if (transparent_hugepage_enabled(vma) && 1281 !transparent_hugepage_debug_cow()) { 1282 huge_gfp = alloc_hugepage_gfpmask(transparent_hugepage_defrag(vma), 0); 1283 new_page = alloc_hugepage_vma(huge_gfp, vma, haddr, HPAGE_PMD_ORDER); 1284 } else 1285 new_page = NULL; 1286 1287 if (likely(new_page)) { 1288 prep_transhuge_page(new_page); 1289 } else { 1290 if (!page) { 1291 split_huge_pmd(vma, pmd, address); 1292 ret |= VM_FAULT_FALLBACK; 1293 } else { 1294 ret = do_huge_pmd_wp_page_fallback(mm, vma, address, 1295 pmd, orig_pmd, page, haddr); 1296 if (ret & VM_FAULT_OOM) { 1297 split_huge_pmd(vma, pmd, address); 1298 ret |= VM_FAULT_FALLBACK; 1299 } 1300 put_page(page); 1301 } 1302 count_vm_event(THP_FAULT_FALLBACK); 1303 goto out; 1304 } 1305 1306 if (unlikely(mem_cgroup_try_charge(new_page, mm, huge_gfp, &memcg, 1307 true))) { 1308 put_page(new_page); 1309 if (page) { 1310 split_huge_pmd(vma, pmd, address); 1311 put_page(page); 1312 } else 1313 split_huge_pmd(vma, pmd, address); 1314 ret |= VM_FAULT_FALLBACK; 1315 count_vm_event(THP_FAULT_FALLBACK); 1316 goto out; 1317 } 1318 1319 count_vm_event(THP_FAULT_ALLOC); 1320 1321 if (!page) 1322 clear_huge_page(new_page, haddr, HPAGE_PMD_NR); 1323 else 1324 copy_user_huge_page(new_page, page, haddr, vma, HPAGE_PMD_NR); 1325 __SetPageUptodate(new_page); 1326 1327 mmun_start = haddr; 1328 mmun_end = haddr + HPAGE_PMD_SIZE; 1329 mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end); 1330 1331 spin_lock(ptl); 1332 if (page) 1333 put_page(page); 1334 if (unlikely(!pmd_same(*pmd, orig_pmd))) { 1335 spin_unlock(ptl); 1336 mem_cgroup_cancel_charge(new_page, memcg, true); 1337 put_page(new_page); 1338 goto out_mn; 1339 } else { 1340 pmd_t entry; 1341 entry = mk_huge_pmd(new_page, vma->vm_page_prot); 1342 entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma); 1343 pmdp_huge_clear_flush_notify(vma, haddr, pmd); 1344 page_add_new_anon_rmap(new_page, vma, haddr, true); 1345 mem_cgroup_commit_charge(new_page, memcg, false, true); 1346 lru_cache_add_active_or_unevictable(new_page, vma); 1347 set_pmd_at(mm, haddr, pmd, entry); 1348 update_mmu_cache_pmd(vma, address, pmd); 1349 if (!page) { 1350 add_mm_counter(mm, MM_ANONPAGES, HPAGE_PMD_NR); 1351 put_huge_zero_page(); 1352 } else { 1353 VM_BUG_ON_PAGE(!PageHead(page), page); 1354 page_remove_rmap(page, true); 1355 put_page(page); 1356 } 1357 ret |= VM_FAULT_WRITE; 1358 } 1359 spin_unlock(ptl); 1360 out_mn: 1361 mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end); 1362 out: 1363 return ret; 1364 out_unlock: 1365 spin_unlock(ptl); 1366 return ret; 1367 } 1368 1369 struct page *follow_trans_huge_pmd(struct vm_area_struct *vma, 1370 unsigned long addr, 1371 pmd_t *pmd, 1372 unsigned int flags) 1373 { 1374 struct mm_struct *mm = vma->vm_mm; 1375 struct page *page = NULL; 1376 1377 assert_spin_locked(pmd_lockptr(mm, pmd)); 1378 1379 if (flags & FOLL_WRITE && !pmd_write(*pmd)) 1380 goto out; 1381 1382 /* Avoid dumping huge zero page */ 1383 if ((flags & FOLL_DUMP) && is_huge_zero_pmd(*pmd)) 1384 return ERR_PTR(-EFAULT); 1385 1386 /* Full NUMA hinting faults to serialise migration in fault paths */ 1387 if ((flags & FOLL_NUMA) && pmd_protnone(*pmd)) 1388 goto out; 1389 1390 page = pmd_page(*pmd); 1391 VM_BUG_ON_PAGE(!PageHead(page), page); 1392 if (flags & FOLL_TOUCH) 1393 touch_pmd(vma, addr, pmd); 1394 if ((flags & FOLL_MLOCK) && (vma->vm_flags & VM_LOCKED)) { 1395 /* 1396 * We don't mlock() pte-mapped THPs. This way we can avoid 1397 * leaking mlocked pages into non-VM_LOCKED VMAs. 1398 * 1399 * In most cases the pmd is the only mapping of the page as we 1400 * break COW for the mlock() -- see gup_flags |= FOLL_WRITE for 1401 * writable private mappings in populate_vma_page_range(). 1402 * 1403 * The only scenario when we have the page shared here is if we 1404 * mlocking read-only mapping shared over fork(). We skip 1405 * mlocking such pages. 1406 */ 1407 if (compound_mapcount(page) == 1 && !PageDoubleMap(page) && 1408 page->mapping && trylock_page(page)) { 1409 lru_add_drain(); 1410 if (page->mapping) 1411 mlock_vma_page(page); 1412 unlock_page(page); 1413 } 1414 } 1415 page += (addr & ~HPAGE_PMD_MASK) >> PAGE_SHIFT; 1416 VM_BUG_ON_PAGE(!PageCompound(page), page); 1417 if (flags & FOLL_GET) 1418 get_page(page); 1419 1420 out: 1421 return page; 1422 } 1423 1424 /* NUMA hinting page fault entry point for trans huge pmds */ 1425 int do_huge_pmd_numa_page(struct mm_struct *mm, struct vm_area_struct *vma, 1426 unsigned long addr, pmd_t pmd, pmd_t *pmdp) 1427 { 1428 spinlock_t *ptl; 1429 struct anon_vma *anon_vma = NULL; 1430 struct page *page; 1431 unsigned long haddr = addr & HPAGE_PMD_MASK; 1432 int page_nid = -1, this_nid = numa_node_id(); 1433 int target_nid, last_cpupid = -1; 1434 bool page_locked; 1435 bool migrated = false; 1436 bool was_writable; 1437 int flags = 0; 1438 1439 /* A PROT_NONE fault should not end up here */ 1440 BUG_ON(!(vma->vm_flags & (VM_READ | VM_EXEC | VM_WRITE))); 1441 1442 ptl = pmd_lock(mm, pmdp); 1443 if (unlikely(!pmd_same(pmd, *pmdp))) 1444 goto out_unlock; 1445 1446 /* 1447 * If there are potential migrations, wait for completion and retry 1448 * without disrupting NUMA hinting information. Do not relock and 1449 * check_same as the page may no longer be mapped. 1450 */ 1451 if (unlikely(pmd_trans_migrating(*pmdp))) { 1452 page = pmd_page(*pmdp); 1453 spin_unlock(ptl); 1454 wait_on_page_locked(page); 1455 goto out; 1456 } 1457 1458 page = pmd_page(pmd); 1459 BUG_ON(is_huge_zero_page(page)); 1460 page_nid = page_to_nid(page); 1461 last_cpupid = page_cpupid_last(page); 1462 count_vm_numa_event(NUMA_HINT_FAULTS); 1463 if (page_nid == this_nid) { 1464 count_vm_numa_event(NUMA_HINT_FAULTS_LOCAL); 1465 flags |= TNF_FAULT_LOCAL; 1466 } 1467 1468 /* See similar comment in do_numa_page for explanation */ 1469 if (!(vma->vm_flags & VM_WRITE)) 1470 flags |= TNF_NO_GROUP; 1471 1472 /* 1473 * Acquire the page lock to serialise THP migrations but avoid dropping 1474 * page_table_lock if at all possible 1475 */ 1476 page_locked = trylock_page(page); 1477 target_nid = mpol_misplaced(page, vma, haddr); 1478 if (target_nid == -1) { 1479 /* If the page was locked, there are no parallel migrations */ 1480 if (page_locked) 1481 goto clear_pmdnuma; 1482 } 1483 1484 /* Migration could have started since the pmd_trans_migrating check */ 1485 if (!page_locked) { 1486 spin_unlock(ptl); 1487 wait_on_page_locked(page); 1488 page_nid = -1; 1489 goto out; 1490 } 1491 1492 /* 1493 * Page is misplaced. Page lock serialises migrations. Acquire anon_vma 1494 * to serialises splits 1495 */ 1496 get_page(page); 1497 spin_unlock(ptl); 1498 anon_vma = page_lock_anon_vma_read(page); 1499 1500 /* Confirm the PMD did not change while page_table_lock was released */ 1501 spin_lock(ptl); 1502 if (unlikely(!pmd_same(pmd, *pmdp))) { 1503 unlock_page(page); 1504 put_page(page); 1505 page_nid = -1; 1506 goto out_unlock; 1507 } 1508 1509 /* Bail if we fail to protect against THP splits for any reason */ 1510 if (unlikely(!anon_vma)) { 1511 put_page(page); 1512 page_nid = -1; 1513 goto clear_pmdnuma; 1514 } 1515 1516 /* 1517 * Migrate the THP to the requested node, returns with page unlocked 1518 * and access rights restored. 1519 */ 1520 spin_unlock(ptl); 1521 migrated = migrate_misplaced_transhuge_page(mm, vma, 1522 pmdp, pmd, addr, page, target_nid); 1523 if (migrated) { 1524 flags |= TNF_MIGRATED; 1525 page_nid = target_nid; 1526 } else 1527 flags |= TNF_MIGRATE_FAIL; 1528 1529 goto out; 1530 clear_pmdnuma: 1531 BUG_ON(!PageLocked(page)); 1532 was_writable = pmd_write(pmd); 1533 pmd = pmd_modify(pmd, vma->vm_page_prot); 1534 pmd = pmd_mkyoung(pmd); 1535 if (was_writable) 1536 pmd = pmd_mkwrite(pmd); 1537 set_pmd_at(mm, haddr, pmdp, pmd); 1538 update_mmu_cache_pmd(vma, addr, pmdp); 1539 unlock_page(page); 1540 out_unlock: 1541 spin_unlock(ptl); 1542 1543 out: 1544 if (anon_vma) 1545 page_unlock_anon_vma_read(anon_vma); 1546 1547 if (page_nid != -1) 1548 task_numa_fault(last_cpupid, page_nid, HPAGE_PMD_NR, flags); 1549 1550 return 0; 1551 } 1552 1553 int madvise_free_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, 1554 pmd_t *pmd, unsigned long addr, unsigned long next) 1555 1556 { 1557 spinlock_t *ptl; 1558 pmd_t orig_pmd; 1559 struct page *page; 1560 struct mm_struct *mm = tlb->mm; 1561 int ret = 0; 1562 1563 ptl = pmd_trans_huge_lock(pmd, vma); 1564 if (!ptl) 1565 goto out_unlocked; 1566 1567 orig_pmd = *pmd; 1568 if (is_huge_zero_pmd(orig_pmd)) { 1569 ret = 1; 1570 goto out; 1571 } 1572 1573 page = pmd_page(orig_pmd); 1574 /* 1575 * If other processes are mapping this page, we couldn't discard 1576 * the page unless they all do MADV_FREE so let's skip the page. 1577 */ 1578 if (page_mapcount(page) != 1) 1579 goto out; 1580 1581 if (!trylock_page(page)) 1582 goto out; 1583 1584 /* 1585 * If user want to discard part-pages of THP, split it so MADV_FREE 1586 * will deactivate only them. 1587 */ 1588 if (next - addr != HPAGE_PMD_SIZE) { 1589 get_page(page); 1590 spin_unlock(ptl); 1591 if (split_huge_page(page)) { 1592 put_page(page); 1593 unlock_page(page); 1594 goto out_unlocked; 1595 } 1596 put_page(page); 1597 unlock_page(page); 1598 ret = 1; 1599 goto out_unlocked; 1600 } 1601 1602 if (PageDirty(page)) 1603 ClearPageDirty(page); 1604 unlock_page(page); 1605 1606 if (PageActive(page)) 1607 deactivate_page(page); 1608 1609 if (pmd_young(orig_pmd) || pmd_dirty(orig_pmd)) { 1610 orig_pmd = pmdp_huge_get_and_clear_full(tlb->mm, addr, pmd, 1611 tlb->fullmm); 1612 orig_pmd = pmd_mkold(orig_pmd); 1613 orig_pmd = pmd_mkclean(orig_pmd); 1614 1615 set_pmd_at(mm, addr, pmd, orig_pmd); 1616 tlb_remove_pmd_tlb_entry(tlb, pmd, addr); 1617 } 1618 ret = 1; 1619 out: 1620 spin_unlock(ptl); 1621 out_unlocked: 1622 return ret; 1623 } 1624 1625 int zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, 1626 pmd_t *pmd, unsigned long addr) 1627 { 1628 pmd_t orig_pmd; 1629 spinlock_t *ptl; 1630 1631 ptl = __pmd_trans_huge_lock(pmd, vma); 1632 if (!ptl) 1633 return 0; 1634 /* 1635 * For architectures like ppc64 we look at deposited pgtable 1636 * when calling pmdp_huge_get_and_clear. So do the 1637 * pgtable_trans_huge_withdraw after finishing pmdp related 1638 * operations. 1639 */ 1640 orig_pmd = pmdp_huge_get_and_clear_full(tlb->mm, addr, pmd, 1641 tlb->fullmm); 1642 tlb_remove_pmd_tlb_entry(tlb, pmd, addr); 1643 if (vma_is_dax(vma)) { 1644 spin_unlock(ptl); 1645 if (is_huge_zero_pmd(orig_pmd)) 1646 put_huge_zero_page(); 1647 } else if (is_huge_zero_pmd(orig_pmd)) { 1648 pte_free(tlb->mm, pgtable_trans_huge_withdraw(tlb->mm, pmd)); 1649 atomic_long_dec(&tlb->mm->nr_ptes); 1650 spin_unlock(ptl); 1651 put_huge_zero_page(); 1652 } else { 1653 struct page *page = pmd_page(orig_pmd); 1654 page_remove_rmap(page, true); 1655 VM_BUG_ON_PAGE(page_mapcount(page) < 0, page); 1656 add_mm_counter(tlb->mm, MM_ANONPAGES, -HPAGE_PMD_NR); 1657 VM_BUG_ON_PAGE(!PageHead(page), page); 1658 pte_free(tlb->mm, pgtable_trans_huge_withdraw(tlb->mm, pmd)); 1659 atomic_long_dec(&tlb->mm->nr_ptes); 1660 spin_unlock(ptl); 1661 tlb_remove_page(tlb, page); 1662 } 1663 return 1; 1664 } 1665 1666 bool move_huge_pmd(struct vm_area_struct *vma, struct vm_area_struct *new_vma, 1667 unsigned long old_addr, 1668 unsigned long new_addr, unsigned long old_end, 1669 pmd_t *old_pmd, pmd_t *new_pmd) 1670 { 1671 spinlock_t *old_ptl, *new_ptl; 1672 pmd_t pmd; 1673 1674 struct mm_struct *mm = vma->vm_mm; 1675 1676 if ((old_addr & ~HPAGE_PMD_MASK) || 1677 (new_addr & ~HPAGE_PMD_MASK) || 1678 old_end - old_addr < HPAGE_PMD_SIZE || 1679 (new_vma->vm_flags & VM_NOHUGEPAGE)) 1680 return false; 1681 1682 /* 1683 * The destination pmd shouldn't be established, free_pgtables() 1684 * should have release it. 1685 */ 1686 if (WARN_ON(!pmd_none(*new_pmd))) { 1687 VM_BUG_ON(pmd_trans_huge(*new_pmd)); 1688 return false; 1689 } 1690 1691 /* 1692 * We don't have to worry about the ordering of src and dst 1693 * ptlocks because exclusive mmap_sem prevents deadlock. 1694 */ 1695 old_ptl = __pmd_trans_huge_lock(old_pmd, vma); 1696 if (old_ptl) { 1697 new_ptl = pmd_lockptr(mm, new_pmd); 1698 if (new_ptl != old_ptl) 1699 spin_lock_nested(new_ptl, SINGLE_DEPTH_NESTING); 1700 pmd = pmdp_huge_get_and_clear(mm, old_addr, old_pmd); 1701 VM_BUG_ON(!pmd_none(*new_pmd)); 1702 1703 if (pmd_move_must_withdraw(new_ptl, old_ptl)) { 1704 pgtable_t pgtable; 1705 pgtable = pgtable_trans_huge_withdraw(mm, old_pmd); 1706 pgtable_trans_huge_deposit(mm, new_pmd, pgtable); 1707 } 1708 set_pmd_at(mm, new_addr, new_pmd, pmd_mksoft_dirty(pmd)); 1709 if (new_ptl != old_ptl) 1710 spin_unlock(new_ptl); 1711 spin_unlock(old_ptl); 1712 return true; 1713 } 1714 return false; 1715 } 1716 1717 /* 1718 * Returns 1719 * - 0 if PMD could not be locked 1720 * - 1 if PMD was locked but protections unchange and TLB flush unnecessary 1721 * - HPAGE_PMD_NR is protections changed and TLB flush necessary 1722 */ 1723 int change_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd, 1724 unsigned long addr, pgprot_t newprot, int prot_numa) 1725 { 1726 struct mm_struct *mm = vma->vm_mm; 1727 spinlock_t *ptl; 1728 int ret = 0; 1729 1730 ptl = __pmd_trans_huge_lock(pmd, vma); 1731 if (ptl) { 1732 pmd_t entry; 1733 bool preserve_write = prot_numa && pmd_write(*pmd); 1734 ret = 1; 1735 1736 /* 1737 * Avoid trapping faults against the zero page. The read-only 1738 * data is likely to be read-cached on the local CPU and 1739 * local/remote hits to the zero page are not interesting. 1740 */ 1741 if (prot_numa && is_huge_zero_pmd(*pmd)) { 1742 spin_unlock(ptl); 1743 return ret; 1744 } 1745 1746 if (!prot_numa || !pmd_protnone(*pmd)) { 1747 entry = pmdp_huge_get_and_clear_notify(mm, addr, pmd); 1748 entry = pmd_modify(entry, newprot); 1749 if (preserve_write) 1750 entry = pmd_mkwrite(entry); 1751 ret = HPAGE_PMD_NR; 1752 set_pmd_at(mm, addr, pmd, entry); 1753 BUG_ON(!preserve_write && pmd_write(entry)); 1754 } 1755 spin_unlock(ptl); 1756 } 1757 1758 return ret; 1759 } 1760 1761 /* 1762 * Returns true if a given pmd maps a thp, false otherwise. 1763 * 1764 * Note that if it returns true, this routine returns without unlocking page 1765 * table lock. So callers must unlock it. 1766 */ 1767 spinlock_t *__pmd_trans_huge_lock(pmd_t *pmd, struct vm_area_struct *vma) 1768 { 1769 spinlock_t *ptl; 1770 ptl = pmd_lock(vma->vm_mm, pmd); 1771 if (likely(pmd_trans_huge(*pmd) || pmd_devmap(*pmd))) 1772 return ptl; 1773 spin_unlock(ptl); 1774 return NULL; 1775 } 1776 1777 #define VM_NO_THP (VM_SPECIAL | VM_HUGETLB | VM_SHARED | VM_MAYSHARE) 1778 1779 int hugepage_madvise(struct vm_area_struct *vma, 1780 unsigned long *vm_flags, int advice) 1781 { 1782 switch (advice) { 1783 case MADV_HUGEPAGE: 1784 #ifdef CONFIG_S390 1785 /* 1786 * qemu blindly sets MADV_HUGEPAGE on all allocations, but s390 1787 * can't handle this properly after s390_enable_sie, so we simply 1788 * ignore the madvise to prevent qemu from causing a SIGSEGV. 1789 */ 1790 if (mm_has_pgste(vma->vm_mm)) 1791 return 0; 1792 #endif 1793 /* 1794 * Be somewhat over-protective like KSM for now! 1795 */ 1796 if (*vm_flags & VM_NO_THP) 1797 return -EINVAL; 1798 *vm_flags &= ~VM_NOHUGEPAGE; 1799 *vm_flags |= VM_HUGEPAGE; 1800 /* 1801 * If the vma become good for khugepaged to scan, 1802 * register it here without waiting a page fault that 1803 * may not happen any time soon. 1804 */ 1805 if (unlikely(khugepaged_enter_vma_merge(vma, *vm_flags))) 1806 return -ENOMEM; 1807 break; 1808 case MADV_NOHUGEPAGE: 1809 /* 1810 * Be somewhat over-protective like KSM for now! 1811 */ 1812 if (*vm_flags & VM_NO_THP) 1813 return -EINVAL; 1814 *vm_flags &= ~VM_HUGEPAGE; 1815 *vm_flags |= VM_NOHUGEPAGE; 1816 /* 1817 * Setting VM_NOHUGEPAGE will prevent khugepaged from scanning 1818 * this vma even if we leave the mm registered in khugepaged if 1819 * it got registered before VM_NOHUGEPAGE was set. 1820 */ 1821 break; 1822 } 1823 1824 return 0; 1825 } 1826 1827 static int __init khugepaged_slab_init(void) 1828 { 1829 mm_slot_cache = kmem_cache_create("khugepaged_mm_slot", 1830 sizeof(struct mm_slot), 1831 __alignof__(struct mm_slot), 0, NULL); 1832 if (!mm_slot_cache) 1833 return -ENOMEM; 1834 1835 return 0; 1836 } 1837 1838 static void __init khugepaged_slab_exit(void) 1839 { 1840 kmem_cache_destroy(mm_slot_cache); 1841 } 1842 1843 static inline struct mm_slot *alloc_mm_slot(void) 1844 { 1845 if (!mm_slot_cache) /* initialization failed */ 1846 return NULL; 1847 return kmem_cache_zalloc(mm_slot_cache, GFP_KERNEL); 1848 } 1849 1850 static inline void free_mm_slot(struct mm_slot *mm_slot) 1851 { 1852 kmem_cache_free(mm_slot_cache, mm_slot); 1853 } 1854 1855 static struct mm_slot *get_mm_slot(struct mm_struct *mm) 1856 { 1857 struct mm_slot *mm_slot; 1858 1859 hash_for_each_possible(mm_slots_hash, mm_slot, hash, (unsigned long)mm) 1860 if (mm == mm_slot->mm) 1861 return mm_slot; 1862 1863 return NULL; 1864 } 1865 1866 static void insert_to_mm_slots_hash(struct mm_struct *mm, 1867 struct mm_slot *mm_slot) 1868 { 1869 mm_slot->mm = mm; 1870 hash_add(mm_slots_hash, &mm_slot->hash, (long)mm); 1871 } 1872 1873 static inline int khugepaged_test_exit(struct mm_struct *mm) 1874 { 1875 return atomic_read(&mm->mm_users) == 0; 1876 } 1877 1878 int __khugepaged_enter(struct mm_struct *mm) 1879 { 1880 struct mm_slot *mm_slot; 1881 int wakeup; 1882 1883 mm_slot = alloc_mm_slot(); 1884 if (!mm_slot) 1885 return -ENOMEM; 1886 1887 /* __khugepaged_exit() must not run from under us */ 1888 VM_BUG_ON_MM(khugepaged_test_exit(mm), mm); 1889 if (unlikely(test_and_set_bit(MMF_VM_HUGEPAGE, &mm->flags))) { 1890 free_mm_slot(mm_slot); 1891 return 0; 1892 } 1893 1894 spin_lock(&khugepaged_mm_lock); 1895 insert_to_mm_slots_hash(mm, mm_slot); 1896 /* 1897 * Insert just behind the scanning cursor, to let the area settle 1898 * down a little. 1899 */ 1900 wakeup = list_empty(&khugepaged_scan.mm_head); 1901 list_add_tail(&mm_slot->mm_node, &khugepaged_scan.mm_head); 1902 spin_unlock(&khugepaged_mm_lock); 1903 1904 atomic_inc(&mm->mm_count); 1905 if (wakeup) 1906 wake_up_interruptible(&khugepaged_wait); 1907 1908 return 0; 1909 } 1910 1911 int khugepaged_enter_vma_merge(struct vm_area_struct *vma, 1912 unsigned long vm_flags) 1913 { 1914 unsigned long hstart, hend; 1915 if (!vma->anon_vma) 1916 /* 1917 * Not yet faulted in so we will register later in the 1918 * page fault if needed. 1919 */ 1920 return 0; 1921 if (vma->vm_ops) 1922 /* khugepaged not yet working on file or special mappings */ 1923 return 0; 1924 VM_BUG_ON_VMA(vm_flags & VM_NO_THP, vma); 1925 hstart = (vma->vm_start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK; 1926 hend = vma->vm_end & HPAGE_PMD_MASK; 1927 if (hstart < hend) 1928 return khugepaged_enter(vma, vm_flags); 1929 return 0; 1930 } 1931 1932 void __khugepaged_exit(struct mm_struct *mm) 1933 { 1934 struct mm_slot *mm_slot; 1935 int free = 0; 1936 1937 spin_lock(&khugepaged_mm_lock); 1938 mm_slot = get_mm_slot(mm); 1939 if (mm_slot && khugepaged_scan.mm_slot != mm_slot) { 1940 hash_del(&mm_slot->hash); 1941 list_del(&mm_slot->mm_node); 1942 free = 1; 1943 } 1944 spin_unlock(&khugepaged_mm_lock); 1945 1946 if (free) { 1947 clear_bit(MMF_VM_HUGEPAGE, &mm->flags); 1948 free_mm_slot(mm_slot); 1949 mmdrop(mm); 1950 } else if (mm_slot) { 1951 /* 1952 * This is required to serialize against 1953 * khugepaged_test_exit() (which is guaranteed to run 1954 * under mmap sem read mode). Stop here (after we 1955 * return all pagetables will be destroyed) until 1956 * khugepaged has finished working on the pagetables 1957 * under the mmap_sem. 1958 */ 1959 down_write(&mm->mmap_sem); 1960 up_write(&mm->mmap_sem); 1961 } 1962 } 1963 1964 static void release_pte_page(struct page *page) 1965 { 1966 /* 0 stands for page_is_file_cache(page) == false */ 1967 dec_zone_page_state(page, NR_ISOLATED_ANON + 0); 1968 unlock_page(page); 1969 putback_lru_page(page); 1970 } 1971 1972 static void release_pte_pages(pte_t *pte, pte_t *_pte) 1973 { 1974 while (--_pte >= pte) { 1975 pte_t pteval = *_pte; 1976 if (!pte_none(pteval) && !is_zero_pfn(pte_pfn(pteval))) 1977 release_pte_page(pte_page(pteval)); 1978 } 1979 } 1980 1981 static int __collapse_huge_page_isolate(struct vm_area_struct *vma, 1982 unsigned long address, 1983 pte_t *pte) 1984 { 1985 struct page *page = NULL; 1986 pte_t *_pte; 1987 int none_or_zero = 0, result = 0; 1988 bool referenced = false, writable = false; 1989 1990 for (_pte = pte; _pte < pte+HPAGE_PMD_NR; 1991 _pte++, address += PAGE_SIZE) { 1992 pte_t pteval = *_pte; 1993 if (pte_none(pteval) || (pte_present(pteval) && 1994 is_zero_pfn(pte_pfn(pteval)))) { 1995 if (!userfaultfd_armed(vma) && 1996 ++none_or_zero <= khugepaged_max_ptes_none) { 1997 continue; 1998 } else { 1999 result = SCAN_EXCEED_NONE_PTE; 2000 goto out; 2001 } 2002 } 2003 if (!pte_present(pteval)) { 2004 result = SCAN_PTE_NON_PRESENT; 2005 goto out; 2006 } 2007 page = vm_normal_page(vma, address, pteval); 2008 if (unlikely(!page)) { 2009 result = SCAN_PAGE_NULL; 2010 goto out; 2011 } 2012 2013 VM_BUG_ON_PAGE(PageCompound(page), page); 2014 VM_BUG_ON_PAGE(!PageAnon(page), page); 2015 VM_BUG_ON_PAGE(!PageSwapBacked(page), page); 2016 2017 /* 2018 * We can do it before isolate_lru_page because the 2019 * page can't be freed from under us. NOTE: PG_lock 2020 * is needed to serialize against split_huge_page 2021 * when invoked from the VM. 2022 */ 2023 if (!trylock_page(page)) { 2024 result = SCAN_PAGE_LOCK; 2025 goto out; 2026 } 2027 2028 /* 2029 * cannot use mapcount: can't collapse if there's a gup pin. 2030 * The page must only be referenced by the scanned process 2031 * and page swap cache. 2032 */ 2033 if (page_count(page) != 1 + !!PageSwapCache(page)) { 2034 unlock_page(page); 2035 result = SCAN_PAGE_COUNT; 2036 goto out; 2037 } 2038 if (pte_write(pteval)) { 2039 writable = true; 2040 } else { 2041 if (PageSwapCache(page) && !reuse_swap_page(page)) { 2042 unlock_page(page); 2043 result = SCAN_SWAP_CACHE_PAGE; 2044 goto out; 2045 } 2046 /* 2047 * Page is not in the swap cache. It can be collapsed 2048 * into a THP. 2049 */ 2050 } 2051 2052 /* 2053 * Isolate the page to avoid collapsing an hugepage 2054 * currently in use by the VM. 2055 */ 2056 if (isolate_lru_page(page)) { 2057 unlock_page(page); 2058 result = SCAN_DEL_PAGE_LRU; 2059 goto out; 2060 } 2061 /* 0 stands for page_is_file_cache(page) == false */ 2062 inc_zone_page_state(page, NR_ISOLATED_ANON + 0); 2063 VM_BUG_ON_PAGE(!PageLocked(page), page); 2064 VM_BUG_ON_PAGE(PageLRU(page), page); 2065 2066 /* If there is no mapped pte young don't collapse the page */ 2067 if (pte_young(pteval) || 2068 page_is_young(page) || PageReferenced(page) || 2069 mmu_notifier_test_young(vma->vm_mm, address)) 2070 referenced = true; 2071 } 2072 if (likely(writable)) { 2073 if (likely(referenced)) { 2074 result = SCAN_SUCCEED; 2075 trace_mm_collapse_huge_page_isolate(page, none_or_zero, 2076 referenced, writable, result); 2077 return 1; 2078 } 2079 } else { 2080 result = SCAN_PAGE_RO; 2081 } 2082 2083 out: 2084 release_pte_pages(pte, _pte); 2085 trace_mm_collapse_huge_page_isolate(page, none_or_zero, 2086 referenced, writable, result); 2087 return 0; 2088 } 2089 2090 static void __collapse_huge_page_copy(pte_t *pte, struct page *page, 2091 struct vm_area_struct *vma, 2092 unsigned long address, 2093 spinlock_t *ptl) 2094 { 2095 pte_t *_pte; 2096 for (_pte = pte; _pte < pte+HPAGE_PMD_NR; _pte++) { 2097 pte_t pteval = *_pte; 2098 struct page *src_page; 2099 2100 if (pte_none(pteval) || is_zero_pfn(pte_pfn(pteval))) { 2101 clear_user_highpage(page, address); 2102 add_mm_counter(vma->vm_mm, MM_ANONPAGES, 1); 2103 if (is_zero_pfn(pte_pfn(pteval))) { 2104 /* 2105 * ptl mostly unnecessary. 2106 */ 2107 spin_lock(ptl); 2108 /* 2109 * paravirt calls inside pte_clear here are 2110 * superfluous. 2111 */ 2112 pte_clear(vma->vm_mm, address, _pte); 2113 spin_unlock(ptl); 2114 } 2115 } else { 2116 src_page = pte_page(pteval); 2117 copy_user_highpage(page, src_page, address, vma); 2118 VM_BUG_ON_PAGE(page_mapcount(src_page) != 1, src_page); 2119 release_pte_page(src_page); 2120 /* 2121 * ptl mostly unnecessary, but preempt has to 2122 * be disabled to update the per-cpu stats 2123 * inside page_remove_rmap(). 2124 */ 2125 spin_lock(ptl); 2126 /* 2127 * paravirt calls inside pte_clear here are 2128 * superfluous. 2129 */ 2130 pte_clear(vma->vm_mm, address, _pte); 2131 page_remove_rmap(src_page, false); 2132 spin_unlock(ptl); 2133 free_page_and_swap_cache(src_page); 2134 } 2135 2136 address += PAGE_SIZE; 2137 page++; 2138 } 2139 } 2140 2141 static void khugepaged_alloc_sleep(void) 2142 { 2143 DEFINE_WAIT(wait); 2144 2145 add_wait_queue(&khugepaged_wait, &wait); 2146 freezable_schedule_timeout_interruptible( 2147 msecs_to_jiffies(khugepaged_alloc_sleep_millisecs)); 2148 remove_wait_queue(&khugepaged_wait, &wait); 2149 } 2150 2151 static int khugepaged_node_load[MAX_NUMNODES]; 2152 2153 static bool khugepaged_scan_abort(int nid) 2154 { 2155 int i; 2156 2157 /* 2158 * If zone_reclaim_mode is disabled, then no extra effort is made to 2159 * allocate memory locally. 2160 */ 2161 if (!zone_reclaim_mode) 2162 return false; 2163 2164 /* If there is a count for this node already, it must be acceptable */ 2165 if (khugepaged_node_load[nid]) 2166 return false; 2167 2168 for (i = 0; i < MAX_NUMNODES; i++) { 2169 if (!khugepaged_node_load[i]) 2170 continue; 2171 if (node_distance(nid, i) > RECLAIM_DISTANCE) 2172 return true; 2173 } 2174 return false; 2175 } 2176 2177 #ifdef CONFIG_NUMA 2178 static int khugepaged_find_target_node(void) 2179 { 2180 static int last_khugepaged_target_node = NUMA_NO_NODE; 2181 int nid, target_node = 0, max_value = 0; 2182 2183 /* find first node with max normal pages hit */ 2184 for (nid = 0; nid < MAX_NUMNODES; nid++) 2185 if (khugepaged_node_load[nid] > max_value) { 2186 max_value = khugepaged_node_load[nid]; 2187 target_node = nid; 2188 } 2189 2190 /* do some balance if several nodes have the same hit record */ 2191 if (target_node <= last_khugepaged_target_node) 2192 for (nid = last_khugepaged_target_node + 1; nid < MAX_NUMNODES; 2193 nid++) 2194 if (max_value == khugepaged_node_load[nid]) { 2195 target_node = nid; 2196 break; 2197 } 2198 2199 last_khugepaged_target_node = target_node; 2200 return target_node; 2201 } 2202 2203 static bool khugepaged_prealloc_page(struct page **hpage, bool *wait) 2204 { 2205 if (IS_ERR(*hpage)) { 2206 if (!*wait) 2207 return false; 2208 2209 *wait = false; 2210 *hpage = NULL; 2211 khugepaged_alloc_sleep(); 2212 } else if (*hpage) { 2213 put_page(*hpage); 2214 *hpage = NULL; 2215 } 2216 2217 return true; 2218 } 2219 2220 static struct page * 2221 khugepaged_alloc_page(struct page **hpage, gfp_t gfp, struct mm_struct *mm, 2222 unsigned long address, int node) 2223 { 2224 VM_BUG_ON_PAGE(*hpage, *hpage); 2225 2226 /* 2227 * Before allocating the hugepage, release the mmap_sem read lock. 2228 * The allocation can take potentially a long time if it involves 2229 * sync compaction, and we do not need to hold the mmap_sem during 2230 * that. We will recheck the vma after taking it again in write mode. 2231 */ 2232 up_read(&mm->mmap_sem); 2233 2234 *hpage = __alloc_pages_node(node, gfp, HPAGE_PMD_ORDER); 2235 if (unlikely(!*hpage)) { 2236 count_vm_event(THP_COLLAPSE_ALLOC_FAILED); 2237 *hpage = ERR_PTR(-ENOMEM); 2238 return NULL; 2239 } 2240 2241 prep_transhuge_page(*hpage); 2242 count_vm_event(THP_COLLAPSE_ALLOC); 2243 return *hpage; 2244 } 2245 #else 2246 static int khugepaged_find_target_node(void) 2247 { 2248 return 0; 2249 } 2250 2251 static inline struct page *alloc_hugepage(int defrag) 2252 { 2253 struct page *page; 2254 2255 page = alloc_pages(alloc_hugepage_gfpmask(defrag, 0), HPAGE_PMD_ORDER); 2256 if (page) 2257 prep_transhuge_page(page); 2258 return page; 2259 } 2260 2261 static struct page *khugepaged_alloc_hugepage(bool *wait) 2262 { 2263 struct page *hpage; 2264 2265 do { 2266 hpage = alloc_hugepage(khugepaged_defrag()); 2267 if (!hpage) { 2268 count_vm_event(THP_COLLAPSE_ALLOC_FAILED); 2269 if (!*wait) 2270 return NULL; 2271 2272 *wait = false; 2273 khugepaged_alloc_sleep(); 2274 } else 2275 count_vm_event(THP_COLLAPSE_ALLOC); 2276 } while (unlikely(!hpage) && likely(khugepaged_enabled())); 2277 2278 return hpage; 2279 } 2280 2281 static bool khugepaged_prealloc_page(struct page **hpage, bool *wait) 2282 { 2283 if (!*hpage) 2284 *hpage = khugepaged_alloc_hugepage(wait); 2285 2286 if (unlikely(!*hpage)) 2287 return false; 2288 2289 return true; 2290 } 2291 2292 static struct page * 2293 khugepaged_alloc_page(struct page **hpage, gfp_t gfp, struct mm_struct *mm, 2294 unsigned long address, int node) 2295 { 2296 up_read(&mm->mmap_sem); 2297 VM_BUG_ON(!*hpage); 2298 2299 return *hpage; 2300 } 2301 #endif 2302 2303 static bool hugepage_vma_check(struct vm_area_struct *vma) 2304 { 2305 if ((!(vma->vm_flags & VM_HUGEPAGE) && !khugepaged_always()) || 2306 (vma->vm_flags & VM_NOHUGEPAGE)) 2307 return false; 2308 if (!vma->anon_vma || vma->vm_ops) 2309 return false; 2310 if (is_vma_temporary_stack(vma)) 2311 return false; 2312 VM_BUG_ON_VMA(vma->vm_flags & VM_NO_THP, vma); 2313 return true; 2314 } 2315 2316 static void collapse_huge_page(struct mm_struct *mm, 2317 unsigned long address, 2318 struct page **hpage, 2319 struct vm_area_struct *vma, 2320 int node) 2321 { 2322 pmd_t *pmd, _pmd; 2323 pte_t *pte; 2324 pgtable_t pgtable; 2325 struct page *new_page; 2326 spinlock_t *pmd_ptl, *pte_ptl; 2327 int isolated = 0, result = 0; 2328 unsigned long hstart, hend; 2329 struct mem_cgroup *memcg; 2330 unsigned long mmun_start; /* For mmu_notifiers */ 2331 unsigned long mmun_end; /* For mmu_notifiers */ 2332 gfp_t gfp; 2333 2334 VM_BUG_ON(address & ~HPAGE_PMD_MASK); 2335 2336 /* Only allocate from the target node */ 2337 gfp = alloc_hugepage_gfpmask(khugepaged_defrag(), __GFP_OTHER_NODE) | 2338 __GFP_THISNODE; 2339 2340 /* release the mmap_sem read lock. */ 2341 new_page = khugepaged_alloc_page(hpage, gfp, mm, address, node); 2342 if (!new_page) { 2343 result = SCAN_ALLOC_HUGE_PAGE_FAIL; 2344 goto out_nolock; 2345 } 2346 2347 if (unlikely(mem_cgroup_try_charge(new_page, mm, gfp, &memcg, true))) { 2348 result = SCAN_CGROUP_CHARGE_FAIL; 2349 goto out_nolock; 2350 } 2351 2352 /* 2353 * Prevent all access to pagetables with the exception of 2354 * gup_fast later hanlded by the ptep_clear_flush and the VM 2355 * handled by the anon_vma lock + PG_lock. 2356 */ 2357 down_write(&mm->mmap_sem); 2358 if (unlikely(khugepaged_test_exit(mm))) { 2359 result = SCAN_ANY_PROCESS; 2360 goto out; 2361 } 2362 2363 vma = find_vma(mm, address); 2364 if (!vma) { 2365 result = SCAN_VMA_NULL; 2366 goto out; 2367 } 2368 hstart = (vma->vm_start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK; 2369 hend = vma->vm_end & HPAGE_PMD_MASK; 2370 if (address < hstart || address + HPAGE_PMD_SIZE > hend) { 2371 result = SCAN_ADDRESS_RANGE; 2372 goto out; 2373 } 2374 if (!hugepage_vma_check(vma)) { 2375 result = SCAN_VMA_CHECK; 2376 goto out; 2377 } 2378 pmd = mm_find_pmd(mm, address); 2379 if (!pmd) { 2380 result = SCAN_PMD_NULL; 2381 goto out; 2382 } 2383 2384 anon_vma_lock_write(vma->anon_vma); 2385 2386 pte = pte_offset_map(pmd, address); 2387 pte_ptl = pte_lockptr(mm, pmd); 2388 2389 mmun_start = address; 2390 mmun_end = address + HPAGE_PMD_SIZE; 2391 mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end); 2392 pmd_ptl = pmd_lock(mm, pmd); /* probably unnecessary */ 2393 /* 2394 * After this gup_fast can't run anymore. This also removes 2395 * any huge TLB entry from the CPU so we won't allow 2396 * huge and small TLB entries for the same virtual address 2397 * to avoid the risk of CPU bugs in that area. 2398 */ 2399 _pmd = pmdp_collapse_flush(vma, address, pmd); 2400 spin_unlock(pmd_ptl); 2401 mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end); 2402 2403 spin_lock(pte_ptl); 2404 isolated = __collapse_huge_page_isolate(vma, address, pte); 2405 spin_unlock(pte_ptl); 2406 2407 if (unlikely(!isolated)) { 2408 pte_unmap(pte); 2409 spin_lock(pmd_ptl); 2410 BUG_ON(!pmd_none(*pmd)); 2411 /* 2412 * We can only use set_pmd_at when establishing 2413 * hugepmds and never for establishing regular pmds that 2414 * points to regular pagetables. Use pmd_populate for that 2415 */ 2416 pmd_populate(mm, pmd, pmd_pgtable(_pmd)); 2417 spin_unlock(pmd_ptl); 2418 anon_vma_unlock_write(vma->anon_vma); 2419 result = SCAN_FAIL; 2420 goto out; 2421 } 2422 2423 /* 2424 * All pages are isolated and locked so anon_vma rmap 2425 * can't run anymore. 2426 */ 2427 anon_vma_unlock_write(vma->anon_vma); 2428 2429 __collapse_huge_page_copy(pte, new_page, vma, address, pte_ptl); 2430 pte_unmap(pte); 2431 __SetPageUptodate(new_page); 2432 pgtable = pmd_pgtable(_pmd); 2433 2434 _pmd = mk_huge_pmd(new_page, vma->vm_page_prot); 2435 _pmd = maybe_pmd_mkwrite(pmd_mkdirty(_pmd), vma); 2436 2437 /* 2438 * spin_lock() below is not the equivalent of smp_wmb(), so 2439 * this is needed to avoid the copy_huge_page writes to become 2440 * visible after the set_pmd_at() write. 2441 */ 2442 smp_wmb(); 2443 2444 spin_lock(pmd_ptl); 2445 BUG_ON(!pmd_none(*pmd)); 2446 page_add_new_anon_rmap(new_page, vma, address, true); 2447 mem_cgroup_commit_charge(new_page, memcg, false, true); 2448 lru_cache_add_active_or_unevictable(new_page, vma); 2449 pgtable_trans_huge_deposit(mm, pmd, pgtable); 2450 set_pmd_at(mm, address, pmd, _pmd); 2451 update_mmu_cache_pmd(vma, address, pmd); 2452 spin_unlock(pmd_ptl); 2453 2454 *hpage = NULL; 2455 2456 khugepaged_pages_collapsed++; 2457 result = SCAN_SUCCEED; 2458 out_up_write: 2459 up_write(&mm->mmap_sem); 2460 trace_mm_collapse_huge_page(mm, isolated, result); 2461 return; 2462 2463 out_nolock: 2464 trace_mm_collapse_huge_page(mm, isolated, result); 2465 return; 2466 out: 2467 mem_cgroup_cancel_charge(new_page, memcg, true); 2468 goto out_up_write; 2469 } 2470 2471 static int khugepaged_scan_pmd(struct mm_struct *mm, 2472 struct vm_area_struct *vma, 2473 unsigned long address, 2474 struct page **hpage) 2475 { 2476 pmd_t *pmd; 2477 pte_t *pte, *_pte; 2478 int ret = 0, none_or_zero = 0, result = 0; 2479 struct page *page = NULL; 2480 unsigned long _address; 2481 spinlock_t *ptl; 2482 int node = NUMA_NO_NODE; 2483 bool writable = false, referenced = false; 2484 2485 VM_BUG_ON(address & ~HPAGE_PMD_MASK); 2486 2487 pmd = mm_find_pmd(mm, address); 2488 if (!pmd) { 2489 result = SCAN_PMD_NULL; 2490 goto out; 2491 } 2492 2493 memset(khugepaged_node_load, 0, sizeof(khugepaged_node_load)); 2494 pte = pte_offset_map_lock(mm, pmd, address, &ptl); 2495 for (_address = address, _pte = pte; _pte < pte+HPAGE_PMD_NR; 2496 _pte++, _address += PAGE_SIZE) { 2497 pte_t pteval = *_pte; 2498 if (pte_none(pteval) || is_zero_pfn(pte_pfn(pteval))) { 2499 if (!userfaultfd_armed(vma) && 2500 ++none_or_zero <= khugepaged_max_ptes_none) { 2501 continue; 2502 } else { 2503 result = SCAN_EXCEED_NONE_PTE; 2504 goto out_unmap; 2505 } 2506 } 2507 if (!pte_present(pteval)) { 2508 result = SCAN_PTE_NON_PRESENT; 2509 goto out_unmap; 2510 } 2511 if (pte_write(pteval)) 2512 writable = true; 2513 2514 page = vm_normal_page(vma, _address, pteval); 2515 if (unlikely(!page)) { 2516 result = SCAN_PAGE_NULL; 2517 goto out_unmap; 2518 } 2519 2520 /* TODO: teach khugepaged to collapse THP mapped with pte */ 2521 if (PageCompound(page)) { 2522 result = SCAN_PAGE_COMPOUND; 2523 goto out_unmap; 2524 } 2525 2526 /* 2527 * Record which node the original page is from and save this 2528 * information to khugepaged_node_load[]. 2529 * Khupaged will allocate hugepage from the node has the max 2530 * hit record. 2531 */ 2532 node = page_to_nid(page); 2533 if (khugepaged_scan_abort(node)) { 2534 result = SCAN_SCAN_ABORT; 2535 goto out_unmap; 2536 } 2537 khugepaged_node_load[node]++; 2538 if (!PageLRU(page)) { 2539 result = SCAN_SCAN_ABORT; 2540 goto out_unmap; 2541 } 2542 if (PageLocked(page)) { 2543 result = SCAN_PAGE_LOCK; 2544 goto out_unmap; 2545 } 2546 if (!PageAnon(page)) { 2547 result = SCAN_PAGE_ANON; 2548 goto out_unmap; 2549 } 2550 2551 /* 2552 * cannot use mapcount: can't collapse if there's a gup pin. 2553 * The page must only be referenced by the scanned process 2554 * and page swap cache. 2555 */ 2556 if (page_count(page) != 1 + !!PageSwapCache(page)) { 2557 result = SCAN_PAGE_COUNT; 2558 goto out_unmap; 2559 } 2560 if (pte_young(pteval) || 2561 page_is_young(page) || PageReferenced(page) || 2562 mmu_notifier_test_young(vma->vm_mm, address)) 2563 referenced = true; 2564 } 2565 if (writable) { 2566 if (referenced) { 2567 result = SCAN_SUCCEED; 2568 ret = 1; 2569 } else { 2570 result = SCAN_NO_REFERENCED_PAGE; 2571 } 2572 } else { 2573 result = SCAN_PAGE_RO; 2574 } 2575 out_unmap: 2576 pte_unmap_unlock(pte, ptl); 2577 if (ret) { 2578 node = khugepaged_find_target_node(); 2579 /* collapse_huge_page will return with the mmap_sem released */ 2580 collapse_huge_page(mm, address, hpage, vma, node); 2581 } 2582 out: 2583 trace_mm_khugepaged_scan_pmd(mm, page, writable, referenced, 2584 none_or_zero, result); 2585 return ret; 2586 } 2587 2588 static void collect_mm_slot(struct mm_slot *mm_slot) 2589 { 2590 struct mm_struct *mm = mm_slot->mm; 2591 2592 VM_BUG_ON(NR_CPUS != 1 && !spin_is_locked(&khugepaged_mm_lock)); 2593 2594 if (khugepaged_test_exit(mm)) { 2595 /* free mm_slot */ 2596 hash_del(&mm_slot->hash); 2597 list_del(&mm_slot->mm_node); 2598 2599 /* 2600 * Not strictly needed because the mm exited already. 2601 * 2602 * clear_bit(MMF_VM_HUGEPAGE, &mm->flags); 2603 */ 2604 2605 /* khugepaged_mm_lock actually not necessary for the below */ 2606 free_mm_slot(mm_slot); 2607 mmdrop(mm); 2608 } 2609 } 2610 2611 static unsigned int khugepaged_scan_mm_slot(unsigned int pages, 2612 struct page **hpage) 2613 __releases(&khugepaged_mm_lock) 2614 __acquires(&khugepaged_mm_lock) 2615 { 2616 struct mm_slot *mm_slot; 2617 struct mm_struct *mm; 2618 struct vm_area_struct *vma; 2619 int progress = 0; 2620 2621 VM_BUG_ON(!pages); 2622 VM_BUG_ON(NR_CPUS != 1 && !spin_is_locked(&khugepaged_mm_lock)); 2623 2624 if (khugepaged_scan.mm_slot) 2625 mm_slot = khugepaged_scan.mm_slot; 2626 else { 2627 mm_slot = list_entry(khugepaged_scan.mm_head.next, 2628 struct mm_slot, mm_node); 2629 khugepaged_scan.address = 0; 2630 khugepaged_scan.mm_slot = mm_slot; 2631 } 2632 spin_unlock(&khugepaged_mm_lock); 2633 2634 mm = mm_slot->mm; 2635 down_read(&mm->mmap_sem); 2636 if (unlikely(khugepaged_test_exit(mm))) 2637 vma = NULL; 2638 else 2639 vma = find_vma(mm, khugepaged_scan.address); 2640 2641 progress++; 2642 for (; vma; vma = vma->vm_next) { 2643 unsigned long hstart, hend; 2644 2645 cond_resched(); 2646 if (unlikely(khugepaged_test_exit(mm))) { 2647 progress++; 2648 break; 2649 } 2650 if (!hugepage_vma_check(vma)) { 2651 skip: 2652 progress++; 2653 continue; 2654 } 2655 hstart = (vma->vm_start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK; 2656 hend = vma->vm_end & HPAGE_PMD_MASK; 2657 if (hstart >= hend) 2658 goto skip; 2659 if (khugepaged_scan.address > hend) 2660 goto skip; 2661 if (khugepaged_scan.address < hstart) 2662 khugepaged_scan.address = hstart; 2663 VM_BUG_ON(khugepaged_scan.address & ~HPAGE_PMD_MASK); 2664 2665 while (khugepaged_scan.address < hend) { 2666 int ret; 2667 cond_resched(); 2668 if (unlikely(khugepaged_test_exit(mm))) 2669 goto breakouterloop; 2670 2671 VM_BUG_ON(khugepaged_scan.address < hstart || 2672 khugepaged_scan.address + HPAGE_PMD_SIZE > 2673 hend); 2674 ret = khugepaged_scan_pmd(mm, vma, 2675 khugepaged_scan.address, 2676 hpage); 2677 /* move to next address */ 2678 khugepaged_scan.address += HPAGE_PMD_SIZE; 2679 progress += HPAGE_PMD_NR; 2680 if (ret) 2681 /* we released mmap_sem so break loop */ 2682 goto breakouterloop_mmap_sem; 2683 if (progress >= pages) 2684 goto breakouterloop; 2685 } 2686 } 2687 breakouterloop: 2688 up_read(&mm->mmap_sem); /* exit_mmap will destroy ptes after this */ 2689 breakouterloop_mmap_sem: 2690 2691 spin_lock(&khugepaged_mm_lock); 2692 VM_BUG_ON(khugepaged_scan.mm_slot != mm_slot); 2693 /* 2694 * Release the current mm_slot if this mm is about to die, or 2695 * if we scanned all vmas of this mm. 2696 */ 2697 if (khugepaged_test_exit(mm) || !vma) { 2698 /* 2699 * Make sure that if mm_users is reaching zero while 2700 * khugepaged runs here, khugepaged_exit will find 2701 * mm_slot not pointing to the exiting mm. 2702 */ 2703 if (mm_slot->mm_node.next != &khugepaged_scan.mm_head) { 2704 khugepaged_scan.mm_slot = list_entry( 2705 mm_slot->mm_node.next, 2706 struct mm_slot, mm_node); 2707 khugepaged_scan.address = 0; 2708 } else { 2709 khugepaged_scan.mm_slot = NULL; 2710 khugepaged_full_scans++; 2711 } 2712 2713 collect_mm_slot(mm_slot); 2714 } 2715 2716 return progress; 2717 } 2718 2719 static int khugepaged_has_work(void) 2720 { 2721 return !list_empty(&khugepaged_scan.mm_head) && 2722 khugepaged_enabled(); 2723 } 2724 2725 static int khugepaged_wait_event(void) 2726 { 2727 return !list_empty(&khugepaged_scan.mm_head) || 2728 kthread_should_stop(); 2729 } 2730 2731 static void khugepaged_do_scan(void) 2732 { 2733 struct page *hpage = NULL; 2734 unsigned int progress = 0, pass_through_head = 0; 2735 unsigned int pages = khugepaged_pages_to_scan; 2736 bool wait = true; 2737 2738 barrier(); /* write khugepaged_pages_to_scan to local stack */ 2739 2740 while (progress < pages) { 2741 if (!khugepaged_prealloc_page(&hpage, &wait)) 2742 break; 2743 2744 cond_resched(); 2745 2746 if (unlikely(kthread_should_stop() || try_to_freeze())) 2747 break; 2748 2749 spin_lock(&khugepaged_mm_lock); 2750 if (!khugepaged_scan.mm_slot) 2751 pass_through_head++; 2752 if (khugepaged_has_work() && 2753 pass_through_head < 2) 2754 progress += khugepaged_scan_mm_slot(pages - progress, 2755 &hpage); 2756 else 2757 progress = pages; 2758 spin_unlock(&khugepaged_mm_lock); 2759 } 2760 2761 if (!IS_ERR_OR_NULL(hpage)) 2762 put_page(hpage); 2763 } 2764 2765 static void khugepaged_wait_work(void) 2766 { 2767 if (khugepaged_has_work()) { 2768 if (!khugepaged_scan_sleep_millisecs) 2769 return; 2770 2771 wait_event_freezable_timeout(khugepaged_wait, 2772 kthread_should_stop(), 2773 msecs_to_jiffies(khugepaged_scan_sleep_millisecs)); 2774 return; 2775 } 2776 2777 if (khugepaged_enabled()) 2778 wait_event_freezable(khugepaged_wait, khugepaged_wait_event()); 2779 } 2780 2781 static int khugepaged(void *none) 2782 { 2783 struct mm_slot *mm_slot; 2784 2785 set_freezable(); 2786 set_user_nice(current, MAX_NICE); 2787 2788 while (!kthread_should_stop()) { 2789 khugepaged_do_scan(); 2790 khugepaged_wait_work(); 2791 } 2792 2793 spin_lock(&khugepaged_mm_lock); 2794 mm_slot = khugepaged_scan.mm_slot; 2795 khugepaged_scan.mm_slot = NULL; 2796 if (mm_slot) 2797 collect_mm_slot(mm_slot); 2798 spin_unlock(&khugepaged_mm_lock); 2799 return 0; 2800 } 2801 2802 static void __split_huge_zero_page_pmd(struct vm_area_struct *vma, 2803 unsigned long haddr, pmd_t *pmd) 2804 { 2805 struct mm_struct *mm = vma->vm_mm; 2806 pgtable_t pgtable; 2807 pmd_t _pmd; 2808 int i; 2809 2810 /* leave pmd empty until pte is filled */ 2811 pmdp_huge_clear_flush_notify(vma, haddr, pmd); 2812 2813 pgtable = pgtable_trans_huge_withdraw(mm, pmd); 2814 pmd_populate(mm, &_pmd, pgtable); 2815 2816 for (i = 0; i < HPAGE_PMD_NR; i++, haddr += PAGE_SIZE) { 2817 pte_t *pte, entry; 2818 entry = pfn_pte(my_zero_pfn(haddr), vma->vm_page_prot); 2819 entry = pte_mkspecial(entry); 2820 pte = pte_offset_map(&_pmd, haddr); 2821 VM_BUG_ON(!pte_none(*pte)); 2822 set_pte_at(mm, haddr, pte, entry); 2823 pte_unmap(pte); 2824 } 2825 smp_wmb(); /* make pte visible before pmd */ 2826 pmd_populate(mm, pmd, pgtable); 2827 put_huge_zero_page(); 2828 } 2829 2830 static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd, 2831 unsigned long haddr, bool freeze) 2832 { 2833 struct mm_struct *mm = vma->vm_mm; 2834 struct page *page; 2835 pgtable_t pgtable; 2836 pmd_t _pmd; 2837 bool young, write, dirty; 2838 int i; 2839 2840 VM_BUG_ON(haddr & ~HPAGE_PMD_MASK); 2841 VM_BUG_ON_VMA(vma->vm_start > haddr, vma); 2842 VM_BUG_ON_VMA(vma->vm_end < haddr + HPAGE_PMD_SIZE, vma); 2843 VM_BUG_ON(!pmd_trans_huge(*pmd) && !pmd_devmap(*pmd)); 2844 2845 count_vm_event(THP_SPLIT_PMD); 2846 2847 if (vma_is_dax(vma)) { 2848 pmd_t _pmd = pmdp_huge_clear_flush_notify(vma, haddr, pmd); 2849 if (is_huge_zero_pmd(_pmd)) 2850 put_huge_zero_page(); 2851 return; 2852 } else if (is_huge_zero_pmd(*pmd)) { 2853 return __split_huge_zero_page_pmd(vma, haddr, pmd); 2854 } 2855 2856 page = pmd_page(*pmd); 2857 VM_BUG_ON_PAGE(!page_count(page), page); 2858 atomic_add(HPAGE_PMD_NR - 1, &page->_count); 2859 write = pmd_write(*pmd); 2860 young = pmd_young(*pmd); 2861 dirty = pmd_dirty(*pmd); 2862 2863 pgtable = pgtable_trans_huge_withdraw(mm, pmd); 2864 pmd_populate(mm, &_pmd, pgtable); 2865 2866 for (i = 0; i < HPAGE_PMD_NR; i++, haddr += PAGE_SIZE) { 2867 pte_t entry, *pte; 2868 /* 2869 * Note that NUMA hinting access restrictions are not 2870 * transferred to avoid any possibility of altering 2871 * permissions across VMAs. 2872 */ 2873 if (freeze) { 2874 swp_entry_t swp_entry; 2875 swp_entry = make_migration_entry(page + i, write); 2876 entry = swp_entry_to_pte(swp_entry); 2877 } else { 2878 entry = mk_pte(page + i, vma->vm_page_prot); 2879 entry = maybe_mkwrite(entry, vma); 2880 if (!write) 2881 entry = pte_wrprotect(entry); 2882 if (!young) 2883 entry = pte_mkold(entry); 2884 } 2885 if (dirty) 2886 SetPageDirty(page + i); 2887 pte = pte_offset_map(&_pmd, haddr); 2888 BUG_ON(!pte_none(*pte)); 2889 set_pte_at(mm, haddr, pte, entry); 2890 atomic_inc(&page[i]._mapcount); 2891 pte_unmap(pte); 2892 } 2893 2894 /* 2895 * Set PG_double_map before dropping compound_mapcount to avoid 2896 * false-negative page_mapped(). 2897 */ 2898 if (compound_mapcount(page) > 1 && !TestSetPageDoubleMap(page)) { 2899 for (i = 0; i < HPAGE_PMD_NR; i++) 2900 atomic_inc(&page[i]._mapcount); 2901 } 2902 2903 if (atomic_add_negative(-1, compound_mapcount_ptr(page))) { 2904 /* Last compound_mapcount is gone. */ 2905 __dec_zone_page_state(page, NR_ANON_TRANSPARENT_HUGEPAGES); 2906 if (TestClearPageDoubleMap(page)) { 2907 /* No need in mapcount reference anymore */ 2908 for (i = 0; i < HPAGE_PMD_NR; i++) 2909 atomic_dec(&page[i]._mapcount); 2910 } 2911 } 2912 2913 smp_wmb(); /* make pte visible before pmd */ 2914 /* 2915 * Up to this point the pmd is present and huge and userland has the 2916 * whole access to the hugepage during the split (which happens in 2917 * place). If we overwrite the pmd with the not-huge version pointing 2918 * to the pte here (which of course we could if all CPUs were bug 2919 * free), userland could trigger a small page size TLB miss on the 2920 * small sized TLB while the hugepage TLB entry is still established in 2921 * the huge TLB. Some CPU doesn't like that. 2922 * See http://support.amd.com/us/Processor_TechDocs/41322.pdf, Erratum 2923 * 383 on page 93. Intel should be safe but is also warns that it's 2924 * only safe if the permission and cache attributes of the two entries 2925 * loaded in the two TLB is identical (which should be the case here). 2926 * But it is generally safer to never allow small and huge TLB entries 2927 * for the same virtual address to be loaded simultaneously. So instead 2928 * of doing "pmd_populate(); flush_pmd_tlb_range();" we first mark the 2929 * current pmd notpresent (atomically because here the pmd_trans_huge 2930 * and pmd_trans_splitting must remain set at all times on the pmd 2931 * until the split is complete for this pmd), then we flush the SMP TLB 2932 * and finally we write the non-huge version of the pmd entry with 2933 * pmd_populate. 2934 */ 2935 pmdp_invalidate(vma, haddr, pmd); 2936 pmd_populate(mm, pmd, pgtable); 2937 2938 if (freeze) { 2939 for (i = 0; i < HPAGE_PMD_NR; i++, haddr += PAGE_SIZE) { 2940 page_remove_rmap(page + i, false); 2941 put_page(page + i); 2942 } 2943 } 2944 } 2945 2946 void __split_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd, 2947 unsigned long address) 2948 { 2949 spinlock_t *ptl; 2950 struct mm_struct *mm = vma->vm_mm; 2951 struct page *page = NULL; 2952 unsigned long haddr = address & HPAGE_PMD_MASK; 2953 2954 mmu_notifier_invalidate_range_start(mm, haddr, haddr + HPAGE_PMD_SIZE); 2955 ptl = pmd_lock(mm, pmd); 2956 if (pmd_trans_huge(*pmd)) { 2957 page = pmd_page(*pmd); 2958 if (PageMlocked(page)) 2959 get_page(page); 2960 else 2961 page = NULL; 2962 } else if (!pmd_devmap(*pmd)) 2963 goto out; 2964 __split_huge_pmd_locked(vma, pmd, haddr, false); 2965 out: 2966 spin_unlock(ptl); 2967 mmu_notifier_invalidate_range_end(mm, haddr, haddr + HPAGE_PMD_SIZE); 2968 if (page) { 2969 lock_page(page); 2970 munlock_vma_page(page); 2971 unlock_page(page); 2972 put_page(page); 2973 } 2974 } 2975 2976 static void split_huge_pmd_address(struct vm_area_struct *vma, 2977 unsigned long address) 2978 { 2979 pgd_t *pgd; 2980 pud_t *pud; 2981 pmd_t *pmd; 2982 2983 VM_BUG_ON(!(address & ~HPAGE_PMD_MASK)); 2984 2985 pgd = pgd_offset(vma->vm_mm, address); 2986 if (!pgd_present(*pgd)) 2987 return; 2988 2989 pud = pud_offset(pgd, address); 2990 if (!pud_present(*pud)) 2991 return; 2992 2993 pmd = pmd_offset(pud, address); 2994 if (!pmd_present(*pmd) || (!pmd_trans_huge(*pmd) && !pmd_devmap(*pmd))) 2995 return; 2996 /* 2997 * Caller holds the mmap_sem write mode, so a huge pmd cannot 2998 * materialize from under us. 2999 */ 3000 split_huge_pmd(vma, pmd, address); 3001 } 3002 3003 void vma_adjust_trans_huge(struct vm_area_struct *vma, 3004 unsigned long start, 3005 unsigned long end, 3006 long adjust_next) 3007 { 3008 /* 3009 * If the new start address isn't hpage aligned and it could 3010 * previously contain an hugepage: check if we need to split 3011 * an huge pmd. 3012 */ 3013 if (start & ~HPAGE_PMD_MASK && 3014 (start & HPAGE_PMD_MASK) >= vma->vm_start && 3015 (start & HPAGE_PMD_MASK) + HPAGE_PMD_SIZE <= vma->vm_end) 3016 split_huge_pmd_address(vma, start); 3017 3018 /* 3019 * If the new end address isn't hpage aligned and it could 3020 * previously contain an hugepage: check if we need to split 3021 * an huge pmd. 3022 */ 3023 if (end & ~HPAGE_PMD_MASK && 3024 (end & HPAGE_PMD_MASK) >= vma->vm_start && 3025 (end & HPAGE_PMD_MASK) + HPAGE_PMD_SIZE <= vma->vm_end) 3026 split_huge_pmd_address(vma, end); 3027 3028 /* 3029 * If we're also updating the vma->vm_next->vm_start, if the new 3030 * vm_next->vm_start isn't page aligned and it could previously 3031 * contain an hugepage: check if we need to split an huge pmd. 3032 */ 3033 if (adjust_next > 0) { 3034 struct vm_area_struct *next = vma->vm_next; 3035 unsigned long nstart = next->vm_start; 3036 nstart += adjust_next << PAGE_SHIFT; 3037 if (nstart & ~HPAGE_PMD_MASK && 3038 (nstart & HPAGE_PMD_MASK) >= next->vm_start && 3039 (nstart & HPAGE_PMD_MASK) + HPAGE_PMD_SIZE <= next->vm_end) 3040 split_huge_pmd_address(next, nstart); 3041 } 3042 } 3043 3044 static void freeze_page_vma(struct vm_area_struct *vma, struct page *page, 3045 unsigned long address) 3046 { 3047 unsigned long haddr = address & HPAGE_PMD_MASK; 3048 spinlock_t *ptl; 3049 pgd_t *pgd; 3050 pud_t *pud; 3051 pmd_t *pmd; 3052 pte_t *pte; 3053 int i, nr = HPAGE_PMD_NR; 3054 3055 /* Skip pages which doesn't belong to the VMA */ 3056 if (address < vma->vm_start) { 3057 int off = (vma->vm_start - address) >> PAGE_SHIFT; 3058 page += off; 3059 nr -= off; 3060 address = vma->vm_start; 3061 } 3062 3063 pgd = pgd_offset(vma->vm_mm, address); 3064 if (!pgd_present(*pgd)) 3065 return; 3066 pud = pud_offset(pgd, address); 3067 if (!pud_present(*pud)) 3068 return; 3069 pmd = pmd_offset(pud, address); 3070 ptl = pmd_lock(vma->vm_mm, pmd); 3071 if (!pmd_present(*pmd)) { 3072 spin_unlock(ptl); 3073 return; 3074 } 3075 if (pmd_trans_huge(*pmd)) { 3076 if (page == pmd_page(*pmd)) 3077 __split_huge_pmd_locked(vma, pmd, haddr, true); 3078 spin_unlock(ptl); 3079 return; 3080 } 3081 spin_unlock(ptl); 3082 3083 pte = pte_offset_map_lock(vma->vm_mm, pmd, address, &ptl); 3084 for (i = 0; i < nr; i++, address += PAGE_SIZE, page++, pte++) { 3085 pte_t entry, swp_pte; 3086 swp_entry_t swp_entry; 3087 3088 /* 3089 * We've just crossed page table boundary: need to map next one. 3090 * It can happen if THP was mremaped to non PMD-aligned address. 3091 */ 3092 if (unlikely(address == haddr + HPAGE_PMD_SIZE)) { 3093 pte_unmap_unlock(pte - 1, ptl); 3094 pmd = mm_find_pmd(vma->vm_mm, address); 3095 if (!pmd) 3096 return; 3097 pte = pte_offset_map_lock(vma->vm_mm, pmd, 3098 address, &ptl); 3099 } 3100 3101 if (!pte_present(*pte)) 3102 continue; 3103 if (page_to_pfn(page) != pte_pfn(*pte)) 3104 continue; 3105 flush_cache_page(vma, address, page_to_pfn(page)); 3106 entry = ptep_clear_flush(vma, address, pte); 3107 if (pte_dirty(entry)) 3108 SetPageDirty(page); 3109 swp_entry = make_migration_entry(page, pte_write(entry)); 3110 swp_pte = swp_entry_to_pte(swp_entry); 3111 if (pte_soft_dirty(entry)) 3112 swp_pte = pte_swp_mksoft_dirty(swp_pte); 3113 set_pte_at(vma->vm_mm, address, pte, swp_pte); 3114 page_remove_rmap(page, false); 3115 put_page(page); 3116 } 3117 pte_unmap_unlock(pte - 1, ptl); 3118 } 3119 3120 static void freeze_page(struct anon_vma *anon_vma, struct page *page) 3121 { 3122 struct anon_vma_chain *avc; 3123 pgoff_t pgoff = page_to_pgoff(page); 3124 3125 VM_BUG_ON_PAGE(!PageHead(page), page); 3126 3127 anon_vma_interval_tree_foreach(avc, &anon_vma->rb_root, pgoff, 3128 pgoff + HPAGE_PMD_NR - 1) { 3129 unsigned long address = __vma_address(page, avc->vma); 3130 3131 mmu_notifier_invalidate_range_start(avc->vma->vm_mm, 3132 address, address + HPAGE_PMD_SIZE); 3133 freeze_page_vma(avc->vma, page, address); 3134 mmu_notifier_invalidate_range_end(avc->vma->vm_mm, 3135 address, address + HPAGE_PMD_SIZE); 3136 } 3137 } 3138 3139 static void unfreeze_page_vma(struct vm_area_struct *vma, struct page *page, 3140 unsigned long address) 3141 { 3142 spinlock_t *ptl; 3143 pmd_t *pmd; 3144 pte_t *pte, entry; 3145 swp_entry_t swp_entry; 3146 unsigned long haddr = address & HPAGE_PMD_MASK; 3147 int i, nr = HPAGE_PMD_NR; 3148 3149 /* Skip pages which doesn't belong to the VMA */ 3150 if (address < vma->vm_start) { 3151 int off = (vma->vm_start - address) >> PAGE_SHIFT; 3152 page += off; 3153 nr -= off; 3154 address = vma->vm_start; 3155 } 3156 3157 pmd = mm_find_pmd(vma->vm_mm, address); 3158 if (!pmd) 3159 return; 3160 3161 pte = pte_offset_map_lock(vma->vm_mm, pmd, address, &ptl); 3162 for (i = 0; i < nr; i++, address += PAGE_SIZE, page++, pte++) { 3163 /* 3164 * We've just crossed page table boundary: need to map next one. 3165 * It can happen if THP was mremaped to non-PMD aligned address. 3166 */ 3167 if (unlikely(address == haddr + HPAGE_PMD_SIZE)) { 3168 pte_unmap_unlock(pte - 1, ptl); 3169 pmd = mm_find_pmd(vma->vm_mm, address); 3170 if (!pmd) 3171 return; 3172 pte = pte_offset_map_lock(vma->vm_mm, pmd, 3173 address, &ptl); 3174 } 3175 3176 if (!is_swap_pte(*pte)) 3177 continue; 3178 3179 swp_entry = pte_to_swp_entry(*pte); 3180 if (!is_migration_entry(swp_entry)) 3181 continue; 3182 if (migration_entry_to_page(swp_entry) != page) 3183 continue; 3184 3185 get_page(page); 3186 page_add_anon_rmap(page, vma, address, false); 3187 3188 entry = pte_mkold(mk_pte(page, vma->vm_page_prot)); 3189 if (PageDirty(page)) 3190 entry = pte_mkdirty(entry); 3191 if (is_write_migration_entry(swp_entry)) 3192 entry = maybe_mkwrite(entry, vma); 3193 3194 flush_dcache_page(page); 3195 set_pte_at(vma->vm_mm, address, pte, entry); 3196 3197 /* No need to invalidate - it was non-present before */ 3198 update_mmu_cache(vma, address, pte); 3199 } 3200 pte_unmap_unlock(pte - 1, ptl); 3201 } 3202 3203 static void unfreeze_page(struct anon_vma *anon_vma, struct page *page) 3204 { 3205 struct anon_vma_chain *avc; 3206 pgoff_t pgoff = page_to_pgoff(page); 3207 3208 anon_vma_interval_tree_foreach(avc, &anon_vma->rb_root, 3209 pgoff, pgoff + HPAGE_PMD_NR - 1) { 3210 unsigned long address = __vma_address(page, avc->vma); 3211 3212 mmu_notifier_invalidate_range_start(avc->vma->vm_mm, 3213 address, address + HPAGE_PMD_SIZE); 3214 unfreeze_page_vma(avc->vma, page, address); 3215 mmu_notifier_invalidate_range_end(avc->vma->vm_mm, 3216 address, address + HPAGE_PMD_SIZE); 3217 } 3218 } 3219 3220 static int __split_huge_page_tail(struct page *head, int tail, 3221 struct lruvec *lruvec, struct list_head *list) 3222 { 3223 int mapcount; 3224 struct page *page_tail = head + tail; 3225 3226 mapcount = atomic_read(&page_tail->_mapcount) + 1; 3227 VM_BUG_ON_PAGE(atomic_read(&page_tail->_count) != 0, page_tail); 3228 3229 /* 3230 * tail_page->_count is zero and not changing from under us. But 3231 * get_page_unless_zero() may be running from under us on the 3232 * tail_page. If we used atomic_set() below instead of atomic_add(), we 3233 * would then run atomic_set() concurrently with 3234 * get_page_unless_zero(), and atomic_set() is implemented in C not 3235 * using locked ops. spin_unlock on x86 sometime uses locked ops 3236 * because of PPro errata 66, 92, so unless somebody can guarantee 3237 * atomic_set() here would be safe on all archs (and not only on x86), 3238 * it's safer to use atomic_add(). 3239 */ 3240 atomic_add(mapcount + 1, &page_tail->_count); 3241 3242 3243 page_tail->flags &= ~PAGE_FLAGS_CHECK_AT_PREP; 3244 page_tail->flags |= (head->flags & 3245 ((1L << PG_referenced) | 3246 (1L << PG_swapbacked) | 3247 (1L << PG_mlocked) | 3248 (1L << PG_uptodate) | 3249 (1L << PG_active) | 3250 (1L << PG_locked) | 3251 (1L << PG_unevictable) | 3252 (1L << PG_dirty))); 3253 3254 /* 3255 * After clearing PageTail the gup refcount can be released. 3256 * Page flags also must be visible before we make the page non-compound. 3257 */ 3258 smp_wmb(); 3259 3260 clear_compound_head(page_tail); 3261 3262 if (page_is_young(head)) 3263 set_page_young(page_tail); 3264 if (page_is_idle(head)) 3265 set_page_idle(page_tail); 3266 3267 /* ->mapping in first tail page is compound_mapcount */ 3268 VM_BUG_ON_PAGE(tail > 2 && page_tail->mapping != TAIL_MAPPING, 3269 page_tail); 3270 page_tail->mapping = head->mapping; 3271 3272 page_tail->index = head->index + tail; 3273 page_cpupid_xchg_last(page_tail, page_cpupid_last(head)); 3274 lru_add_page_tail(head, page_tail, lruvec, list); 3275 3276 return mapcount; 3277 } 3278 3279 static void __split_huge_page(struct page *page, struct list_head *list) 3280 { 3281 struct page *head = compound_head(page); 3282 struct zone *zone = page_zone(head); 3283 struct lruvec *lruvec; 3284 int i, tail_mapcount; 3285 3286 /* prevent PageLRU to go away from under us, and freeze lru stats */ 3287 spin_lock_irq(&zone->lru_lock); 3288 lruvec = mem_cgroup_page_lruvec(head, zone); 3289 3290 /* complete memcg works before add pages to LRU */ 3291 mem_cgroup_split_huge_fixup(head); 3292 3293 tail_mapcount = 0; 3294 for (i = HPAGE_PMD_NR - 1; i >= 1; i--) 3295 tail_mapcount += __split_huge_page_tail(head, i, lruvec, list); 3296 atomic_sub(tail_mapcount, &head->_count); 3297 3298 ClearPageCompound(head); 3299 spin_unlock_irq(&zone->lru_lock); 3300 3301 unfreeze_page(page_anon_vma(head), head); 3302 3303 for (i = 0; i < HPAGE_PMD_NR; i++) { 3304 struct page *subpage = head + i; 3305 if (subpage == page) 3306 continue; 3307 unlock_page(subpage); 3308 3309 /* 3310 * Subpages may be freed if there wasn't any mapping 3311 * like if add_to_swap() is running on a lru page that 3312 * had its mapping zapped. And freeing these pages 3313 * requires taking the lru_lock so we do the put_page 3314 * of the tail pages after the split is complete. 3315 */ 3316 put_page(subpage); 3317 } 3318 } 3319 3320 int total_mapcount(struct page *page) 3321 { 3322 int i, ret; 3323 3324 VM_BUG_ON_PAGE(PageTail(page), page); 3325 3326 if (likely(!PageCompound(page))) 3327 return atomic_read(&page->_mapcount) + 1; 3328 3329 ret = compound_mapcount(page); 3330 if (PageHuge(page)) 3331 return ret; 3332 for (i = 0; i < HPAGE_PMD_NR; i++) 3333 ret += atomic_read(&page[i]._mapcount) + 1; 3334 if (PageDoubleMap(page)) 3335 ret -= HPAGE_PMD_NR; 3336 return ret; 3337 } 3338 3339 /* 3340 * This function splits huge page into normal pages. @page can point to any 3341 * subpage of huge page to split. Split doesn't change the position of @page. 3342 * 3343 * Only caller must hold pin on the @page, otherwise split fails with -EBUSY. 3344 * The huge page must be locked. 3345 * 3346 * If @list is null, tail pages will be added to LRU list, otherwise, to @list. 3347 * 3348 * Both head page and tail pages will inherit mapping, flags, and so on from 3349 * the hugepage. 3350 * 3351 * GUP pin and PG_locked transferred to @page. Rest subpages can be freed if 3352 * they are not mapped. 3353 * 3354 * Returns 0 if the hugepage is split successfully. 3355 * Returns -EBUSY if the page is pinned or if anon_vma disappeared from under 3356 * us. 3357 */ 3358 int split_huge_page_to_list(struct page *page, struct list_head *list) 3359 { 3360 struct page *head = compound_head(page); 3361 struct anon_vma *anon_vma; 3362 int count, mapcount, ret; 3363 bool mlocked; 3364 unsigned long flags; 3365 3366 VM_BUG_ON_PAGE(is_huge_zero_page(page), page); 3367 VM_BUG_ON_PAGE(!PageAnon(page), page); 3368 VM_BUG_ON_PAGE(!PageLocked(page), page); 3369 VM_BUG_ON_PAGE(!PageSwapBacked(page), page); 3370 VM_BUG_ON_PAGE(!PageCompound(page), page); 3371 3372 /* 3373 * The caller does not necessarily hold an mmap_sem that would prevent 3374 * the anon_vma disappearing so we first we take a reference to it 3375 * and then lock the anon_vma for write. This is similar to 3376 * page_lock_anon_vma_read except the write lock is taken to serialise 3377 * against parallel split or collapse operations. 3378 */ 3379 anon_vma = page_get_anon_vma(head); 3380 if (!anon_vma) { 3381 ret = -EBUSY; 3382 goto out; 3383 } 3384 anon_vma_lock_write(anon_vma); 3385 3386 /* 3387 * Racy check if we can split the page, before freeze_page() will 3388 * split PMDs 3389 */ 3390 if (total_mapcount(head) != page_count(head) - 1) { 3391 ret = -EBUSY; 3392 goto out_unlock; 3393 } 3394 3395 mlocked = PageMlocked(page); 3396 freeze_page(anon_vma, head); 3397 VM_BUG_ON_PAGE(compound_mapcount(head), head); 3398 3399 /* Make sure the page is not on per-CPU pagevec as it takes pin */ 3400 if (mlocked) 3401 lru_add_drain(); 3402 3403 /* Prevent deferred_split_scan() touching ->_count */ 3404 spin_lock_irqsave(&split_queue_lock, flags); 3405 count = page_count(head); 3406 mapcount = total_mapcount(head); 3407 if (!mapcount && count == 1) { 3408 if (!list_empty(page_deferred_list(head))) { 3409 split_queue_len--; 3410 list_del(page_deferred_list(head)); 3411 } 3412 spin_unlock_irqrestore(&split_queue_lock, flags); 3413 __split_huge_page(page, list); 3414 ret = 0; 3415 } else if (IS_ENABLED(CONFIG_DEBUG_VM) && mapcount) { 3416 spin_unlock_irqrestore(&split_queue_lock, flags); 3417 pr_alert("total_mapcount: %u, page_count(): %u\n", 3418 mapcount, count); 3419 if (PageTail(page)) 3420 dump_page(head, NULL); 3421 dump_page(page, "total_mapcount(head) > 0"); 3422 BUG(); 3423 } else { 3424 spin_unlock_irqrestore(&split_queue_lock, flags); 3425 unfreeze_page(anon_vma, head); 3426 ret = -EBUSY; 3427 } 3428 3429 out_unlock: 3430 anon_vma_unlock_write(anon_vma); 3431 put_anon_vma(anon_vma); 3432 out: 3433 count_vm_event(!ret ? THP_SPLIT_PAGE : THP_SPLIT_PAGE_FAILED); 3434 return ret; 3435 } 3436 3437 void free_transhuge_page(struct page *page) 3438 { 3439 unsigned long flags; 3440 3441 spin_lock_irqsave(&split_queue_lock, flags); 3442 if (!list_empty(page_deferred_list(page))) { 3443 split_queue_len--; 3444 list_del(page_deferred_list(page)); 3445 } 3446 spin_unlock_irqrestore(&split_queue_lock, flags); 3447 free_compound_page(page); 3448 } 3449 3450 void deferred_split_huge_page(struct page *page) 3451 { 3452 unsigned long flags; 3453 3454 VM_BUG_ON_PAGE(!PageTransHuge(page), page); 3455 3456 spin_lock_irqsave(&split_queue_lock, flags); 3457 if (list_empty(page_deferred_list(page))) { 3458 list_add_tail(page_deferred_list(page), &split_queue); 3459 split_queue_len++; 3460 } 3461 spin_unlock_irqrestore(&split_queue_lock, flags); 3462 } 3463 3464 static unsigned long deferred_split_count(struct shrinker *shrink, 3465 struct shrink_control *sc) 3466 { 3467 /* 3468 * Split a page from split_queue will free up at least one page, 3469 * at most HPAGE_PMD_NR - 1. We don't track exact number. 3470 * Let's use HPAGE_PMD_NR / 2 as ballpark. 3471 */ 3472 return ACCESS_ONCE(split_queue_len) * HPAGE_PMD_NR / 2; 3473 } 3474 3475 static unsigned long deferred_split_scan(struct shrinker *shrink, 3476 struct shrink_control *sc) 3477 { 3478 unsigned long flags; 3479 LIST_HEAD(list), *pos, *next; 3480 struct page *page; 3481 int split = 0; 3482 3483 spin_lock_irqsave(&split_queue_lock, flags); 3484 list_splice_init(&split_queue, &list); 3485 3486 /* Take pin on all head pages to avoid freeing them under us */ 3487 list_for_each_safe(pos, next, &list) { 3488 page = list_entry((void *)pos, struct page, mapping); 3489 page = compound_head(page); 3490 /* race with put_compound_page() */ 3491 if (!get_page_unless_zero(page)) { 3492 list_del_init(page_deferred_list(page)); 3493 split_queue_len--; 3494 } 3495 } 3496 spin_unlock_irqrestore(&split_queue_lock, flags); 3497 3498 list_for_each_safe(pos, next, &list) { 3499 page = list_entry((void *)pos, struct page, mapping); 3500 lock_page(page); 3501 /* split_huge_page() removes page from list on success */ 3502 if (!split_huge_page(page)) 3503 split++; 3504 unlock_page(page); 3505 put_page(page); 3506 } 3507 3508 spin_lock_irqsave(&split_queue_lock, flags); 3509 list_splice_tail(&list, &split_queue); 3510 spin_unlock_irqrestore(&split_queue_lock, flags); 3511 3512 return split * HPAGE_PMD_NR / 2; 3513 } 3514 3515 static struct shrinker deferred_split_shrinker = { 3516 .count_objects = deferred_split_count, 3517 .scan_objects = deferred_split_scan, 3518 .seeks = DEFAULT_SEEKS, 3519 }; 3520 3521 #ifdef CONFIG_DEBUG_FS 3522 static int split_huge_pages_set(void *data, u64 val) 3523 { 3524 struct zone *zone; 3525 struct page *page; 3526 unsigned long pfn, max_zone_pfn; 3527 unsigned long total = 0, split = 0; 3528 3529 if (val != 1) 3530 return -EINVAL; 3531 3532 for_each_populated_zone(zone) { 3533 max_zone_pfn = zone_end_pfn(zone); 3534 for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++) { 3535 if (!pfn_valid(pfn)) 3536 continue; 3537 3538 page = pfn_to_page(pfn); 3539 if (!get_page_unless_zero(page)) 3540 continue; 3541 3542 if (zone != page_zone(page)) 3543 goto next; 3544 3545 if (!PageHead(page) || !PageAnon(page) || 3546 PageHuge(page)) 3547 goto next; 3548 3549 total++; 3550 lock_page(page); 3551 if (!split_huge_page(page)) 3552 split++; 3553 unlock_page(page); 3554 next: 3555 put_page(page); 3556 } 3557 } 3558 3559 pr_info("%lu of %lu THP split", split, total); 3560 3561 return 0; 3562 } 3563 DEFINE_SIMPLE_ATTRIBUTE(split_huge_pages_fops, NULL, split_huge_pages_set, 3564 "%llu\n"); 3565 3566 static int __init split_huge_pages_debugfs(void) 3567 { 3568 void *ret; 3569 3570 ret = debugfs_create_file("split_huge_pages", 0644, NULL, NULL, 3571 &split_huge_pages_fops); 3572 if (!ret) 3573 pr_warn("Failed to create split_huge_pages in debugfs"); 3574 return 0; 3575 } 3576 late_initcall(split_huge_pages_debugfs); 3577 #endif 3578