1 /* 2 * Copyright (C) 2009 Red Hat, Inc. 3 * 4 * This work is licensed under the terms of the GNU GPL, version 2. See 5 * the COPYING file in the top-level directory. 6 */ 7 8 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 9 10 #include <linux/mm.h> 11 #include <linux/sched.h> 12 #include <linux/sched/coredump.h> 13 #include <linux/sched/numa_balancing.h> 14 #include <linux/highmem.h> 15 #include <linux/hugetlb.h> 16 #include <linux/mmu_notifier.h> 17 #include <linux/rmap.h> 18 #include <linux/swap.h> 19 #include <linux/shrinker.h> 20 #include <linux/mm_inline.h> 21 #include <linux/swapops.h> 22 #include <linux/dax.h> 23 #include <linux/khugepaged.h> 24 #include <linux/freezer.h> 25 #include <linux/pfn_t.h> 26 #include <linux/mman.h> 27 #include <linux/memremap.h> 28 #include <linux/pagemap.h> 29 #include <linux/debugfs.h> 30 #include <linux/migrate.h> 31 #include <linux/hashtable.h> 32 #include <linux/userfaultfd_k.h> 33 #include <linux/page_idle.h> 34 #include <linux/shmem_fs.h> 35 #include <linux/oom.h> 36 #include <linux/numa.h> 37 38 #include <asm/tlb.h> 39 #include <asm/pgalloc.h> 40 #include "internal.h" 41 42 /* 43 * By default, transparent hugepage support is disabled in order to avoid 44 * risking an increased memory footprint for applications that are not 45 * guaranteed to benefit from it. When transparent hugepage support is 46 * enabled, it is for all mappings, and khugepaged scans all mappings. 47 * Defrag is invoked by khugepaged hugepage allocations and by page faults 48 * for all hugepage allocations. 49 */ 50 unsigned long transparent_hugepage_flags __read_mostly = 51 #ifdef CONFIG_TRANSPARENT_HUGEPAGE_ALWAYS 52 (1<<TRANSPARENT_HUGEPAGE_FLAG)| 53 #endif 54 #ifdef CONFIG_TRANSPARENT_HUGEPAGE_MADVISE 55 (1<<TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG)| 56 #endif 57 (1<<TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG)| 58 (1<<TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG)| 59 (1<<TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG); 60 61 static struct shrinker deferred_split_shrinker; 62 63 static atomic_t huge_zero_refcount; 64 struct page *huge_zero_page __read_mostly; 65 66 bool transparent_hugepage_enabled(struct vm_area_struct *vma) 67 { 68 if (vma_is_anonymous(vma)) 69 return __transparent_hugepage_enabled(vma); 70 if (vma_is_shmem(vma) && shmem_huge_enabled(vma)) 71 return __transparent_hugepage_enabled(vma); 72 73 return false; 74 } 75 76 static struct page *get_huge_zero_page(void) 77 { 78 struct page *zero_page; 79 retry: 80 if (likely(atomic_inc_not_zero(&huge_zero_refcount))) 81 return READ_ONCE(huge_zero_page); 82 83 zero_page = alloc_pages((GFP_TRANSHUGE | __GFP_ZERO) & ~__GFP_MOVABLE, 84 HPAGE_PMD_ORDER); 85 if (!zero_page) { 86 count_vm_event(THP_ZERO_PAGE_ALLOC_FAILED); 87 return NULL; 88 } 89 count_vm_event(THP_ZERO_PAGE_ALLOC); 90 preempt_disable(); 91 if (cmpxchg(&huge_zero_page, NULL, zero_page)) { 92 preempt_enable(); 93 __free_pages(zero_page, compound_order(zero_page)); 94 goto retry; 95 } 96 97 /* We take additional reference here. It will be put back by shrinker */ 98 atomic_set(&huge_zero_refcount, 2); 99 preempt_enable(); 100 return READ_ONCE(huge_zero_page); 101 } 102 103 static void put_huge_zero_page(void) 104 { 105 /* 106 * Counter should never go to zero here. Only shrinker can put 107 * last reference. 108 */ 109 BUG_ON(atomic_dec_and_test(&huge_zero_refcount)); 110 } 111 112 struct page *mm_get_huge_zero_page(struct mm_struct *mm) 113 { 114 if (test_bit(MMF_HUGE_ZERO_PAGE, &mm->flags)) 115 return READ_ONCE(huge_zero_page); 116 117 if (!get_huge_zero_page()) 118 return NULL; 119 120 if (test_and_set_bit(MMF_HUGE_ZERO_PAGE, &mm->flags)) 121 put_huge_zero_page(); 122 123 return READ_ONCE(huge_zero_page); 124 } 125 126 void mm_put_huge_zero_page(struct mm_struct *mm) 127 { 128 if (test_bit(MMF_HUGE_ZERO_PAGE, &mm->flags)) 129 put_huge_zero_page(); 130 } 131 132 static unsigned long shrink_huge_zero_page_count(struct shrinker *shrink, 133 struct shrink_control *sc) 134 { 135 /* we can free zero page only if last reference remains */ 136 return atomic_read(&huge_zero_refcount) == 1 ? HPAGE_PMD_NR : 0; 137 } 138 139 static unsigned long shrink_huge_zero_page_scan(struct shrinker *shrink, 140 struct shrink_control *sc) 141 { 142 if (atomic_cmpxchg(&huge_zero_refcount, 1, 0) == 1) { 143 struct page *zero_page = xchg(&huge_zero_page, NULL); 144 BUG_ON(zero_page == NULL); 145 __free_pages(zero_page, compound_order(zero_page)); 146 return HPAGE_PMD_NR; 147 } 148 149 return 0; 150 } 151 152 static struct shrinker huge_zero_page_shrinker = { 153 .count_objects = shrink_huge_zero_page_count, 154 .scan_objects = shrink_huge_zero_page_scan, 155 .seeks = DEFAULT_SEEKS, 156 }; 157 158 #ifdef CONFIG_SYSFS 159 static ssize_t enabled_show(struct kobject *kobj, 160 struct kobj_attribute *attr, char *buf) 161 { 162 if (test_bit(TRANSPARENT_HUGEPAGE_FLAG, &transparent_hugepage_flags)) 163 return sprintf(buf, "[always] madvise never\n"); 164 else if (test_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, &transparent_hugepage_flags)) 165 return sprintf(buf, "always [madvise] never\n"); 166 else 167 return sprintf(buf, "always madvise [never]\n"); 168 } 169 170 static ssize_t enabled_store(struct kobject *kobj, 171 struct kobj_attribute *attr, 172 const char *buf, size_t count) 173 { 174 ssize_t ret = count; 175 176 if (!memcmp("always", buf, 177 min(sizeof("always")-1, count))) { 178 clear_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, &transparent_hugepage_flags); 179 set_bit(TRANSPARENT_HUGEPAGE_FLAG, &transparent_hugepage_flags); 180 } else if (!memcmp("madvise", buf, 181 min(sizeof("madvise")-1, count))) { 182 clear_bit(TRANSPARENT_HUGEPAGE_FLAG, &transparent_hugepage_flags); 183 set_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, &transparent_hugepage_flags); 184 } else if (!memcmp("never", buf, 185 min(sizeof("never")-1, count))) { 186 clear_bit(TRANSPARENT_HUGEPAGE_FLAG, &transparent_hugepage_flags); 187 clear_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, &transparent_hugepage_flags); 188 } else 189 ret = -EINVAL; 190 191 if (ret > 0) { 192 int err = start_stop_khugepaged(); 193 if (err) 194 ret = err; 195 } 196 return ret; 197 } 198 static struct kobj_attribute enabled_attr = 199 __ATTR(enabled, 0644, enabled_show, enabled_store); 200 201 ssize_t single_hugepage_flag_show(struct kobject *kobj, 202 struct kobj_attribute *attr, char *buf, 203 enum transparent_hugepage_flag flag) 204 { 205 return sprintf(buf, "%d\n", 206 !!test_bit(flag, &transparent_hugepage_flags)); 207 } 208 209 ssize_t single_hugepage_flag_store(struct kobject *kobj, 210 struct kobj_attribute *attr, 211 const char *buf, size_t count, 212 enum transparent_hugepage_flag flag) 213 { 214 unsigned long value; 215 int ret; 216 217 ret = kstrtoul(buf, 10, &value); 218 if (ret < 0) 219 return ret; 220 if (value > 1) 221 return -EINVAL; 222 223 if (value) 224 set_bit(flag, &transparent_hugepage_flags); 225 else 226 clear_bit(flag, &transparent_hugepage_flags); 227 228 return count; 229 } 230 231 static ssize_t defrag_show(struct kobject *kobj, 232 struct kobj_attribute *attr, char *buf) 233 { 234 if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags)) 235 return sprintf(buf, "[always] defer defer+madvise madvise never\n"); 236 if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags)) 237 return sprintf(buf, "always [defer] defer+madvise madvise never\n"); 238 if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags)) 239 return sprintf(buf, "always defer [defer+madvise] madvise never\n"); 240 if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags)) 241 return sprintf(buf, "always defer defer+madvise [madvise] never\n"); 242 return sprintf(buf, "always defer defer+madvise madvise [never]\n"); 243 } 244 245 static ssize_t defrag_store(struct kobject *kobj, 246 struct kobj_attribute *attr, 247 const char *buf, size_t count) 248 { 249 if (!memcmp("always", buf, 250 min(sizeof("always")-1, count))) { 251 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags); 252 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags); 253 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags); 254 set_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags); 255 } else if (!memcmp("defer+madvise", buf, 256 min(sizeof("defer+madvise")-1, count))) { 257 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags); 258 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags); 259 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags); 260 set_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags); 261 } else if (!memcmp("defer", buf, 262 min(sizeof("defer")-1, count))) { 263 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags); 264 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags); 265 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags); 266 set_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags); 267 } else if (!memcmp("madvise", buf, 268 min(sizeof("madvise")-1, count))) { 269 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags); 270 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags); 271 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags); 272 set_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags); 273 } else if (!memcmp("never", buf, 274 min(sizeof("never")-1, count))) { 275 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags); 276 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags); 277 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags); 278 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags); 279 } else 280 return -EINVAL; 281 282 return count; 283 } 284 static struct kobj_attribute defrag_attr = 285 __ATTR(defrag, 0644, defrag_show, defrag_store); 286 287 static ssize_t use_zero_page_show(struct kobject *kobj, 288 struct kobj_attribute *attr, char *buf) 289 { 290 return single_hugepage_flag_show(kobj, attr, buf, 291 TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG); 292 } 293 static ssize_t use_zero_page_store(struct kobject *kobj, 294 struct kobj_attribute *attr, const char *buf, size_t count) 295 { 296 return single_hugepage_flag_store(kobj, attr, buf, count, 297 TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG); 298 } 299 static struct kobj_attribute use_zero_page_attr = 300 __ATTR(use_zero_page, 0644, use_zero_page_show, use_zero_page_store); 301 302 static ssize_t hpage_pmd_size_show(struct kobject *kobj, 303 struct kobj_attribute *attr, char *buf) 304 { 305 return sprintf(buf, "%lu\n", HPAGE_PMD_SIZE); 306 } 307 static struct kobj_attribute hpage_pmd_size_attr = 308 __ATTR_RO(hpage_pmd_size); 309 310 #ifdef CONFIG_DEBUG_VM 311 static ssize_t debug_cow_show(struct kobject *kobj, 312 struct kobj_attribute *attr, char *buf) 313 { 314 return single_hugepage_flag_show(kobj, attr, buf, 315 TRANSPARENT_HUGEPAGE_DEBUG_COW_FLAG); 316 } 317 static ssize_t debug_cow_store(struct kobject *kobj, 318 struct kobj_attribute *attr, 319 const char *buf, size_t count) 320 { 321 return single_hugepage_flag_store(kobj, attr, buf, count, 322 TRANSPARENT_HUGEPAGE_DEBUG_COW_FLAG); 323 } 324 static struct kobj_attribute debug_cow_attr = 325 __ATTR(debug_cow, 0644, debug_cow_show, debug_cow_store); 326 #endif /* CONFIG_DEBUG_VM */ 327 328 static struct attribute *hugepage_attr[] = { 329 &enabled_attr.attr, 330 &defrag_attr.attr, 331 &use_zero_page_attr.attr, 332 &hpage_pmd_size_attr.attr, 333 #if defined(CONFIG_SHMEM) && defined(CONFIG_TRANSPARENT_HUGE_PAGECACHE) 334 &shmem_enabled_attr.attr, 335 #endif 336 #ifdef CONFIG_DEBUG_VM 337 &debug_cow_attr.attr, 338 #endif 339 NULL, 340 }; 341 342 static const struct attribute_group hugepage_attr_group = { 343 .attrs = hugepage_attr, 344 }; 345 346 static int __init hugepage_init_sysfs(struct kobject **hugepage_kobj) 347 { 348 int err; 349 350 *hugepage_kobj = kobject_create_and_add("transparent_hugepage", mm_kobj); 351 if (unlikely(!*hugepage_kobj)) { 352 pr_err("failed to create transparent hugepage kobject\n"); 353 return -ENOMEM; 354 } 355 356 err = sysfs_create_group(*hugepage_kobj, &hugepage_attr_group); 357 if (err) { 358 pr_err("failed to register transparent hugepage group\n"); 359 goto delete_obj; 360 } 361 362 err = sysfs_create_group(*hugepage_kobj, &khugepaged_attr_group); 363 if (err) { 364 pr_err("failed to register transparent hugepage group\n"); 365 goto remove_hp_group; 366 } 367 368 return 0; 369 370 remove_hp_group: 371 sysfs_remove_group(*hugepage_kobj, &hugepage_attr_group); 372 delete_obj: 373 kobject_put(*hugepage_kobj); 374 return err; 375 } 376 377 static void __init hugepage_exit_sysfs(struct kobject *hugepage_kobj) 378 { 379 sysfs_remove_group(hugepage_kobj, &khugepaged_attr_group); 380 sysfs_remove_group(hugepage_kobj, &hugepage_attr_group); 381 kobject_put(hugepage_kobj); 382 } 383 #else 384 static inline int hugepage_init_sysfs(struct kobject **hugepage_kobj) 385 { 386 return 0; 387 } 388 389 static inline void hugepage_exit_sysfs(struct kobject *hugepage_kobj) 390 { 391 } 392 #endif /* CONFIG_SYSFS */ 393 394 static int __init hugepage_init(void) 395 { 396 int err; 397 struct kobject *hugepage_kobj; 398 399 if (!has_transparent_hugepage()) { 400 transparent_hugepage_flags = 0; 401 return -EINVAL; 402 } 403 404 /* 405 * hugepages can't be allocated by the buddy allocator 406 */ 407 MAYBE_BUILD_BUG_ON(HPAGE_PMD_ORDER >= MAX_ORDER); 408 /* 409 * we use page->mapping and page->index in second tail page 410 * as list_head: assuming THP order >= 2 411 */ 412 MAYBE_BUILD_BUG_ON(HPAGE_PMD_ORDER < 2); 413 414 err = hugepage_init_sysfs(&hugepage_kobj); 415 if (err) 416 goto err_sysfs; 417 418 err = khugepaged_init(); 419 if (err) 420 goto err_slab; 421 422 err = register_shrinker(&huge_zero_page_shrinker); 423 if (err) 424 goto err_hzp_shrinker; 425 err = register_shrinker(&deferred_split_shrinker); 426 if (err) 427 goto err_split_shrinker; 428 429 /* 430 * By default disable transparent hugepages on smaller systems, 431 * where the extra memory used could hurt more than TLB overhead 432 * is likely to save. The admin can still enable it through /sys. 433 */ 434 if (totalram_pages() < (512 << (20 - PAGE_SHIFT))) { 435 transparent_hugepage_flags = 0; 436 return 0; 437 } 438 439 err = start_stop_khugepaged(); 440 if (err) 441 goto err_khugepaged; 442 443 return 0; 444 err_khugepaged: 445 unregister_shrinker(&deferred_split_shrinker); 446 err_split_shrinker: 447 unregister_shrinker(&huge_zero_page_shrinker); 448 err_hzp_shrinker: 449 khugepaged_destroy(); 450 err_slab: 451 hugepage_exit_sysfs(hugepage_kobj); 452 err_sysfs: 453 return err; 454 } 455 subsys_initcall(hugepage_init); 456 457 static int __init setup_transparent_hugepage(char *str) 458 { 459 int ret = 0; 460 if (!str) 461 goto out; 462 if (!strcmp(str, "always")) { 463 set_bit(TRANSPARENT_HUGEPAGE_FLAG, 464 &transparent_hugepage_flags); 465 clear_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, 466 &transparent_hugepage_flags); 467 ret = 1; 468 } else if (!strcmp(str, "madvise")) { 469 clear_bit(TRANSPARENT_HUGEPAGE_FLAG, 470 &transparent_hugepage_flags); 471 set_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, 472 &transparent_hugepage_flags); 473 ret = 1; 474 } else if (!strcmp(str, "never")) { 475 clear_bit(TRANSPARENT_HUGEPAGE_FLAG, 476 &transparent_hugepage_flags); 477 clear_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, 478 &transparent_hugepage_flags); 479 ret = 1; 480 } 481 out: 482 if (!ret) 483 pr_warn("transparent_hugepage= cannot parse, ignored\n"); 484 return ret; 485 } 486 __setup("transparent_hugepage=", setup_transparent_hugepage); 487 488 pmd_t maybe_pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma) 489 { 490 if (likely(vma->vm_flags & VM_WRITE)) 491 pmd = pmd_mkwrite(pmd); 492 return pmd; 493 } 494 495 static inline struct list_head *page_deferred_list(struct page *page) 496 { 497 /* ->lru in the tail pages is occupied by compound_head. */ 498 return &page[2].deferred_list; 499 } 500 501 void prep_transhuge_page(struct page *page) 502 { 503 /* 504 * we use page->mapping and page->indexlru in second tail page 505 * as list_head: assuming THP order >= 2 506 */ 507 508 INIT_LIST_HEAD(page_deferred_list(page)); 509 set_compound_page_dtor(page, TRANSHUGE_PAGE_DTOR); 510 } 511 512 static unsigned long __thp_get_unmapped_area(struct file *filp, unsigned long len, 513 loff_t off, unsigned long flags, unsigned long size) 514 { 515 unsigned long addr; 516 loff_t off_end = off + len; 517 loff_t off_align = round_up(off, size); 518 unsigned long len_pad; 519 520 if (off_end <= off_align || (off_end - off_align) < size) 521 return 0; 522 523 len_pad = len + size; 524 if (len_pad < len || (off + len_pad) < off) 525 return 0; 526 527 addr = current->mm->get_unmapped_area(filp, 0, len_pad, 528 off >> PAGE_SHIFT, flags); 529 if (IS_ERR_VALUE(addr)) 530 return 0; 531 532 addr += (off - addr) & (size - 1); 533 return addr; 534 } 535 536 unsigned long thp_get_unmapped_area(struct file *filp, unsigned long addr, 537 unsigned long len, unsigned long pgoff, unsigned long flags) 538 { 539 loff_t off = (loff_t)pgoff << PAGE_SHIFT; 540 541 if (addr) 542 goto out; 543 if (!IS_DAX(filp->f_mapping->host) || !IS_ENABLED(CONFIG_FS_DAX_PMD)) 544 goto out; 545 546 addr = __thp_get_unmapped_area(filp, len, off, flags, PMD_SIZE); 547 if (addr) 548 return addr; 549 550 out: 551 return current->mm->get_unmapped_area(filp, addr, len, pgoff, flags); 552 } 553 EXPORT_SYMBOL_GPL(thp_get_unmapped_area); 554 555 static vm_fault_t __do_huge_pmd_anonymous_page(struct vm_fault *vmf, 556 struct page *page, gfp_t gfp) 557 { 558 struct vm_area_struct *vma = vmf->vma; 559 struct mem_cgroup *memcg; 560 pgtable_t pgtable; 561 unsigned long haddr = vmf->address & HPAGE_PMD_MASK; 562 vm_fault_t ret = 0; 563 564 VM_BUG_ON_PAGE(!PageCompound(page), page); 565 566 if (mem_cgroup_try_charge_delay(page, vma->vm_mm, gfp, &memcg, true)) { 567 put_page(page); 568 count_vm_event(THP_FAULT_FALLBACK); 569 return VM_FAULT_FALLBACK; 570 } 571 572 pgtable = pte_alloc_one(vma->vm_mm); 573 if (unlikely(!pgtable)) { 574 ret = VM_FAULT_OOM; 575 goto release; 576 } 577 578 clear_huge_page(page, vmf->address, HPAGE_PMD_NR); 579 /* 580 * The memory barrier inside __SetPageUptodate makes sure that 581 * clear_huge_page writes become visible before the set_pmd_at() 582 * write. 583 */ 584 __SetPageUptodate(page); 585 586 vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd); 587 if (unlikely(!pmd_none(*vmf->pmd))) { 588 goto unlock_release; 589 } else { 590 pmd_t entry; 591 592 ret = check_stable_address_space(vma->vm_mm); 593 if (ret) 594 goto unlock_release; 595 596 /* Deliver the page fault to userland */ 597 if (userfaultfd_missing(vma)) { 598 vm_fault_t ret2; 599 600 spin_unlock(vmf->ptl); 601 mem_cgroup_cancel_charge(page, memcg, true); 602 put_page(page); 603 pte_free(vma->vm_mm, pgtable); 604 ret2 = handle_userfault(vmf, VM_UFFD_MISSING); 605 VM_BUG_ON(ret2 & VM_FAULT_FALLBACK); 606 return ret2; 607 } 608 609 entry = mk_huge_pmd(page, vma->vm_page_prot); 610 entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma); 611 page_add_new_anon_rmap(page, vma, haddr, true); 612 mem_cgroup_commit_charge(page, memcg, false, true); 613 lru_cache_add_active_or_unevictable(page, vma); 614 pgtable_trans_huge_deposit(vma->vm_mm, vmf->pmd, pgtable); 615 set_pmd_at(vma->vm_mm, haddr, vmf->pmd, entry); 616 add_mm_counter(vma->vm_mm, MM_ANONPAGES, HPAGE_PMD_NR); 617 mm_inc_nr_ptes(vma->vm_mm); 618 spin_unlock(vmf->ptl); 619 count_vm_event(THP_FAULT_ALLOC); 620 count_memcg_events(memcg, THP_FAULT_ALLOC, 1); 621 } 622 623 return 0; 624 unlock_release: 625 spin_unlock(vmf->ptl); 626 release: 627 if (pgtable) 628 pte_free(vma->vm_mm, pgtable); 629 mem_cgroup_cancel_charge(page, memcg, true); 630 put_page(page); 631 return ret; 632 633 } 634 635 /* 636 * always: directly stall for all thp allocations 637 * defer: wake kswapd and fail if not immediately available 638 * defer+madvise: wake kswapd and directly stall for MADV_HUGEPAGE, otherwise 639 * fail if not immediately available 640 * madvise: directly stall for MADV_HUGEPAGE, otherwise fail if not immediately 641 * available 642 * never: never stall for any thp allocation 643 */ 644 static inline gfp_t alloc_hugepage_direct_gfpmask(struct vm_area_struct *vma) 645 { 646 const bool vma_madvised = !!(vma->vm_flags & VM_HUGEPAGE); 647 648 /* Always do synchronous compaction */ 649 if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags)) 650 return GFP_TRANSHUGE | (vma_madvised ? 0 : __GFP_NORETRY); 651 652 /* Kick kcompactd and fail quickly */ 653 if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags)) 654 return GFP_TRANSHUGE_LIGHT | __GFP_KSWAPD_RECLAIM; 655 656 /* Synchronous compaction if madvised, otherwise kick kcompactd */ 657 if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags)) 658 return GFP_TRANSHUGE_LIGHT | 659 (vma_madvised ? __GFP_DIRECT_RECLAIM : 660 __GFP_KSWAPD_RECLAIM); 661 662 /* Only do synchronous compaction if madvised */ 663 if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags)) 664 return GFP_TRANSHUGE_LIGHT | 665 (vma_madvised ? __GFP_DIRECT_RECLAIM : 0); 666 667 return GFP_TRANSHUGE_LIGHT; 668 } 669 670 /* Caller must hold page table lock. */ 671 static bool set_huge_zero_page(pgtable_t pgtable, struct mm_struct *mm, 672 struct vm_area_struct *vma, unsigned long haddr, pmd_t *pmd, 673 struct page *zero_page) 674 { 675 pmd_t entry; 676 if (!pmd_none(*pmd)) 677 return false; 678 entry = mk_pmd(zero_page, vma->vm_page_prot); 679 entry = pmd_mkhuge(entry); 680 if (pgtable) 681 pgtable_trans_huge_deposit(mm, pmd, pgtable); 682 set_pmd_at(mm, haddr, pmd, entry); 683 mm_inc_nr_ptes(mm); 684 return true; 685 } 686 687 vm_fault_t do_huge_pmd_anonymous_page(struct vm_fault *vmf) 688 { 689 struct vm_area_struct *vma = vmf->vma; 690 gfp_t gfp; 691 struct page *page; 692 unsigned long haddr = vmf->address & HPAGE_PMD_MASK; 693 694 if (haddr < vma->vm_start || haddr + HPAGE_PMD_SIZE > vma->vm_end) 695 return VM_FAULT_FALLBACK; 696 if (unlikely(anon_vma_prepare(vma))) 697 return VM_FAULT_OOM; 698 if (unlikely(khugepaged_enter(vma, vma->vm_flags))) 699 return VM_FAULT_OOM; 700 if (!(vmf->flags & FAULT_FLAG_WRITE) && 701 !mm_forbids_zeropage(vma->vm_mm) && 702 transparent_hugepage_use_zero_page()) { 703 pgtable_t pgtable; 704 struct page *zero_page; 705 bool set; 706 vm_fault_t ret; 707 pgtable = pte_alloc_one(vma->vm_mm); 708 if (unlikely(!pgtable)) 709 return VM_FAULT_OOM; 710 zero_page = mm_get_huge_zero_page(vma->vm_mm); 711 if (unlikely(!zero_page)) { 712 pte_free(vma->vm_mm, pgtable); 713 count_vm_event(THP_FAULT_FALLBACK); 714 return VM_FAULT_FALLBACK; 715 } 716 vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd); 717 ret = 0; 718 set = false; 719 if (pmd_none(*vmf->pmd)) { 720 ret = check_stable_address_space(vma->vm_mm); 721 if (ret) { 722 spin_unlock(vmf->ptl); 723 } else if (userfaultfd_missing(vma)) { 724 spin_unlock(vmf->ptl); 725 ret = handle_userfault(vmf, VM_UFFD_MISSING); 726 VM_BUG_ON(ret & VM_FAULT_FALLBACK); 727 } else { 728 set_huge_zero_page(pgtable, vma->vm_mm, vma, 729 haddr, vmf->pmd, zero_page); 730 spin_unlock(vmf->ptl); 731 set = true; 732 } 733 } else 734 spin_unlock(vmf->ptl); 735 if (!set) 736 pte_free(vma->vm_mm, pgtable); 737 return ret; 738 } 739 gfp = alloc_hugepage_direct_gfpmask(vma); 740 page = alloc_hugepage_vma(gfp, vma, haddr, HPAGE_PMD_ORDER); 741 if (unlikely(!page)) { 742 count_vm_event(THP_FAULT_FALLBACK); 743 return VM_FAULT_FALLBACK; 744 } 745 prep_transhuge_page(page); 746 return __do_huge_pmd_anonymous_page(vmf, page, gfp); 747 } 748 749 static void insert_pfn_pmd(struct vm_area_struct *vma, unsigned long addr, 750 pmd_t *pmd, pfn_t pfn, pgprot_t prot, bool write, 751 pgtable_t pgtable) 752 { 753 struct mm_struct *mm = vma->vm_mm; 754 pmd_t entry; 755 spinlock_t *ptl; 756 757 ptl = pmd_lock(mm, pmd); 758 if (!pmd_none(*pmd)) { 759 if (write) { 760 if (pmd_pfn(*pmd) != pfn_t_to_pfn(pfn)) { 761 WARN_ON_ONCE(!is_huge_zero_pmd(*pmd)); 762 goto out_unlock; 763 } 764 entry = pmd_mkyoung(*pmd); 765 entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma); 766 if (pmdp_set_access_flags(vma, addr, pmd, entry, 1)) 767 update_mmu_cache_pmd(vma, addr, pmd); 768 } 769 770 goto out_unlock; 771 } 772 773 entry = pmd_mkhuge(pfn_t_pmd(pfn, prot)); 774 if (pfn_t_devmap(pfn)) 775 entry = pmd_mkdevmap(entry); 776 if (write) { 777 entry = pmd_mkyoung(pmd_mkdirty(entry)); 778 entry = maybe_pmd_mkwrite(entry, vma); 779 } 780 781 if (pgtable) { 782 pgtable_trans_huge_deposit(mm, pmd, pgtable); 783 mm_inc_nr_ptes(mm); 784 pgtable = NULL; 785 } 786 787 set_pmd_at(mm, addr, pmd, entry); 788 update_mmu_cache_pmd(vma, addr, pmd); 789 790 out_unlock: 791 spin_unlock(ptl); 792 if (pgtable) 793 pte_free(mm, pgtable); 794 } 795 796 vm_fault_t vmf_insert_pfn_pmd(struct vm_fault *vmf, pfn_t pfn, bool write) 797 { 798 unsigned long addr = vmf->address & PMD_MASK; 799 struct vm_area_struct *vma = vmf->vma; 800 pgprot_t pgprot = vma->vm_page_prot; 801 pgtable_t pgtable = NULL; 802 803 /* 804 * If we had pmd_special, we could avoid all these restrictions, 805 * but we need to be consistent with PTEs and architectures that 806 * can't support a 'special' bit. 807 */ 808 BUG_ON(!(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) && 809 !pfn_t_devmap(pfn)); 810 BUG_ON((vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) == 811 (VM_PFNMAP|VM_MIXEDMAP)); 812 BUG_ON((vma->vm_flags & VM_PFNMAP) && is_cow_mapping(vma->vm_flags)); 813 814 if (addr < vma->vm_start || addr >= vma->vm_end) 815 return VM_FAULT_SIGBUS; 816 817 if (arch_needs_pgtable_deposit()) { 818 pgtable = pte_alloc_one(vma->vm_mm); 819 if (!pgtable) 820 return VM_FAULT_OOM; 821 } 822 823 track_pfn_insert(vma, &pgprot, pfn); 824 825 insert_pfn_pmd(vma, addr, vmf->pmd, pfn, pgprot, write, pgtable); 826 return VM_FAULT_NOPAGE; 827 } 828 EXPORT_SYMBOL_GPL(vmf_insert_pfn_pmd); 829 830 #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD 831 static pud_t maybe_pud_mkwrite(pud_t pud, struct vm_area_struct *vma) 832 { 833 if (likely(vma->vm_flags & VM_WRITE)) 834 pud = pud_mkwrite(pud); 835 return pud; 836 } 837 838 static void insert_pfn_pud(struct vm_area_struct *vma, unsigned long addr, 839 pud_t *pud, pfn_t pfn, pgprot_t prot, bool write) 840 { 841 struct mm_struct *mm = vma->vm_mm; 842 pud_t entry; 843 spinlock_t *ptl; 844 845 ptl = pud_lock(mm, pud); 846 if (!pud_none(*pud)) { 847 if (write) { 848 if (pud_pfn(*pud) != pfn_t_to_pfn(pfn)) { 849 WARN_ON_ONCE(!is_huge_zero_pud(*pud)); 850 goto out_unlock; 851 } 852 entry = pud_mkyoung(*pud); 853 entry = maybe_pud_mkwrite(pud_mkdirty(entry), vma); 854 if (pudp_set_access_flags(vma, addr, pud, entry, 1)) 855 update_mmu_cache_pud(vma, addr, pud); 856 } 857 goto out_unlock; 858 } 859 860 entry = pud_mkhuge(pfn_t_pud(pfn, prot)); 861 if (pfn_t_devmap(pfn)) 862 entry = pud_mkdevmap(entry); 863 if (write) { 864 entry = pud_mkyoung(pud_mkdirty(entry)); 865 entry = maybe_pud_mkwrite(entry, vma); 866 } 867 set_pud_at(mm, addr, pud, entry); 868 update_mmu_cache_pud(vma, addr, pud); 869 870 out_unlock: 871 spin_unlock(ptl); 872 } 873 874 vm_fault_t vmf_insert_pfn_pud(struct vm_fault *vmf, pfn_t pfn, bool write) 875 { 876 unsigned long addr = vmf->address & PUD_MASK; 877 struct vm_area_struct *vma = vmf->vma; 878 pgprot_t pgprot = vma->vm_page_prot; 879 880 /* 881 * If we had pud_special, we could avoid all these restrictions, 882 * but we need to be consistent with PTEs and architectures that 883 * can't support a 'special' bit. 884 */ 885 BUG_ON(!(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) && 886 !pfn_t_devmap(pfn)); 887 BUG_ON((vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) == 888 (VM_PFNMAP|VM_MIXEDMAP)); 889 BUG_ON((vma->vm_flags & VM_PFNMAP) && is_cow_mapping(vma->vm_flags)); 890 891 if (addr < vma->vm_start || addr >= vma->vm_end) 892 return VM_FAULT_SIGBUS; 893 894 track_pfn_insert(vma, &pgprot, pfn); 895 896 insert_pfn_pud(vma, addr, vmf->pud, pfn, pgprot, write); 897 return VM_FAULT_NOPAGE; 898 } 899 EXPORT_SYMBOL_GPL(vmf_insert_pfn_pud); 900 #endif /* CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */ 901 902 static void touch_pmd(struct vm_area_struct *vma, unsigned long addr, 903 pmd_t *pmd, int flags) 904 { 905 pmd_t _pmd; 906 907 _pmd = pmd_mkyoung(*pmd); 908 if (flags & FOLL_WRITE) 909 _pmd = pmd_mkdirty(_pmd); 910 if (pmdp_set_access_flags(vma, addr & HPAGE_PMD_MASK, 911 pmd, _pmd, flags & FOLL_WRITE)) 912 update_mmu_cache_pmd(vma, addr, pmd); 913 } 914 915 struct page *follow_devmap_pmd(struct vm_area_struct *vma, unsigned long addr, 916 pmd_t *pmd, int flags, struct dev_pagemap **pgmap) 917 { 918 unsigned long pfn = pmd_pfn(*pmd); 919 struct mm_struct *mm = vma->vm_mm; 920 struct page *page; 921 922 assert_spin_locked(pmd_lockptr(mm, pmd)); 923 924 /* 925 * When we COW a devmap PMD entry, we split it into PTEs, so we should 926 * not be in this function with `flags & FOLL_COW` set. 927 */ 928 WARN_ONCE(flags & FOLL_COW, "mm: In follow_devmap_pmd with FOLL_COW set"); 929 930 if (flags & FOLL_WRITE && !pmd_write(*pmd)) 931 return NULL; 932 933 if (pmd_present(*pmd) && pmd_devmap(*pmd)) 934 /* pass */; 935 else 936 return NULL; 937 938 if (flags & FOLL_TOUCH) 939 touch_pmd(vma, addr, pmd, flags); 940 941 /* 942 * device mapped pages can only be returned if the 943 * caller will manage the page reference count. 944 */ 945 if (!(flags & FOLL_GET)) 946 return ERR_PTR(-EEXIST); 947 948 pfn += (addr & ~PMD_MASK) >> PAGE_SHIFT; 949 *pgmap = get_dev_pagemap(pfn, *pgmap); 950 if (!*pgmap) 951 return ERR_PTR(-EFAULT); 952 page = pfn_to_page(pfn); 953 get_page(page); 954 955 return page; 956 } 957 958 int copy_huge_pmd(struct mm_struct *dst_mm, struct mm_struct *src_mm, 959 pmd_t *dst_pmd, pmd_t *src_pmd, unsigned long addr, 960 struct vm_area_struct *vma) 961 { 962 spinlock_t *dst_ptl, *src_ptl; 963 struct page *src_page; 964 pmd_t pmd; 965 pgtable_t pgtable = NULL; 966 int ret = -ENOMEM; 967 968 /* Skip if can be re-fill on fault */ 969 if (!vma_is_anonymous(vma)) 970 return 0; 971 972 pgtable = pte_alloc_one(dst_mm); 973 if (unlikely(!pgtable)) 974 goto out; 975 976 dst_ptl = pmd_lock(dst_mm, dst_pmd); 977 src_ptl = pmd_lockptr(src_mm, src_pmd); 978 spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING); 979 980 ret = -EAGAIN; 981 pmd = *src_pmd; 982 983 #ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION 984 if (unlikely(is_swap_pmd(pmd))) { 985 swp_entry_t entry = pmd_to_swp_entry(pmd); 986 987 VM_BUG_ON(!is_pmd_migration_entry(pmd)); 988 if (is_write_migration_entry(entry)) { 989 make_migration_entry_read(&entry); 990 pmd = swp_entry_to_pmd(entry); 991 if (pmd_swp_soft_dirty(*src_pmd)) 992 pmd = pmd_swp_mksoft_dirty(pmd); 993 set_pmd_at(src_mm, addr, src_pmd, pmd); 994 } 995 add_mm_counter(dst_mm, MM_ANONPAGES, HPAGE_PMD_NR); 996 mm_inc_nr_ptes(dst_mm); 997 pgtable_trans_huge_deposit(dst_mm, dst_pmd, pgtable); 998 set_pmd_at(dst_mm, addr, dst_pmd, pmd); 999 ret = 0; 1000 goto out_unlock; 1001 } 1002 #endif 1003 1004 if (unlikely(!pmd_trans_huge(pmd))) { 1005 pte_free(dst_mm, pgtable); 1006 goto out_unlock; 1007 } 1008 /* 1009 * When page table lock is held, the huge zero pmd should not be 1010 * under splitting since we don't split the page itself, only pmd to 1011 * a page table. 1012 */ 1013 if (is_huge_zero_pmd(pmd)) { 1014 struct page *zero_page; 1015 /* 1016 * get_huge_zero_page() will never allocate a new page here, 1017 * since we already have a zero page to copy. It just takes a 1018 * reference. 1019 */ 1020 zero_page = mm_get_huge_zero_page(dst_mm); 1021 set_huge_zero_page(pgtable, dst_mm, vma, addr, dst_pmd, 1022 zero_page); 1023 ret = 0; 1024 goto out_unlock; 1025 } 1026 1027 src_page = pmd_page(pmd); 1028 VM_BUG_ON_PAGE(!PageHead(src_page), src_page); 1029 get_page(src_page); 1030 page_dup_rmap(src_page, true); 1031 add_mm_counter(dst_mm, MM_ANONPAGES, HPAGE_PMD_NR); 1032 mm_inc_nr_ptes(dst_mm); 1033 pgtable_trans_huge_deposit(dst_mm, dst_pmd, pgtable); 1034 1035 pmdp_set_wrprotect(src_mm, addr, src_pmd); 1036 pmd = pmd_mkold(pmd_wrprotect(pmd)); 1037 set_pmd_at(dst_mm, addr, dst_pmd, pmd); 1038 1039 ret = 0; 1040 out_unlock: 1041 spin_unlock(src_ptl); 1042 spin_unlock(dst_ptl); 1043 out: 1044 return ret; 1045 } 1046 1047 #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD 1048 static void touch_pud(struct vm_area_struct *vma, unsigned long addr, 1049 pud_t *pud, int flags) 1050 { 1051 pud_t _pud; 1052 1053 _pud = pud_mkyoung(*pud); 1054 if (flags & FOLL_WRITE) 1055 _pud = pud_mkdirty(_pud); 1056 if (pudp_set_access_flags(vma, addr & HPAGE_PUD_MASK, 1057 pud, _pud, flags & FOLL_WRITE)) 1058 update_mmu_cache_pud(vma, addr, pud); 1059 } 1060 1061 struct page *follow_devmap_pud(struct vm_area_struct *vma, unsigned long addr, 1062 pud_t *pud, int flags, struct dev_pagemap **pgmap) 1063 { 1064 unsigned long pfn = pud_pfn(*pud); 1065 struct mm_struct *mm = vma->vm_mm; 1066 struct page *page; 1067 1068 assert_spin_locked(pud_lockptr(mm, pud)); 1069 1070 if (flags & FOLL_WRITE && !pud_write(*pud)) 1071 return NULL; 1072 1073 if (pud_present(*pud) && pud_devmap(*pud)) 1074 /* pass */; 1075 else 1076 return NULL; 1077 1078 if (flags & FOLL_TOUCH) 1079 touch_pud(vma, addr, pud, flags); 1080 1081 /* 1082 * device mapped pages can only be returned if the 1083 * caller will manage the page reference count. 1084 */ 1085 if (!(flags & FOLL_GET)) 1086 return ERR_PTR(-EEXIST); 1087 1088 pfn += (addr & ~PUD_MASK) >> PAGE_SHIFT; 1089 *pgmap = get_dev_pagemap(pfn, *pgmap); 1090 if (!*pgmap) 1091 return ERR_PTR(-EFAULT); 1092 page = pfn_to_page(pfn); 1093 get_page(page); 1094 1095 return page; 1096 } 1097 1098 int copy_huge_pud(struct mm_struct *dst_mm, struct mm_struct *src_mm, 1099 pud_t *dst_pud, pud_t *src_pud, unsigned long addr, 1100 struct vm_area_struct *vma) 1101 { 1102 spinlock_t *dst_ptl, *src_ptl; 1103 pud_t pud; 1104 int ret; 1105 1106 dst_ptl = pud_lock(dst_mm, dst_pud); 1107 src_ptl = pud_lockptr(src_mm, src_pud); 1108 spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING); 1109 1110 ret = -EAGAIN; 1111 pud = *src_pud; 1112 if (unlikely(!pud_trans_huge(pud) && !pud_devmap(pud))) 1113 goto out_unlock; 1114 1115 /* 1116 * When page table lock is held, the huge zero pud should not be 1117 * under splitting since we don't split the page itself, only pud to 1118 * a page table. 1119 */ 1120 if (is_huge_zero_pud(pud)) { 1121 /* No huge zero pud yet */ 1122 } 1123 1124 pudp_set_wrprotect(src_mm, addr, src_pud); 1125 pud = pud_mkold(pud_wrprotect(pud)); 1126 set_pud_at(dst_mm, addr, dst_pud, pud); 1127 1128 ret = 0; 1129 out_unlock: 1130 spin_unlock(src_ptl); 1131 spin_unlock(dst_ptl); 1132 return ret; 1133 } 1134 1135 void huge_pud_set_accessed(struct vm_fault *vmf, pud_t orig_pud) 1136 { 1137 pud_t entry; 1138 unsigned long haddr; 1139 bool write = vmf->flags & FAULT_FLAG_WRITE; 1140 1141 vmf->ptl = pud_lock(vmf->vma->vm_mm, vmf->pud); 1142 if (unlikely(!pud_same(*vmf->pud, orig_pud))) 1143 goto unlock; 1144 1145 entry = pud_mkyoung(orig_pud); 1146 if (write) 1147 entry = pud_mkdirty(entry); 1148 haddr = vmf->address & HPAGE_PUD_MASK; 1149 if (pudp_set_access_flags(vmf->vma, haddr, vmf->pud, entry, write)) 1150 update_mmu_cache_pud(vmf->vma, vmf->address, vmf->pud); 1151 1152 unlock: 1153 spin_unlock(vmf->ptl); 1154 } 1155 #endif /* CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */ 1156 1157 void huge_pmd_set_accessed(struct vm_fault *vmf, pmd_t orig_pmd) 1158 { 1159 pmd_t entry; 1160 unsigned long haddr; 1161 bool write = vmf->flags & FAULT_FLAG_WRITE; 1162 1163 vmf->ptl = pmd_lock(vmf->vma->vm_mm, vmf->pmd); 1164 if (unlikely(!pmd_same(*vmf->pmd, orig_pmd))) 1165 goto unlock; 1166 1167 entry = pmd_mkyoung(orig_pmd); 1168 if (write) 1169 entry = pmd_mkdirty(entry); 1170 haddr = vmf->address & HPAGE_PMD_MASK; 1171 if (pmdp_set_access_flags(vmf->vma, haddr, vmf->pmd, entry, write)) 1172 update_mmu_cache_pmd(vmf->vma, vmf->address, vmf->pmd); 1173 1174 unlock: 1175 spin_unlock(vmf->ptl); 1176 } 1177 1178 static vm_fault_t do_huge_pmd_wp_page_fallback(struct vm_fault *vmf, 1179 pmd_t orig_pmd, struct page *page) 1180 { 1181 struct vm_area_struct *vma = vmf->vma; 1182 unsigned long haddr = vmf->address & HPAGE_PMD_MASK; 1183 struct mem_cgroup *memcg; 1184 pgtable_t pgtable; 1185 pmd_t _pmd; 1186 int i; 1187 vm_fault_t ret = 0; 1188 struct page **pages; 1189 struct mmu_notifier_range range; 1190 1191 pages = kmalloc_array(HPAGE_PMD_NR, sizeof(struct page *), 1192 GFP_KERNEL); 1193 if (unlikely(!pages)) { 1194 ret |= VM_FAULT_OOM; 1195 goto out; 1196 } 1197 1198 for (i = 0; i < HPAGE_PMD_NR; i++) { 1199 pages[i] = alloc_page_vma_node(GFP_HIGHUSER_MOVABLE, vma, 1200 vmf->address, page_to_nid(page)); 1201 if (unlikely(!pages[i] || 1202 mem_cgroup_try_charge_delay(pages[i], vma->vm_mm, 1203 GFP_KERNEL, &memcg, false))) { 1204 if (pages[i]) 1205 put_page(pages[i]); 1206 while (--i >= 0) { 1207 memcg = (void *)page_private(pages[i]); 1208 set_page_private(pages[i], 0); 1209 mem_cgroup_cancel_charge(pages[i], memcg, 1210 false); 1211 put_page(pages[i]); 1212 } 1213 kfree(pages); 1214 ret |= VM_FAULT_OOM; 1215 goto out; 1216 } 1217 set_page_private(pages[i], (unsigned long)memcg); 1218 } 1219 1220 for (i = 0; i < HPAGE_PMD_NR; i++) { 1221 copy_user_highpage(pages[i], page + i, 1222 haddr + PAGE_SIZE * i, vma); 1223 __SetPageUptodate(pages[i]); 1224 cond_resched(); 1225 } 1226 1227 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, vma->vm_mm, 1228 haddr, haddr + HPAGE_PMD_SIZE); 1229 mmu_notifier_invalidate_range_start(&range); 1230 1231 vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd); 1232 if (unlikely(!pmd_same(*vmf->pmd, orig_pmd))) 1233 goto out_free_pages; 1234 VM_BUG_ON_PAGE(!PageHead(page), page); 1235 1236 /* 1237 * Leave pmd empty until pte is filled note we must notify here as 1238 * concurrent CPU thread might write to new page before the call to 1239 * mmu_notifier_invalidate_range_end() happens which can lead to a 1240 * device seeing memory write in different order than CPU. 1241 * 1242 * See Documentation/vm/mmu_notifier.rst 1243 */ 1244 pmdp_huge_clear_flush_notify(vma, haddr, vmf->pmd); 1245 1246 pgtable = pgtable_trans_huge_withdraw(vma->vm_mm, vmf->pmd); 1247 pmd_populate(vma->vm_mm, &_pmd, pgtable); 1248 1249 for (i = 0; i < HPAGE_PMD_NR; i++, haddr += PAGE_SIZE) { 1250 pte_t entry; 1251 entry = mk_pte(pages[i], vma->vm_page_prot); 1252 entry = maybe_mkwrite(pte_mkdirty(entry), vma); 1253 memcg = (void *)page_private(pages[i]); 1254 set_page_private(pages[i], 0); 1255 page_add_new_anon_rmap(pages[i], vmf->vma, haddr, false); 1256 mem_cgroup_commit_charge(pages[i], memcg, false, false); 1257 lru_cache_add_active_or_unevictable(pages[i], vma); 1258 vmf->pte = pte_offset_map(&_pmd, haddr); 1259 VM_BUG_ON(!pte_none(*vmf->pte)); 1260 set_pte_at(vma->vm_mm, haddr, vmf->pte, entry); 1261 pte_unmap(vmf->pte); 1262 } 1263 kfree(pages); 1264 1265 smp_wmb(); /* make pte visible before pmd */ 1266 pmd_populate(vma->vm_mm, vmf->pmd, pgtable); 1267 page_remove_rmap(page, true); 1268 spin_unlock(vmf->ptl); 1269 1270 /* 1271 * No need to double call mmu_notifier->invalidate_range() callback as 1272 * the above pmdp_huge_clear_flush_notify() did already call it. 1273 */ 1274 mmu_notifier_invalidate_range_only_end(&range); 1275 1276 ret |= VM_FAULT_WRITE; 1277 put_page(page); 1278 1279 out: 1280 return ret; 1281 1282 out_free_pages: 1283 spin_unlock(vmf->ptl); 1284 mmu_notifier_invalidate_range_end(&range); 1285 for (i = 0; i < HPAGE_PMD_NR; i++) { 1286 memcg = (void *)page_private(pages[i]); 1287 set_page_private(pages[i], 0); 1288 mem_cgroup_cancel_charge(pages[i], memcg, false); 1289 put_page(pages[i]); 1290 } 1291 kfree(pages); 1292 goto out; 1293 } 1294 1295 vm_fault_t do_huge_pmd_wp_page(struct vm_fault *vmf, pmd_t orig_pmd) 1296 { 1297 struct vm_area_struct *vma = vmf->vma; 1298 struct page *page = NULL, *new_page; 1299 struct mem_cgroup *memcg; 1300 unsigned long haddr = vmf->address & HPAGE_PMD_MASK; 1301 struct mmu_notifier_range range; 1302 gfp_t huge_gfp; /* for allocation and charge */ 1303 vm_fault_t ret = 0; 1304 1305 vmf->ptl = pmd_lockptr(vma->vm_mm, vmf->pmd); 1306 VM_BUG_ON_VMA(!vma->anon_vma, vma); 1307 if (is_huge_zero_pmd(orig_pmd)) 1308 goto alloc; 1309 spin_lock(vmf->ptl); 1310 if (unlikely(!pmd_same(*vmf->pmd, orig_pmd))) 1311 goto out_unlock; 1312 1313 page = pmd_page(orig_pmd); 1314 VM_BUG_ON_PAGE(!PageCompound(page) || !PageHead(page), page); 1315 /* 1316 * We can only reuse the page if nobody else maps the huge page or it's 1317 * part. 1318 */ 1319 if (!trylock_page(page)) { 1320 get_page(page); 1321 spin_unlock(vmf->ptl); 1322 lock_page(page); 1323 spin_lock(vmf->ptl); 1324 if (unlikely(!pmd_same(*vmf->pmd, orig_pmd))) { 1325 unlock_page(page); 1326 put_page(page); 1327 goto out_unlock; 1328 } 1329 put_page(page); 1330 } 1331 if (reuse_swap_page(page, NULL)) { 1332 pmd_t entry; 1333 entry = pmd_mkyoung(orig_pmd); 1334 entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma); 1335 if (pmdp_set_access_flags(vma, haddr, vmf->pmd, entry, 1)) 1336 update_mmu_cache_pmd(vma, vmf->address, vmf->pmd); 1337 ret |= VM_FAULT_WRITE; 1338 unlock_page(page); 1339 goto out_unlock; 1340 } 1341 unlock_page(page); 1342 get_page(page); 1343 spin_unlock(vmf->ptl); 1344 alloc: 1345 if (__transparent_hugepage_enabled(vma) && 1346 !transparent_hugepage_debug_cow()) { 1347 huge_gfp = alloc_hugepage_direct_gfpmask(vma); 1348 new_page = alloc_hugepage_vma(huge_gfp, vma, haddr, HPAGE_PMD_ORDER); 1349 } else 1350 new_page = NULL; 1351 1352 if (likely(new_page)) { 1353 prep_transhuge_page(new_page); 1354 } else { 1355 if (!page) { 1356 split_huge_pmd(vma, vmf->pmd, vmf->address); 1357 ret |= VM_FAULT_FALLBACK; 1358 } else { 1359 ret = do_huge_pmd_wp_page_fallback(vmf, orig_pmd, page); 1360 if (ret & VM_FAULT_OOM) { 1361 split_huge_pmd(vma, vmf->pmd, vmf->address); 1362 ret |= VM_FAULT_FALLBACK; 1363 } 1364 put_page(page); 1365 } 1366 count_vm_event(THP_FAULT_FALLBACK); 1367 goto out; 1368 } 1369 1370 if (unlikely(mem_cgroup_try_charge_delay(new_page, vma->vm_mm, 1371 huge_gfp, &memcg, true))) { 1372 put_page(new_page); 1373 split_huge_pmd(vma, vmf->pmd, vmf->address); 1374 if (page) 1375 put_page(page); 1376 ret |= VM_FAULT_FALLBACK; 1377 count_vm_event(THP_FAULT_FALLBACK); 1378 goto out; 1379 } 1380 1381 count_vm_event(THP_FAULT_ALLOC); 1382 count_memcg_events(memcg, THP_FAULT_ALLOC, 1); 1383 1384 if (!page) 1385 clear_huge_page(new_page, vmf->address, HPAGE_PMD_NR); 1386 else 1387 copy_user_huge_page(new_page, page, vmf->address, 1388 vma, HPAGE_PMD_NR); 1389 __SetPageUptodate(new_page); 1390 1391 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, vma->vm_mm, 1392 haddr, haddr + HPAGE_PMD_SIZE); 1393 mmu_notifier_invalidate_range_start(&range); 1394 1395 spin_lock(vmf->ptl); 1396 if (page) 1397 put_page(page); 1398 if (unlikely(!pmd_same(*vmf->pmd, orig_pmd))) { 1399 spin_unlock(vmf->ptl); 1400 mem_cgroup_cancel_charge(new_page, memcg, true); 1401 put_page(new_page); 1402 goto out_mn; 1403 } else { 1404 pmd_t entry; 1405 entry = mk_huge_pmd(new_page, vma->vm_page_prot); 1406 entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma); 1407 pmdp_huge_clear_flush_notify(vma, haddr, vmf->pmd); 1408 page_add_new_anon_rmap(new_page, vma, haddr, true); 1409 mem_cgroup_commit_charge(new_page, memcg, false, true); 1410 lru_cache_add_active_or_unevictable(new_page, vma); 1411 set_pmd_at(vma->vm_mm, haddr, vmf->pmd, entry); 1412 update_mmu_cache_pmd(vma, vmf->address, vmf->pmd); 1413 if (!page) { 1414 add_mm_counter(vma->vm_mm, MM_ANONPAGES, HPAGE_PMD_NR); 1415 } else { 1416 VM_BUG_ON_PAGE(!PageHead(page), page); 1417 page_remove_rmap(page, true); 1418 put_page(page); 1419 } 1420 ret |= VM_FAULT_WRITE; 1421 } 1422 spin_unlock(vmf->ptl); 1423 out_mn: 1424 /* 1425 * No need to double call mmu_notifier->invalidate_range() callback as 1426 * the above pmdp_huge_clear_flush_notify() did already call it. 1427 */ 1428 mmu_notifier_invalidate_range_only_end(&range); 1429 out: 1430 return ret; 1431 out_unlock: 1432 spin_unlock(vmf->ptl); 1433 return ret; 1434 } 1435 1436 /* 1437 * FOLL_FORCE can write to even unwritable pmd's, but only 1438 * after we've gone through a COW cycle and they are dirty. 1439 */ 1440 static inline bool can_follow_write_pmd(pmd_t pmd, unsigned int flags) 1441 { 1442 return pmd_write(pmd) || 1443 ((flags & FOLL_FORCE) && (flags & FOLL_COW) && pmd_dirty(pmd)); 1444 } 1445 1446 struct page *follow_trans_huge_pmd(struct vm_area_struct *vma, 1447 unsigned long addr, 1448 pmd_t *pmd, 1449 unsigned int flags) 1450 { 1451 struct mm_struct *mm = vma->vm_mm; 1452 struct page *page = NULL; 1453 1454 assert_spin_locked(pmd_lockptr(mm, pmd)); 1455 1456 if (flags & FOLL_WRITE && !can_follow_write_pmd(*pmd, flags)) 1457 goto out; 1458 1459 /* Avoid dumping huge zero page */ 1460 if ((flags & FOLL_DUMP) && is_huge_zero_pmd(*pmd)) 1461 return ERR_PTR(-EFAULT); 1462 1463 /* Full NUMA hinting faults to serialise migration in fault paths */ 1464 if ((flags & FOLL_NUMA) && pmd_protnone(*pmd)) 1465 goto out; 1466 1467 page = pmd_page(*pmd); 1468 VM_BUG_ON_PAGE(!PageHead(page) && !is_zone_device_page(page), page); 1469 if (flags & FOLL_TOUCH) 1470 touch_pmd(vma, addr, pmd, flags); 1471 if ((flags & FOLL_MLOCK) && (vma->vm_flags & VM_LOCKED)) { 1472 /* 1473 * We don't mlock() pte-mapped THPs. This way we can avoid 1474 * leaking mlocked pages into non-VM_LOCKED VMAs. 1475 * 1476 * For anon THP: 1477 * 1478 * In most cases the pmd is the only mapping of the page as we 1479 * break COW for the mlock() -- see gup_flags |= FOLL_WRITE for 1480 * writable private mappings in populate_vma_page_range(). 1481 * 1482 * The only scenario when we have the page shared here is if we 1483 * mlocking read-only mapping shared over fork(). We skip 1484 * mlocking such pages. 1485 * 1486 * For file THP: 1487 * 1488 * We can expect PageDoubleMap() to be stable under page lock: 1489 * for file pages we set it in page_add_file_rmap(), which 1490 * requires page to be locked. 1491 */ 1492 1493 if (PageAnon(page) && compound_mapcount(page) != 1) 1494 goto skip_mlock; 1495 if (PageDoubleMap(page) || !page->mapping) 1496 goto skip_mlock; 1497 if (!trylock_page(page)) 1498 goto skip_mlock; 1499 lru_add_drain(); 1500 if (page->mapping && !PageDoubleMap(page)) 1501 mlock_vma_page(page); 1502 unlock_page(page); 1503 } 1504 skip_mlock: 1505 page += (addr & ~HPAGE_PMD_MASK) >> PAGE_SHIFT; 1506 VM_BUG_ON_PAGE(!PageCompound(page) && !is_zone_device_page(page), page); 1507 if (flags & FOLL_GET) 1508 get_page(page); 1509 1510 out: 1511 return page; 1512 } 1513 1514 /* NUMA hinting page fault entry point for trans huge pmds */ 1515 vm_fault_t do_huge_pmd_numa_page(struct vm_fault *vmf, pmd_t pmd) 1516 { 1517 struct vm_area_struct *vma = vmf->vma; 1518 struct anon_vma *anon_vma = NULL; 1519 struct page *page; 1520 unsigned long haddr = vmf->address & HPAGE_PMD_MASK; 1521 int page_nid = NUMA_NO_NODE, this_nid = numa_node_id(); 1522 int target_nid, last_cpupid = -1; 1523 bool page_locked; 1524 bool migrated = false; 1525 bool was_writable; 1526 int flags = 0; 1527 1528 vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd); 1529 if (unlikely(!pmd_same(pmd, *vmf->pmd))) 1530 goto out_unlock; 1531 1532 /* 1533 * If there are potential migrations, wait for completion and retry 1534 * without disrupting NUMA hinting information. Do not relock and 1535 * check_same as the page may no longer be mapped. 1536 */ 1537 if (unlikely(pmd_trans_migrating(*vmf->pmd))) { 1538 page = pmd_page(*vmf->pmd); 1539 if (!get_page_unless_zero(page)) 1540 goto out_unlock; 1541 spin_unlock(vmf->ptl); 1542 put_and_wait_on_page_locked(page); 1543 goto out; 1544 } 1545 1546 page = pmd_page(pmd); 1547 BUG_ON(is_huge_zero_page(page)); 1548 page_nid = page_to_nid(page); 1549 last_cpupid = page_cpupid_last(page); 1550 count_vm_numa_event(NUMA_HINT_FAULTS); 1551 if (page_nid == this_nid) { 1552 count_vm_numa_event(NUMA_HINT_FAULTS_LOCAL); 1553 flags |= TNF_FAULT_LOCAL; 1554 } 1555 1556 /* See similar comment in do_numa_page for explanation */ 1557 if (!pmd_savedwrite(pmd)) 1558 flags |= TNF_NO_GROUP; 1559 1560 /* 1561 * Acquire the page lock to serialise THP migrations but avoid dropping 1562 * page_table_lock if at all possible 1563 */ 1564 page_locked = trylock_page(page); 1565 target_nid = mpol_misplaced(page, vma, haddr); 1566 if (target_nid == NUMA_NO_NODE) { 1567 /* If the page was locked, there are no parallel migrations */ 1568 if (page_locked) 1569 goto clear_pmdnuma; 1570 } 1571 1572 /* Migration could have started since the pmd_trans_migrating check */ 1573 if (!page_locked) { 1574 page_nid = NUMA_NO_NODE; 1575 if (!get_page_unless_zero(page)) 1576 goto out_unlock; 1577 spin_unlock(vmf->ptl); 1578 put_and_wait_on_page_locked(page); 1579 goto out; 1580 } 1581 1582 /* 1583 * Page is misplaced. Page lock serialises migrations. Acquire anon_vma 1584 * to serialises splits 1585 */ 1586 get_page(page); 1587 spin_unlock(vmf->ptl); 1588 anon_vma = page_lock_anon_vma_read(page); 1589 1590 /* Confirm the PMD did not change while page_table_lock was released */ 1591 spin_lock(vmf->ptl); 1592 if (unlikely(!pmd_same(pmd, *vmf->pmd))) { 1593 unlock_page(page); 1594 put_page(page); 1595 page_nid = NUMA_NO_NODE; 1596 goto out_unlock; 1597 } 1598 1599 /* Bail if we fail to protect against THP splits for any reason */ 1600 if (unlikely(!anon_vma)) { 1601 put_page(page); 1602 page_nid = NUMA_NO_NODE; 1603 goto clear_pmdnuma; 1604 } 1605 1606 /* 1607 * Since we took the NUMA fault, we must have observed the !accessible 1608 * bit. Make sure all other CPUs agree with that, to avoid them 1609 * modifying the page we're about to migrate. 1610 * 1611 * Must be done under PTL such that we'll observe the relevant 1612 * inc_tlb_flush_pending(). 1613 * 1614 * We are not sure a pending tlb flush here is for a huge page 1615 * mapping or not. Hence use the tlb range variant 1616 */ 1617 if (mm_tlb_flush_pending(vma->vm_mm)) { 1618 flush_tlb_range(vma, haddr, haddr + HPAGE_PMD_SIZE); 1619 /* 1620 * change_huge_pmd() released the pmd lock before 1621 * invalidating the secondary MMUs sharing the primary 1622 * MMU pagetables (with ->invalidate_range()). The 1623 * mmu_notifier_invalidate_range_end() (which 1624 * internally calls ->invalidate_range()) in 1625 * change_pmd_range() will run after us, so we can't 1626 * rely on it here and we need an explicit invalidate. 1627 */ 1628 mmu_notifier_invalidate_range(vma->vm_mm, haddr, 1629 haddr + HPAGE_PMD_SIZE); 1630 } 1631 1632 /* 1633 * Migrate the THP to the requested node, returns with page unlocked 1634 * and access rights restored. 1635 */ 1636 spin_unlock(vmf->ptl); 1637 1638 migrated = migrate_misplaced_transhuge_page(vma->vm_mm, vma, 1639 vmf->pmd, pmd, vmf->address, page, target_nid); 1640 if (migrated) { 1641 flags |= TNF_MIGRATED; 1642 page_nid = target_nid; 1643 } else 1644 flags |= TNF_MIGRATE_FAIL; 1645 1646 goto out; 1647 clear_pmdnuma: 1648 BUG_ON(!PageLocked(page)); 1649 was_writable = pmd_savedwrite(pmd); 1650 pmd = pmd_modify(pmd, vma->vm_page_prot); 1651 pmd = pmd_mkyoung(pmd); 1652 if (was_writable) 1653 pmd = pmd_mkwrite(pmd); 1654 set_pmd_at(vma->vm_mm, haddr, vmf->pmd, pmd); 1655 update_mmu_cache_pmd(vma, vmf->address, vmf->pmd); 1656 unlock_page(page); 1657 out_unlock: 1658 spin_unlock(vmf->ptl); 1659 1660 out: 1661 if (anon_vma) 1662 page_unlock_anon_vma_read(anon_vma); 1663 1664 if (page_nid != NUMA_NO_NODE) 1665 task_numa_fault(last_cpupid, page_nid, HPAGE_PMD_NR, 1666 flags); 1667 1668 return 0; 1669 } 1670 1671 /* 1672 * Return true if we do MADV_FREE successfully on entire pmd page. 1673 * Otherwise, return false. 1674 */ 1675 bool madvise_free_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, 1676 pmd_t *pmd, unsigned long addr, unsigned long next) 1677 { 1678 spinlock_t *ptl; 1679 pmd_t orig_pmd; 1680 struct page *page; 1681 struct mm_struct *mm = tlb->mm; 1682 bool ret = false; 1683 1684 tlb_change_page_size(tlb, HPAGE_PMD_SIZE); 1685 1686 ptl = pmd_trans_huge_lock(pmd, vma); 1687 if (!ptl) 1688 goto out_unlocked; 1689 1690 orig_pmd = *pmd; 1691 if (is_huge_zero_pmd(orig_pmd)) 1692 goto out; 1693 1694 if (unlikely(!pmd_present(orig_pmd))) { 1695 VM_BUG_ON(thp_migration_supported() && 1696 !is_pmd_migration_entry(orig_pmd)); 1697 goto out; 1698 } 1699 1700 page = pmd_page(orig_pmd); 1701 /* 1702 * If other processes are mapping this page, we couldn't discard 1703 * the page unless they all do MADV_FREE so let's skip the page. 1704 */ 1705 if (page_mapcount(page) != 1) 1706 goto out; 1707 1708 if (!trylock_page(page)) 1709 goto out; 1710 1711 /* 1712 * If user want to discard part-pages of THP, split it so MADV_FREE 1713 * will deactivate only them. 1714 */ 1715 if (next - addr != HPAGE_PMD_SIZE) { 1716 get_page(page); 1717 spin_unlock(ptl); 1718 split_huge_page(page); 1719 unlock_page(page); 1720 put_page(page); 1721 goto out_unlocked; 1722 } 1723 1724 if (PageDirty(page)) 1725 ClearPageDirty(page); 1726 unlock_page(page); 1727 1728 if (pmd_young(orig_pmd) || pmd_dirty(orig_pmd)) { 1729 pmdp_invalidate(vma, addr, pmd); 1730 orig_pmd = pmd_mkold(orig_pmd); 1731 orig_pmd = pmd_mkclean(orig_pmd); 1732 1733 set_pmd_at(mm, addr, pmd, orig_pmd); 1734 tlb_remove_pmd_tlb_entry(tlb, pmd, addr); 1735 } 1736 1737 mark_page_lazyfree(page); 1738 ret = true; 1739 out: 1740 spin_unlock(ptl); 1741 out_unlocked: 1742 return ret; 1743 } 1744 1745 static inline void zap_deposited_table(struct mm_struct *mm, pmd_t *pmd) 1746 { 1747 pgtable_t pgtable; 1748 1749 pgtable = pgtable_trans_huge_withdraw(mm, pmd); 1750 pte_free(mm, pgtable); 1751 mm_dec_nr_ptes(mm); 1752 } 1753 1754 int zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, 1755 pmd_t *pmd, unsigned long addr) 1756 { 1757 pmd_t orig_pmd; 1758 spinlock_t *ptl; 1759 1760 tlb_change_page_size(tlb, HPAGE_PMD_SIZE); 1761 1762 ptl = __pmd_trans_huge_lock(pmd, vma); 1763 if (!ptl) 1764 return 0; 1765 /* 1766 * For architectures like ppc64 we look at deposited pgtable 1767 * when calling pmdp_huge_get_and_clear. So do the 1768 * pgtable_trans_huge_withdraw after finishing pmdp related 1769 * operations. 1770 */ 1771 orig_pmd = pmdp_huge_get_and_clear_full(tlb->mm, addr, pmd, 1772 tlb->fullmm); 1773 tlb_remove_pmd_tlb_entry(tlb, pmd, addr); 1774 if (vma_is_dax(vma)) { 1775 if (arch_needs_pgtable_deposit()) 1776 zap_deposited_table(tlb->mm, pmd); 1777 spin_unlock(ptl); 1778 if (is_huge_zero_pmd(orig_pmd)) 1779 tlb_remove_page_size(tlb, pmd_page(orig_pmd), HPAGE_PMD_SIZE); 1780 } else if (is_huge_zero_pmd(orig_pmd)) { 1781 zap_deposited_table(tlb->mm, pmd); 1782 spin_unlock(ptl); 1783 tlb_remove_page_size(tlb, pmd_page(orig_pmd), HPAGE_PMD_SIZE); 1784 } else { 1785 struct page *page = NULL; 1786 int flush_needed = 1; 1787 1788 if (pmd_present(orig_pmd)) { 1789 page = pmd_page(orig_pmd); 1790 page_remove_rmap(page, true); 1791 VM_BUG_ON_PAGE(page_mapcount(page) < 0, page); 1792 VM_BUG_ON_PAGE(!PageHead(page), page); 1793 } else if (thp_migration_supported()) { 1794 swp_entry_t entry; 1795 1796 VM_BUG_ON(!is_pmd_migration_entry(orig_pmd)); 1797 entry = pmd_to_swp_entry(orig_pmd); 1798 page = pfn_to_page(swp_offset(entry)); 1799 flush_needed = 0; 1800 } else 1801 WARN_ONCE(1, "Non present huge pmd without pmd migration enabled!"); 1802 1803 if (PageAnon(page)) { 1804 zap_deposited_table(tlb->mm, pmd); 1805 add_mm_counter(tlb->mm, MM_ANONPAGES, -HPAGE_PMD_NR); 1806 } else { 1807 if (arch_needs_pgtable_deposit()) 1808 zap_deposited_table(tlb->mm, pmd); 1809 add_mm_counter(tlb->mm, mm_counter_file(page), -HPAGE_PMD_NR); 1810 } 1811 1812 spin_unlock(ptl); 1813 if (flush_needed) 1814 tlb_remove_page_size(tlb, page, HPAGE_PMD_SIZE); 1815 } 1816 return 1; 1817 } 1818 1819 #ifndef pmd_move_must_withdraw 1820 static inline int pmd_move_must_withdraw(spinlock_t *new_pmd_ptl, 1821 spinlock_t *old_pmd_ptl, 1822 struct vm_area_struct *vma) 1823 { 1824 /* 1825 * With split pmd lock we also need to move preallocated 1826 * PTE page table if new_pmd is on different PMD page table. 1827 * 1828 * We also don't deposit and withdraw tables for file pages. 1829 */ 1830 return (new_pmd_ptl != old_pmd_ptl) && vma_is_anonymous(vma); 1831 } 1832 #endif 1833 1834 static pmd_t move_soft_dirty_pmd(pmd_t pmd) 1835 { 1836 #ifdef CONFIG_MEM_SOFT_DIRTY 1837 if (unlikely(is_pmd_migration_entry(pmd))) 1838 pmd = pmd_swp_mksoft_dirty(pmd); 1839 else if (pmd_present(pmd)) 1840 pmd = pmd_mksoft_dirty(pmd); 1841 #endif 1842 return pmd; 1843 } 1844 1845 bool move_huge_pmd(struct vm_area_struct *vma, unsigned long old_addr, 1846 unsigned long new_addr, unsigned long old_end, 1847 pmd_t *old_pmd, pmd_t *new_pmd) 1848 { 1849 spinlock_t *old_ptl, *new_ptl; 1850 pmd_t pmd; 1851 struct mm_struct *mm = vma->vm_mm; 1852 bool force_flush = false; 1853 1854 if ((old_addr & ~HPAGE_PMD_MASK) || 1855 (new_addr & ~HPAGE_PMD_MASK) || 1856 old_end - old_addr < HPAGE_PMD_SIZE) 1857 return false; 1858 1859 /* 1860 * The destination pmd shouldn't be established, free_pgtables() 1861 * should have release it. 1862 */ 1863 if (WARN_ON(!pmd_none(*new_pmd))) { 1864 VM_BUG_ON(pmd_trans_huge(*new_pmd)); 1865 return false; 1866 } 1867 1868 /* 1869 * We don't have to worry about the ordering of src and dst 1870 * ptlocks because exclusive mmap_sem prevents deadlock. 1871 */ 1872 old_ptl = __pmd_trans_huge_lock(old_pmd, vma); 1873 if (old_ptl) { 1874 new_ptl = pmd_lockptr(mm, new_pmd); 1875 if (new_ptl != old_ptl) 1876 spin_lock_nested(new_ptl, SINGLE_DEPTH_NESTING); 1877 pmd = pmdp_huge_get_and_clear(mm, old_addr, old_pmd); 1878 if (pmd_present(pmd)) 1879 force_flush = true; 1880 VM_BUG_ON(!pmd_none(*new_pmd)); 1881 1882 if (pmd_move_must_withdraw(new_ptl, old_ptl, vma)) { 1883 pgtable_t pgtable; 1884 pgtable = pgtable_trans_huge_withdraw(mm, old_pmd); 1885 pgtable_trans_huge_deposit(mm, new_pmd, pgtable); 1886 } 1887 pmd = move_soft_dirty_pmd(pmd); 1888 set_pmd_at(mm, new_addr, new_pmd, pmd); 1889 if (force_flush) 1890 flush_tlb_range(vma, old_addr, old_addr + PMD_SIZE); 1891 if (new_ptl != old_ptl) 1892 spin_unlock(new_ptl); 1893 spin_unlock(old_ptl); 1894 return true; 1895 } 1896 return false; 1897 } 1898 1899 /* 1900 * Returns 1901 * - 0 if PMD could not be locked 1902 * - 1 if PMD was locked but protections unchange and TLB flush unnecessary 1903 * - HPAGE_PMD_NR is protections changed and TLB flush necessary 1904 */ 1905 int change_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd, 1906 unsigned long addr, pgprot_t newprot, int prot_numa) 1907 { 1908 struct mm_struct *mm = vma->vm_mm; 1909 spinlock_t *ptl; 1910 pmd_t entry; 1911 bool preserve_write; 1912 int ret; 1913 1914 ptl = __pmd_trans_huge_lock(pmd, vma); 1915 if (!ptl) 1916 return 0; 1917 1918 preserve_write = prot_numa && pmd_write(*pmd); 1919 ret = 1; 1920 1921 #ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION 1922 if (is_swap_pmd(*pmd)) { 1923 swp_entry_t entry = pmd_to_swp_entry(*pmd); 1924 1925 VM_BUG_ON(!is_pmd_migration_entry(*pmd)); 1926 if (is_write_migration_entry(entry)) { 1927 pmd_t newpmd; 1928 /* 1929 * A protection check is difficult so 1930 * just be safe and disable write 1931 */ 1932 make_migration_entry_read(&entry); 1933 newpmd = swp_entry_to_pmd(entry); 1934 if (pmd_swp_soft_dirty(*pmd)) 1935 newpmd = pmd_swp_mksoft_dirty(newpmd); 1936 set_pmd_at(mm, addr, pmd, newpmd); 1937 } 1938 goto unlock; 1939 } 1940 #endif 1941 1942 /* 1943 * Avoid trapping faults against the zero page. The read-only 1944 * data is likely to be read-cached on the local CPU and 1945 * local/remote hits to the zero page are not interesting. 1946 */ 1947 if (prot_numa && is_huge_zero_pmd(*pmd)) 1948 goto unlock; 1949 1950 if (prot_numa && pmd_protnone(*pmd)) 1951 goto unlock; 1952 1953 /* 1954 * In case prot_numa, we are under down_read(mmap_sem). It's critical 1955 * to not clear pmd intermittently to avoid race with MADV_DONTNEED 1956 * which is also under down_read(mmap_sem): 1957 * 1958 * CPU0: CPU1: 1959 * change_huge_pmd(prot_numa=1) 1960 * pmdp_huge_get_and_clear_notify() 1961 * madvise_dontneed() 1962 * zap_pmd_range() 1963 * pmd_trans_huge(*pmd) == 0 (without ptl) 1964 * // skip the pmd 1965 * set_pmd_at(); 1966 * // pmd is re-established 1967 * 1968 * The race makes MADV_DONTNEED miss the huge pmd and don't clear it 1969 * which may break userspace. 1970 * 1971 * pmdp_invalidate() is required to make sure we don't miss 1972 * dirty/young flags set by hardware. 1973 */ 1974 entry = pmdp_invalidate(vma, addr, pmd); 1975 1976 entry = pmd_modify(entry, newprot); 1977 if (preserve_write) 1978 entry = pmd_mk_savedwrite(entry); 1979 ret = HPAGE_PMD_NR; 1980 set_pmd_at(mm, addr, pmd, entry); 1981 BUG_ON(vma_is_anonymous(vma) && !preserve_write && pmd_write(entry)); 1982 unlock: 1983 spin_unlock(ptl); 1984 return ret; 1985 } 1986 1987 /* 1988 * Returns page table lock pointer if a given pmd maps a thp, NULL otherwise. 1989 * 1990 * Note that if it returns page table lock pointer, this routine returns without 1991 * unlocking page table lock. So callers must unlock it. 1992 */ 1993 spinlock_t *__pmd_trans_huge_lock(pmd_t *pmd, struct vm_area_struct *vma) 1994 { 1995 spinlock_t *ptl; 1996 ptl = pmd_lock(vma->vm_mm, pmd); 1997 if (likely(is_swap_pmd(*pmd) || pmd_trans_huge(*pmd) || 1998 pmd_devmap(*pmd))) 1999 return ptl; 2000 spin_unlock(ptl); 2001 return NULL; 2002 } 2003 2004 /* 2005 * Returns true if a given pud maps a thp, false otherwise. 2006 * 2007 * Note that if it returns true, this routine returns without unlocking page 2008 * table lock. So callers must unlock it. 2009 */ 2010 spinlock_t *__pud_trans_huge_lock(pud_t *pud, struct vm_area_struct *vma) 2011 { 2012 spinlock_t *ptl; 2013 2014 ptl = pud_lock(vma->vm_mm, pud); 2015 if (likely(pud_trans_huge(*pud) || pud_devmap(*pud))) 2016 return ptl; 2017 spin_unlock(ptl); 2018 return NULL; 2019 } 2020 2021 #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD 2022 int zap_huge_pud(struct mmu_gather *tlb, struct vm_area_struct *vma, 2023 pud_t *pud, unsigned long addr) 2024 { 2025 spinlock_t *ptl; 2026 2027 ptl = __pud_trans_huge_lock(pud, vma); 2028 if (!ptl) 2029 return 0; 2030 /* 2031 * For architectures like ppc64 we look at deposited pgtable 2032 * when calling pudp_huge_get_and_clear. So do the 2033 * pgtable_trans_huge_withdraw after finishing pudp related 2034 * operations. 2035 */ 2036 pudp_huge_get_and_clear_full(tlb->mm, addr, pud, tlb->fullmm); 2037 tlb_remove_pud_tlb_entry(tlb, pud, addr); 2038 if (vma_is_dax(vma)) { 2039 spin_unlock(ptl); 2040 /* No zero page support yet */ 2041 } else { 2042 /* No support for anonymous PUD pages yet */ 2043 BUG(); 2044 } 2045 return 1; 2046 } 2047 2048 static void __split_huge_pud_locked(struct vm_area_struct *vma, pud_t *pud, 2049 unsigned long haddr) 2050 { 2051 VM_BUG_ON(haddr & ~HPAGE_PUD_MASK); 2052 VM_BUG_ON_VMA(vma->vm_start > haddr, vma); 2053 VM_BUG_ON_VMA(vma->vm_end < haddr + HPAGE_PUD_SIZE, vma); 2054 VM_BUG_ON(!pud_trans_huge(*pud) && !pud_devmap(*pud)); 2055 2056 count_vm_event(THP_SPLIT_PUD); 2057 2058 pudp_huge_clear_flush_notify(vma, haddr, pud); 2059 } 2060 2061 void __split_huge_pud(struct vm_area_struct *vma, pud_t *pud, 2062 unsigned long address) 2063 { 2064 spinlock_t *ptl; 2065 struct mmu_notifier_range range; 2066 2067 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, vma->vm_mm, 2068 address & HPAGE_PUD_MASK, 2069 (address & HPAGE_PUD_MASK) + HPAGE_PUD_SIZE); 2070 mmu_notifier_invalidate_range_start(&range); 2071 ptl = pud_lock(vma->vm_mm, pud); 2072 if (unlikely(!pud_trans_huge(*pud) && !pud_devmap(*pud))) 2073 goto out; 2074 __split_huge_pud_locked(vma, pud, range.start); 2075 2076 out: 2077 spin_unlock(ptl); 2078 /* 2079 * No need to double call mmu_notifier->invalidate_range() callback as 2080 * the above pudp_huge_clear_flush_notify() did already call it. 2081 */ 2082 mmu_notifier_invalidate_range_only_end(&range); 2083 } 2084 #endif /* CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */ 2085 2086 static void __split_huge_zero_page_pmd(struct vm_area_struct *vma, 2087 unsigned long haddr, pmd_t *pmd) 2088 { 2089 struct mm_struct *mm = vma->vm_mm; 2090 pgtable_t pgtable; 2091 pmd_t _pmd; 2092 int i; 2093 2094 /* 2095 * Leave pmd empty until pte is filled note that it is fine to delay 2096 * notification until mmu_notifier_invalidate_range_end() as we are 2097 * replacing a zero pmd write protected page with a zero pte write 2098 * protected page. 2099 * 2100 * See Documentation/vm/mmu_notifier.rst 2101 */ 2102 pmdp_huge_clear_flush(vma, haddr, pmd); 2103 2104 pgtable = pgtable_trans_huge_withdraw(mm, pmd); 2105 pmd_populate(mm, &_pmd, pgtable); 2106 2107 for (i = 0; i < HPAGE_PMD_NR; i++, haddr += PAGE_SIZE) { 2108 pte_t *pte, entry; 2109 entry = pfn_pte(my_zero_pfn(haddr), vma->vm_page_prot); 2110 entry = pte_mkspecial(entry); 2111 pte = pte_offset_map(&_pmd, haddr); 2112 VM_BUG_ON(!pte_none(*pte)); 2113 set_pte_at(mm, haddr, pte, entry); 2114 pte_unmap(pte); 2115 } 2116 smp_wmb(); /* make pte visible before pmd */ 2117 pmd_populate(mm, pmd, pgtable); 2118 } 2119 2120 static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd, 2121 unsigned long haddr, bool freeze) 2122 { 2123 struct mm_struct *mm = vma->vm_mm; 2124 struct page *page; 2125 pgtable_t pgtable; 2126 pmd_t old_pmd, _pmd; 2127 bool young, write, soft_dirty, pmd_migration = false; 2128 unsigned long addr; 2129 int i; 2130 2131 VM_BUG_ON(haddr & ~HPAGE_PMD_MASK); 2132 VM_BUG_ON_VMA(vma->vm_start > haddr, vma); 2133 VM_BUG_ON_VMA(vma->vm_end < haddr + HPAGE_PMD_SIZE, vma); 2134 VM_BUG_ON(!is_pmd_migration_entry(*pmd) && !pmd_trans_huge(*pmd) 2135 && !pmd_devmap(*pmd)); 2136 2137 count_vm_event(THP_SPLIT_PMD); 2138 2139 if (!vma_is_anonymous(vma)) { 2140 _pmd = pmdp_huge_clear_flush_notify(vma, haddr, pmd); 2141 /* 2142 * We are going to unmap this huge page. So 2143 * just go ahead and zap it 2144 */ 2145 if (arch_needs_pgtable_deposit()) 2146 zap_deposited_table(mm, pmd); 2147 if (vma_is_dax(vma)) 2148 return; 2149 page = pmd_page(_pmd); 2150 if (!PageDirty(page) && pmd_dirty(_pmd)) 2151 set_page_dirty(page); 2152 if (!PageReferenced(page) && pmd_young(_pmd)) 2153 SetPageReferenced(page); 2154 page_remove_rmap(page, true); 2155 put_page(page); 2156 add_mm_counter(mm, mm_counter_file(page), -HPAGE_PMD_NR); 2157 return; 2158 } else if (is_huge_zero_pmd(*pmd)) { 2159 /* 2160 * FIXME: Do we want to invalidate secondary mmu by calling 2161 * mmu_notifier_invalidate_range() see comments below inside 2162 * __split_huge_pmd() ? 2163 * 2164 * We are going from a zero huge page write protected to zero 2165 * small page also write protected so it does not seems useful 2166 * to invalidate secondary mmu at this time. 2167 */ 2168 return __split_huge_zero_page_pmd(vma, haddr, pmd); 2169 } 2170 2171 /* 2172 * Up to this point the pmd is present and huge and userland has the 2173 * whole access to the hugepage during the split (which happens in 2174 * place). If we overwrite the pmd with the not-huge version pointing 2175 * to the pte here (which of course we could if all CPUs were bug 2176 * free), userland could trigger a small page size TLB miss on the 2177 * small sized TLB while the hugepage TLB entry is still established in 2178 * the huge TLB. Some CPU doesn't like that. 2179 * See http://support.amd.com/us/Processor_TechDocs/41322.pdf, Erratum 2180 * 383 on page 93. Intel should be safe but is also warns that it's 2181 * only safe if the permission and cache attributes of the two entries 2182 * loaded in the two TLB is identical (which should be the case here). 2183 * But it is generally safer to never allow small and huge TLB entries 2184 * for the same virtual address to be loaded simultaneously. So instead 2185 * of doing "pmd_populate(); flush_pmd_tlb_range();" we first mark the 2186 * current pmd notpresent (atomically because here the pmd_trans_huge 2187 * must remain set at all times on the pmd until the split is complete 2188 * for this pmd), then we flush the SMP TLB and finally we write the 2189 * non-huge version of the pmd entry with pmd_populate. 2190 */ 2191 old_pmd = pmdp_invalidate(vma, haddr, pmd); 2192 2193 pmd_migration = is_pmd_migration_entry(old_pmd); 2194 if (unlikely(pmd_migration)) { 2195 swp_entry_t entry; 2196 2197 entry = pmd_to_swp_entry(old_pmd); 2198 page = pfn_to_page(swp_offset(entry)); 2199 write = is_write_migration_entry(entry); 2200 young = false; 2201 soft_dirty = pmd_swp_soft_dirty(old_pmd); 2202 } else { 2203 page = pmd_page(old_pmd); 2204 if (pmd_dirty(old_pmd)) 2205 SetPageDirty(page); 2206 write = pmd_write(old_pmd); 2207 young = pmd_young(old_pmd); 2208 soft_dirty = pmd_soft_dirty(old_pmd); 2209 } 2210 VM_BUG_ON_PAGE(!page_count(page), page); 2211 page_ref_add(page, HPAGE_PMD_NR - 1); 2212 2213 /* 2214 * Withdraw the table only after we mark the pmd entry invalid. 2215 * This's critical for some architectures (Power). 2216 */ 2217 pgtable = pgtable_trans_huge_withdraw(mm, pmd); 2218 pmd_populate(mm, &_pmd, pgtable); 2219 2220 for (i = 0, addr = haddr; i < HPAGE_PMD_NR; i++, addr += PAGE_SIZE) { 2221 pte_t entry, *pte; 2222 /* 2223 * Note that NUMA hinting access restrictions are not 2224 * transferred to avoid any possibility of altering 2225 * permissions across VMAs. 2226 */ 2227 if (freeze || pmd_migration) { 2228 swp_entry_t swp_entry; 2229 swp_entry = make_migration_entry(page + i, write); 2230 entry = swp_entry_to_pte(swp_entry); 2231 if (soft_dirty) 2232 entry = pte_swp_mksoft_dirty(entry); 2233 } else { 2234 entry = mk_pte(page + i, READ_ONCE(vma->vm_page_prot)); 2235 entry = maybe_mkwrite(entry, vma); 2236 if (!write) 2237 entry = pte_wrprotect(entry); 2238 if (!young) 2239 entry = pte_mkold(entry); 2240 if (soft_dirty) 2241 entry = pte_mksoft_dirty(entry); 2242 } 2243 pte = pte_offset_map(&_pmd, addr); 2244 BUG_ON(!pte_none(*pte)); 2245 set_pte_at(mm, addr, pte, entry); 2246 atomic_inc(&page[i]._mapcount); 2247 pte_unmap(pte); 2248 } 2249 2250 /* 2251 * Set PG_double_map before dropping compound_mapcount to avoid 2252 * false-negative page_mapped(). 2253 */ 2254 if (compound_mapcount(page) > 1 && !TestSetPageDoubleMap(page)) { 2255 for (i = 0; i < HPAGE_PMD_NR; i++) 2256 atomic_inc(&page[i]._mapcount); 2257 } 2258 2259 if (atomic_add_negative(-1, compound_mapcount_ptr(page))) { 2260 /* Last compound_mapcount is gone. */ 2261 __dec_node_page_state(page, NR_ANON_THPS); 2262 if (TestClearPageDoubleMap(page)) { 2263 /* No need in mapcount reference anymore */ 2264 for (i = 0; i < HPAGE_PMD_NR; i++) 2265 atomic_dec(&page[i]._mapcount); 2266 } 2267 } 2268 2269 smp_wmb(); /* make pte visible before pmd */ 2270 pmd_populate(mm, pmd, pgtable); 2271 2272 if (freeze) { 2273 for (i = 0; i < HPAGE_PMD_NR; i++) { 2274 page_remove_rmap(page + i, false); 2275 put_page(page + i); 2276 } 2277 } 2278 } 2279 2280 void __split_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd, 2281 unsigned long address, bool freeze, struct page *page) 2282 { 2283 spinlock_t *ptl; 2284 struct mmu_notifier_range range; 2285 2286 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, vma->vm_mm, 2287 address & HPAGE_PMD_MASK, 2288 (address & HPAGE_PMD_MASK) + HPAGE_PMD_SIZE); 2289 mmu_notifier_invalidate_range_start(&range); 2290 ptl = pmd_lock(vma->vm_mm, pmd); 2291 2292 /* 2293 * If caller asks to setup a migration entries, we need a page to check 2294 * pmd against. Otherwise we can end up replacing wrong page. 2295 */ 2296 VM_BUG_ON(freeze && !page); 2297 if (page && page != pmd_page(*pmd)) 2298 goto out; 2299 2300 if (pmd_trans_huge(*pmd)) { 2301 page = pmd_page(*pmd); 2302 if (PageMlocked(page)) 2303 clear_page_mlock(page); 2304 } else if (!(pmd_devmap(*pmd) || is_pmd_migration_entry(*pmd))) 2305 goto out; 2306 __split_huge_pmd_locked(vma, pmd, range.start, freeze); 2307 out: 2308 spin_unlock(ptl); 2309 /* 2310 * No need to double call mmu_notifier->invalidate_range() callback. 2311 * They are 3 cases to consider inside __split_huge_pmd_locked(): 2312 * 1) pmdp_huge_clear_flush_notify() call invalidate_range() obvious 2313 * 2) __split_huge_zero_page_pmd() read only zero page and any write 2314 * fault will trigger a flush_notify before pointing to a new page 2315 * (it is fine if the secondary mmu keeps pointing to the old zero 2316 * page in the meantime) 2317 * 3) Split a huge pmd into pte pointing to the same page. No need 2318 * to invalidate secondary tlb entry they are all still valid. 2319 * any further changes to individual pte will notify. So no need 2320 * to call mmu_notifier->invalidate_range() 2321 */ 2322 mmu_notifier_invalidate_range_only_end(&range); 2323 } 2324 2325 void split_huge_pmd_address(struct vm_area_struct *vma, unsigned long address, 2326 bool freeze, struct page *page) 2327 { 2328 pgd_t *pgd; 2329 p4d_t *p4d; 2330 pud_t *pud; 2331 pmd_t *pmd; 2332 2333 pgd = pgd_offset(vma->vm_mm, address); 2334 if (!pgd_present(*pgd)) 2335 return; 2336 2337 p4d = p4d_offset(pgd, address); 2338 if (!p4d_present(*p4d)) 2339 return; 2340 2341 pud = pud_offset(p4d, address); 2342 if (!pud_present(*pud)) 2343 return; 2344 2345 pmd = pmd_offset(pud, address); 2346 2347 __split_huge_pmd(vma, pmd, address, freeze, page); 2348 } 2349 2350 void vma_adjust_trans_huge(struct vm_area_struct *vma, 2351 unsigned long start, 2352 unsigned long end, 2353 long adjust_next) 2354 { 2355 /* 2356 * If the new start address isn't hpage aligned and it could 2357 * previously contain an hugepage: check if we need to split 2358 * an huge pmd. 2359 */ 2360 if (start & ~HPAGE_PMD_MASK && 2361 (start & HPAGE_PMD_MASK) >= vma->vm_start && 2362 (start & HPAGE_PMD_MASK) + HPAGE_PMD_SIZE <= vma->vm_end) 2363 split_huge_pmd_address(vma, start, false, NULL); 2364 2365 /* 2366 * If the new end address isn't hpage aligned and it could 2367 * previously contain an hugepage: check if we need to split 2368 * an huge pmd. 2369 */ 2370 if (end & ~HPAGE_PMD_MASK && 2371 (end & HPAGE_PMD_MASK) >= vma->vm_start && 2372 (end & HPAGE_PMD_MASK) + HPAGE_PMD_SIZE <= vma->vm_end) 2373 split_huge_pmd_address(vma, end, false, NULL); 2374 2375 /* 2376 * If we're also updating the vma->vm_next->vm_start, if the new 2377 * vm_next->vm_start isn't page aligned and it could previously 2378 * contain an hugepage: check if we need to split an huge pmd. 2379 */ 2380 if (adjust_next > 0) { 2381 struct vm_area_struct *next = vma->vm_next; 2382 unsigned long nstart = next->vm_start; 2383 nstart += adjust_next << PAGE_SHIFT; 2384 if (nstart & ~HPAGE_PMD_MASK && 2385 (nstart & HPAGE_PMD_MASK) >= next->vm_start && 2386 (nstart & HPAGE_PMD_MASK) + HPAGE_PMD_SIZE <= next->vm_end) 2387 split_huge_pmd_address(next, nstart, false, NULL); 2388 } 2389 } 2390 2391 static void unmap_page(struct page *page) 2392 { 2393 enum ttu_flags ttu_flags = TTU_IGNORE_MLOCK | TTU_IGNORE_ACCESS | 2394 TTU_RMAP_LOCKED | TTU_SPLIT_HUGE_PMD; 2395 bool unmap_success; 2396 2397 VM_BUG_ON_PAGE(!PageHead(page), page); 2398 2399 if (PageAnon(page)) 2400 ttu_flags |= TTU_SPLIT_FREEZE; 2401 2402 unmap_success = try_to_unmap(page, ttu_flags); 2403 VM_BUG_ON_PAGE(!unmap_success, page); 2404 } 2405 2406 static void remap_page(struct page *page) 2407 { 2408 int i; 2409 if (PageTransHuge(page)) { 2410 remove_migration_ptes(page, page, true); 2411 } else { 2412 for (i = 0; i < HPAGE_PMD_NR; i++) 2413 remove_migration_ptes(page + i, page + i, true); 2414 } 2415 } 2416 2417 static void __split_huge_page_tail(struct page *head, int tail, 2418 struct lruvec *lruvec, struct list_head *list) 2419 { 2420 struct page *page_tail = head + tail; 2421 2422 VM_BUG_ON_PAGE(atomic_read(&page_tail->_mapcount) != -1, page_tail); 2423 2424 /* 2425 * Clone page flags before unfreezing refcount. 2426 * 2427 * After successful get_page_unless_zero() might follow flags change, 2428 * for exmaple lock_page() which set PG_waiters. 2429 */ 2430 page_tail->flags &= ~PAGE_FLAGS_CHECK_AT_PREP; 2431 page_tail->flags |= (head->flags & 2432 ((1L << PG_referenced) | 2433 (1L << PG_swapbacked) | 2434 (1L << PG_swapcache) | 2435 (1L << PG_mlocked) | 2436 (1L << PG_uptodate) | 2437 (1L << PG_active) | 2438 (1L << PG_workingset) | 2439 (1L << PG_locked) | 2440 (1L << PG_unevictable) | 2441 (1L << PG_dirty))); 2442 2443 /* ->mapping in first tail page is compound_mapcount */ 2444 VM_BUG_ON_PAGE(tail > 2 && page_tail->mapping != TAIL_MAPPING, 2445 page_tail); 2446 page_tail->mapping = head->mapping; 2447 page_tail->index = head->index + tail; 2448 2449 /* Page flags must be visible before we make the page non-compound. */ 2450 smp_wmb(); 2451 2452 /* 2453 * Clear PageTail before unfreezing page refcount. 2454 * 2455 * After successful get_page_unless_zero() might follow put_page() 2456 * which needs correct compound_head(). 2457 */ 2458 clear_compound_head(page_tail); 2459 2460 /* Finally unfreeze refcount. Additional reference from page cache. */ 2461 page_ref_unfreeze(page_tail, 1 + (!PageAnon(head) || 2462 PageSwapCache(head))); 2463 2464 if (page_is_young(head)) 2465 set_page_young(page_tail); 2466 if (page_is_idle(head)) 2467 set_page_idle(page_tail); 2468 2469 page_cpupid_xchg_last(page_tail, page_cpupid_last(head)); 2470 2471 /* 2472 * always add to the tail because some iterators expect new 2473 * pages to show after the currently processed elements - e.g. 2474 * migrate_pages 2475 */ 2476 lru_add_page_tail(head, page_tail, lruvec, list); 2477 } 2478 2479 static void __split_huge_page(struct page *page, struct list_head *list, 2480 pgoff_t end, unsigned long flags) 2481 { 2482 struct page *head = compound_head(page); 2483 pg_data_t *pgdat = page_pgdat(head); 2484 struct lruvec *lruvec; 2485 int i; 2486 2487 lruvec = mem_cgroup_page_lruvec(head, pgdat); 2488 2489 /* complete memcg works before add pages to LRU */ 2490 mem_cgroup_split_huge_fixup(head); 2491 2492 for (i = HPAGE_PMD_NR - 1; i >= 1; i--) { 2493 __split_huge_page_tail(head, i, lruvec, list); 2494 /* Some pages can be beyond i_size: drop them from page cache */ 2495 if (head[i].index >= end) { 2496 ClearPageDirty(head + i); 2497 __delete_from_page_cache(head + i, NULL); 2498 if (IS_ENABLED(CONFIG_SHMEM) && PageSwapBacked(head)) 2499 shmem_uncharge(head->mapping->host, 1); 2500 put_page(head + i); 2501 } else if (!PageAnon(page)) { 2502 __xa_store(&head->mapping->i_pages, head[i].index, 2503 head + i, 0); 2504 } 2505 } 2506 2507 ClearPageCompound(head); 2508 /* See comment in __split_huge_page_tail() */ 2509 if (PageAnon(head)) { 2510 /* Additional pin to swap cache */ 2511 if (PageSwapCache(head)) 2512 page_ref_add(head, 2); 2513 else 2514 page_ref_inc(head); 2515 } else { 2516 /* Additional pin to page cache */ 2517 page_ref_add(head, 2); 2518 xa_unlock(&head->mapping->i_pages); 2519 } 2520 2521 spin_unlock_irqrestore(&pgdat->lru_lock, flags); 2522 2523 remap_page(head); 2524 2525 for (i = 0; i < HPAGE_PMD_NR; i++) { 2526 struct page *subpage = head + i; 2527 if (subpage == page) 2528 continue; 2529 unlock_page(subpage); 2530 2531 /* 2532 * Subpages may be freed if there wasn't any mapping 2533 * like if add_to_swap() is running on a lru page that 2534 * had its mapping zapped. And freeing these pages 2535 * requires taking the lru_lock so we do the put_page 2536 * of the tail pages after the split is complete. 2537 */ 2538 put_page(subpage); 2539 } 2540 } 2541 2542 int total_mapcount(struct page *page) 2543 { 2544 int i, compound, ret; 2545 2546 VM_BUG_ON_PAGE(PageTail(page), page); 2547 2548 if (likely(!PageCompound(page))) 2549 return atomic_read(&page->_mapcount) + 1; 2550 2551 compound = compound_mapcount(page); 2552 if (PageHuge(page)) 2553 return compound; 2554 ret = compound; 2555 for (i = 0; i < HPAGE_PMD_NR; i++) 2556 ret += atomic_read(&page[i]._mapcount) + 1; 2557 /* File pages has compound_mapcount included in _mapcount */ 2558 if (!PageAnon(page)) 2559 return ret - compound * HPAGE_PMD_NR; 2560 if (PageDoubleMap(page)) 2561 ret -= HPAGE_PMD_NR; 2562 return ret; 2563 } 2564 2565 /* 2566 * This calculates accurately how many mappings a transparent hugepage 2567 * has (unlike page_mapcount() which isn't fully accurate). This full 2568 * accuracy is primarily needed to know if copy-on-write faults can 2569 * reuse the page and change the mapping to read-write instead of 2570 * copying them. At the same time this returns the total_mapcount too. 2571 * 2572 * The function returns the highest mapcount any one of the subpages 2573 * has. If the return value is one, even if different processes are 2574 * mapping different subpages of the transparent hugepage, they can 2575 * all reuse it, because each process is reusing a different subpage. 2576 * 2577 * The total_mapcount is instead counting all virtual mappings of the 2578 * subpages. If the total_mapcount is equal to "one", it tells the 2579 * caller all mappings belong to the same "mm" and in turn the 2580 * anon_vma of the transparent hugepage can become the vma->anon_vma 2581 * local one as no other process may be mapping any of the subpages. 2582 * 2583 * It would be more accurate to replace page_mapcount() with 2584 * page_trans_huge_mapcount(), however we only use 2585 * page_trans_huge_mapcount() in the copy-on-write faults where we 2586 * need full accuracy to avoid breaking page pinning, because 2587 * page_trans_huge_mapcount() is slower than page_mapcount(). 2588 */ 2589 int page_trans_huge_mapcount(struct page *page, int *total_mapcount) 2590 { 2591 int i, ret, _total_mapcount, mapcount; 2592 2593 /* hugetlbfs shouldn't call it */ 2594 VM_BUG_ON_PAGE(PageHuge(page), page); 2595 2596 if (likely(!PageTransCompound(page))) { 2597 mapcount = atomic_read(&page->_mapcount) + 1; 2598 if (total_mapcount) 2599 *total_mapcount = mapcount; 2600 return mapcount; 2601 } 2602 2603 page = compound_head(page); 2604 2605 _total_mapcount = ret = 0; 2606 for (i = 0; i < HPAGE_PMD_NR; i++) { 2607 mapcount = atomic_read(&page[i]._mapcount) + 1; 2608 ret = max(ret, mapcount); 2609 _total_mapcount += mapcount; 2610 } 2611 if (PageDoubleMap(page)) { 2612 ret -= 1; 2613 _total_mapcount -= HPAGE_PMD_NR; 2614 } 2615 mapcount = compound_mapcount(page); 2616 ret += mapcount; 2617 _total_mapcount += mapcount; 2618 if (total_mapcount) 2619 *total_mapcount = _total_mapcount; 2620 return ret; 2621 } 2622 2623 /* Racy check whether the huge page can be split */ 2624 bool can_split_huge_page(struct page *page, int *pextra_pins) 2625 { 2626 int extra_pins; 2627 2628 /* Additional pins from page cache */ 2629 if (PageAnon(page)) 2630 extra_pins = PageSwapCache(page) ? HPAGE_PMD_NR : 0; 2631 else 2632 extra_pins = HPAGE_PMD_NR; 2633 if (pextra_pins) 2634 *pextra_pins = extra_pins; 2635 return total_mapcount(page) == page_count(page) - extra_pins - 1; 2636 } 2637 2638 /* 2639 * This function splits huge page into normal pages. @page can point to any 2640 * subpage of huge page to split. Split doesn't change the position of @page. 2641 * 2642 * Only caller must hold pin on the @page, otherwise split fails with -EBUSY. 2643 * The huge page must be locked. 2644 * 2645 * If @list is null, tail pages will be added to LRU list, otherwise, to @list. 2646 * 2647 * Both head page and tail pages will inherit mapping, flags, and so on from 2648 * the hugepage. 2649 * 2650 * GUP pin and PG_locked transferred to @page. Rest subpages can be freed if 2651 * they are not mapped. 2652 * 2653 * Returns 0 if the hugepage is split successfully. 2654 * Returns -EBUSY if the page is pinned or if anon_vma disappeared from under 2655 * us. 2656 */ 2657 int split_huge_page_to_list(struct page *page, struct list_head *list) 2658 { 2659 struct page *head = compound_head(page); 2660 struct pglist_data *pgdata = NODE_DATA(page_to_nid(head)); 2661 struct anon_vma *anon_vma = NULL; 2662 struct address_space *mapping = NULL; 2663 int count, mapcount, extra_pins, ret; 2664 bool mlocked; 2665 unsigned long flags; 2666 pgoff_t end; 2667 2668 VM_BUG_ON_PAGE(is_huge_zero_page(page), page); 2669 VM_BUG_ON_PAGE(!PageLocked(page), page); 2670 VM_BUG_ON_PAGE(!PageCompound(page), page); 2671 2672 if (PageWriteback(page)) 2673 return -EBUSY; 2674 2675 if (PageAnon(head)) { 2676 /* 2677 * The caller does not necessarily hold an mmap_sem that would 2678 * prevent the anon_vma disappearing so we first we take a 2679 * reference to it and then lock the anon_vma for write. This 2680 * is similar to page_lock_anon_vma_read except the write lock 2681 * is taken to serialise against parallel split or collapse 2682 * operations. 2683 */ 2684 anon_vma = page_get_anon_vma(head); 2685 if (!anon_vma) { 2686 ret = -EBUSY; 2687 goto out; 2688 } 2689 end = -1; 2690 mapping = NULL; 2691 anon_vma_lock_write(anon_vma); 2692 } else { 2693 mapping = head->mapping; 2694 2695 /* Truncated ? */ 2696 if (!mapping) { 2697 ret = -EBUSY; 2698 goto out; 2699 } 2700 2701 anon_vma = NULL; 2702 i_mmap_lock_read(mapping); 2703 2704 /* 2705 *__split_huge_page() may need to trim off pages beyond EOF: 2706 * but on 32-bit, i_size_read() takes an irq-unsafe seqlock, 2707 * which cannot be nested inside the page tree lock. So note 2708 * end now: i_size itself may be changed at any moment, but 2709 * head page lock is good enough to serialize the trimming. 2710 */ 2711 end = DIV_ROUND_UP(i_size_read(mapping->host), PAGE_SIZE); 2712 } 2713 2714 /* 2715 * Racy check if we can split the page, before unmap_page() will 2716 * split PMDs 2717 */ 2718 if (!can_split_huge_page(head, &extra_pins)) { 2719 ret = -EBUSY; 2720 goto out_unlock; 2721 } 2722 2723 mlocked = PageMlocked(page); 2724 unmap_page(head); 2725 VM_BUG_ON_PAGE(compound_mapcount(head), head); 2726 2727 /* Make sure the page is not on per-CPU pagevec as it takes pin */ 2728 if (mlocked) 2729 lru_add_drain(); 2730 2731 /* prevent PageLRU to go away from under us, and freeze lru stats */ 2732 spin_lock_irqsave(&pgdata->lru_lock, flags); 2733 2734 if (mapping) { 2735 XA_STATE(xas, &mapping->i_pages, page_index(head)); 2736 2737 /* 2738 * Check if the head page is present in page cache. 2739 * We assume all tail are present too, if head is there. 2740 */ 2741 xa_lock(&mapping->i_pages); 2742 if (xas_load(&xas) != head) 2743 goto fail; 2744 } 2745 2746 /* Prevent deferred_split_scan() touching ->_refcount */ 2747 spin_lock(&pgdata->split_queue_lock); 2748 count = page_count(head); 2749 mapcount = total_mapcount(head); 2750 if (!mapcount && page_ref_freeze(head, 1 + extra_pins)) { 2751 if (!list_empty(page_deferred_list(head))) { 2752 pgdata->split_queue_len--; 2753 list_del(page_deferred_list(head)); 2754 } 2755 if (mapping) 2756 __dec_node_page_state(page, NR_SHMEM_THPS); 2757 spin_unlock(&pgdata->split_queue_lock); 2758 __split_huge_page(page, list, end, flags); 2759 if (PageSwapCache(head)) { 2760 swp_entry_t entry = { .val = page_private(head) }; 2761 2762 ret = split_swap_cluster(entry); 2763 } else 2764 ret = 0; 2765 } else { 2766 if (IS_ENABLED(CONFIG_DEBUG_VM) && mapcount) { 2767 pr_alert("total_mapcount: %u, page_count(): %u\n", 2768 mapcount, count); 2769 if (PageTail(page)) 2770 dump_page(head, NULL); 2771 dump_page(page, "total_mapcount(head) > 0"); 2772 BUG(); 2773 } 2774 spin_unlock(&pgdata->split_queue_lock); 2775 fail: if (mapping) 2776 xa_unlock(&mapping->i_pages); 2777 spin_unlock_irqrestore(&pgdata->lru_lock, flags); 2778 remap_page(head); 2779 ret = -EBUSY; 2780 } 2781 2782 out_unlock: 2783 if (anon_vma) { 2784 anon_vma_unlock_write(anon_vma); 2785 put_anon_vma(anon_vma); 2786 } 2787 if (mapping) 2788 i_mmap_unlock_read(mapping); 2789 out: 2790 count_vm_event(!ret ? THP_SPLIT_PAGE : THP_SPLIT_PAGE_FAILED); 2791 return ret; 2792 } 2793 2794 void free_transhuge_page(struct page *page) 2795 { 2796 struct pglist_data *pgdata = NODE_DATA(page_to_nid(page)); 2797 unsigned long flags; 2798 2799 spin_lock_irqsave(&pgdata->split_queue_lock, flags); 2800 if (!list_empty(page_deferred_list(page))) { 2801 pgdata->split_queue_len--; 2802 list_del(page_deferred_list(page)); 2803 } 2804 spin_unlock_irqrestore(&pgdata->split_queue_lock, flags); 2805 free_compound_page(page); 2806 } 2807 2808 void deferred_split_huge_page(struct page *page) 2809 { 2810 struct pglist_data *pgdata = NODE_DATA(page_to_nid(page)); 2811 unsigned long flags; 2812 2813 VM_BUG_ON_PAGE(!PageTransHuge(page), page); 2814 2815 spin_lock_irqsave(&pgdata->split_queue_lock, flags); 2816 if (list_empty(page_deferred_list(page))) { 2817 count_vm_event(THP_DEFERRED_SPLIT_PAGE); 2818 list_add_tail(page_deferred_list(page), &pgdata->split_queue); 2819 pgdata->split_queue_len++; 2820 } 2821 spin_unlock_irqrestore(&pgdata->split_queue_lock, flags); 2822 } 2823 2824 static unsigned long deferred_split_count(struct shrinker *shrink, 2825 struct shrink_control *sc) 2826 { 2827 struct pglist_data *pgdata = NODE_DATA(sc->nid); 2828 return READ_ONCE(pgdata->split_queue_len); 2829 } 2830 2831 static unsigned long deferred_split_scan(struct shrinker *shrink, 2832 struct shrink_control *sc) 2833 { 2834 struct pglist_data *pgdata = NODE_DATA(sc->nid); 2835 unsigned long flags; 2836 LIST_HEAD(list), *pos, *next; 2837 struct page *page; 2838 int split = 0; 2839 2840 spin_lock_irqsave(&pgdata->split_queue_lock, flags); 2841 /* Take pin on all head pages to avoid freeing them under us */ 2842 list_for_each_safe(pos, next, &pgdata->split_queue) { 2843 page = list_entry((void *)pos, struct page, mapping); 2844 page = compound_head(page); 2845 if (get_page_unless_zero(page)) { 2846 list_move(page_deferred_list(page), &list); 2847 } else { 2848 /* We lost race with put_compound_page() */ 2849 list_del_init(page_deferred_list(page)); 2850 pgdata->split_queue_len--; 2851 } 2852 if (!--sc->nr_to_scan) 2853 break; 2854 } 2855 spin_unlock_irqrestore(&pgdata->split_queue_lock, flags); 2856 2857 list_for_each_safe(pos, next, &list) { 2858 page = list_entry((void *)pos, struct page, mapping); 2859 if (!trylock_page(page)) 2860 goto next; 2861 /* split_huge_page() removes page from list on success */ 2862 if (!split_huge_page(page)) 2863 split++; 2864 unlock_page(page); 2865 next: 2866 put_page(page); 2867 } 2868 2869 spin_lock_irqsave(&pgdata->split_queue_lock, flags); 2870 list_splice_tail(&list, &pgdata->split_queue); 2871 spin_unlock_irqrestore(&pgdata->split_queue_lock, flags); 2872 2873 /* 2874 * Stop shrinker if we didn't split any page, but the queue is empty. 2875 * This can happen if pages were freed under us. 2876 */ 2877 if (!split && list_empty(&pgdata->split_queue)) 2878 return SHRINK_STOP; 2879 return split; 2880 } 2881 2882 static struct shrinker deferred_split_shrinker = { 2883 .count_objects = deferred_split_count, 2884 .scan_objects = deferred_split_scan, 2885 .seeks = DEFAULT_SEEKS, 2886 .flags = SHRINKER_NUMA_AWARE, 2887 }; 2888 2889 #ifdef CONFIG_DEBUG_FS 2890 static int split_huge_pages_set(void *data, u64 val) 2891 { 2892 struct zone *zone; 2893 struct page *page; 2894 unsigned long pfn, max_zone_pfn; 2895 unsigned long total = 0, split = 0; 2896 2897 if (val != 1) 2898 return -EINVAL; 2899 2900 for_each_populated_zone(zone) { 2901 max_zone_pfn = zone_end_pfn(zone); 2902 for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++) { 2903 if (!pfn_valid(pfn)) 2904 continue; 2905 2906 page = pfn_to_page(pfn); 2907 if (!get_page_unless_zero(page)) 2908 continue; 2909 2910 if (zone != page_zone(page)) 2911 goto next; 2912 2913 if (!PageHead(page) || PageHuge(page) || !PageLRU(page)) 2914 goto next; 2915 2916 total++; 2917 lock_page(page); 2918 if (!split_huge_page(page)) 2919 split++; 2920 unlock_page(page); 2921 next: 2922 put_page(page); 2923 } 2924 } 2925 2926 pr_info("%lu of %lu THP split\n", split, total); 2927 2928 return 0; 2929 } 2930 DEFINE_SIMPLE_ATTRIBUTE(split_huge_pages_fops, NULL, split_huge_pages_set, 2931 "%llu\n"); 2932 2933 static int __init split_huge_pages_debugfs(void) 2934 { 2935 debugfs_create_file("split_huge_pages", 0200, NULL, NULL, 2936 &split_huge_pages_fops); 2937 return 0; 2938 } 2939 late_initcall(split_huge_pages_debugfs); 2940 #endif 2941 2942 #ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION 2943 void set_pmd_migration_entry(struct page_vma_mapped_walk *pvmw, 2944 struct page *page) 2945 { 2946 struct vm_area_struct *vma = pvmw->vma; 2947 struct mm_struct *mm = vma->vm_mm; 2948 unsigned long address = pvmw->address; 2949 pmd_t pmdval; 2950 swp_entry_t entry; 2951 pmd_t pmdswp; 2952 2953 if (!(pvmw->pmd && !pvmw->pte)) 2954 return; 2955 2956 flush_cache_range(vma, address, address + HPAGE_PMD_SIZE); 2957 pmdval = *pvmw->pmd; 2958 pmdp_invalidate(vma, address, pvmw->pmd); 2959 if (pmd_dirty(pmdval)) 2960 set_page_dirty(page); 2961 entry = make_migration_entry(page, pmd_write(pmdval)); 2962 pmdswp = swp_entry_to_pmd(entry); 2963 if (pmd_soft_dirty(pmdval)) 2964 pmdswp = pmd_swp_mksoft_dirty(pmdswp); 2965 set_pmd_at(mm, address, pvmw->pmd, pmdswp); 2966 page_remove_rmap(page, true); 2967 put_page(page); 2968 } 2969 2970 void remove_migration_pmd(struct page_vma_mapped_walk *pvmw, struct page *new) 2971 { 2972 struct vm_area_struct *vma = pvmw->vma; 2973 struct mm_struct *mm = vma->vm_mm; 2974 unsigned long address = pvmw->address; 2975 unsigned long mmun_start = address & HPAGE_PMD_MASK; 2976 pmd_t pmde; 2977 swp_entry_t entry; 2978 2979 if (!(pvmw->pmd && !pvmw->pte)) 2980 return; 2981 2982 entry = pmd_to_swp_entry(*pvmw->pmd); 2983 get_page(new); 2984 pmde = pmd_mkold(mk_huge_pmd(new, vma->vm_page_prot)); 2985 if (pmd_swp_soft_dirty(*pvmw->pmd)) 2986 pmde = pmd_mksoft_dirty(pmde); 2987 if (is_write_migration_entry(entry)) 2988 pmde = maybe_pmd_mkwrite(pmde, vma); 2989 2990 flush_cache_range(vma, mmun_start, mmun_start + HPAGE_PMD_SIZE); 2991 if (PageAnon(new)) 2992 page_add_anon_rmap(new, vma, mmun_start, true); 2993 else 2994 page_add_file_rmap(new, true); 2995 set_pmd_at(mm, mmun_start, pvmw->pmd, pmde); 2996 if ((vma->vm_flags & VM_LOCKED) && !PageDoubleMap(new)) 2997 mlock_vma_page(new); 2998 update_mmu_cache_pmd(vma, address, pvmw->pmd); 2999 } 3000 #endif 3001