1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Generic hugetlb support. 4 * (C) Nadia Yvette Chambers, April 2004 5 */ 6 #include <linux/list.h> 7 #include <linux/init.h> 8 #include <linux/mm.h> 9 #include <linux/seq_file.h> 10 #include <linux/sysctl.h> 11 #include <linux/highmem.h> 12 #include <linux/mmu_notifier.h> 13 #include <linux/nodemask.h> 14 #include <linux/pagemap.h> 15 #include <linux/mempolicy.h> 16 #include <linux/compiler.h> 17 #include <linux/cpuset.h> 18 #include <linux/mutex.h> 19 #include <linux/memblock.h> 20 #include <linux/sysfs.h> 21 #include <linux/slab.h> 22 #include <linux/sched/mm.h> 23 #include <linux/mmdebug.h> 24 #include <linux/sched/signal.h> 25 #include <linux/rmap.h> 26 #include <linux/string_helpers.h> 27 #include <linux/swap.h> 28 #include <linux/swapops.h> 29 #include <linux/jhash.h> 30 #include <linux/numa.h> 31 #include <linux/llist.h> 32 #include <linux/cma.h> 33 #include <linux/migrate.h> 34 #include <linux/nospec.h> 35 #include <linux/delayacct.h> 36 #include <linux/memory.h> 37 #include <linux/mm_inline.h> 38 39 #include <asm/page.h> 40 #include <asm/pgalloc.h> 41 #include <asm/tlb.h> 42 43 #include <linux/io.h> 44 #include <linux/hugetlb.h> 45 #include <linux/hugetlb_cgroup.h> 46 #include <linux/node.h> 47 #include <linux/page_owner.h> 48 #include "internal.h" 49 #include "hugetlb_vmemmap.h" 50 51 int hugetlb_max_hstate __read_mostly; 52 unsigned int default_hstate_idx; 53 struct hstate hstates[HUGE_MAX_HSTATE]; 54 55 #ifdef CONFIG_CMA 56 static struct cma *hugetlb_cma[MAX_NUMNODES]; 57 static unsigned long hugetlb_cma_size_in_node[MAX_NUMNODES] __initdata; 58 static bool hugetlb_cma_folio(struct folio *folio, unsigned int order) 59 { 60 return cma_pages_valid(hugetlb_cma[folio_nid(folio)], &folio->page, 61 1 << order); 62 } 63 #else 64 static bool hugetlb_cma_folio(struct folio *folio, unsigned int order) 65 { 66 return false; 67 } 68 #endif 69 static unsigned long hugetlb_cma_size __initdata; 70 71 __initdata LIST_HEAD(huge_boot_pages); 72 73 /* for command line parsing */ 74 static struct hstate * __initdata parsed_hstate; 75 static unsigned long __initdata default_hstate_max_huge_pages; 76 static bool __initdata parsed_valid_hugepagesz = true; 77 static bool __initdata parsed_default_hugepagesz; 78 static unsigned int default_hugepages_in_node[MAX_NUMNODES] __initdata; 79 80 /* 81 * Protects updates to hugepage_freelists, hugepage_activelist, nr_huge_pages, 82 * free_huge_pages, and surplus_huge_pages. 83 */ 84 DEFINE_SPINLOCK(hugetlb_lock); 85 86 /* 87 * Serializes faults on the same logical page. This is used to 88 * prevent spurious OOMs when the hugepage pool is fully utilized. 89 */ 90 static int num_fault_mutexes; 91 struct mutex *hugetlb_fault_mutex_table ____cacheline_aligned_in_smp; 92 93 /* Forward declaration */ 94 static int hugetlb_acct_memory(struct hstate *h, long delta); 95 static void hugetlb_vma_lock_free(struct vm_area_struct *vma); 96 static void hugetlb_vma_lock_alloc(struct vm_area_struct *vma); 97 static void __hugetlb_vma_unlock_write_free(struct vm_area_struct *vma); 98 static void hugetlb_unshare_pmds(struct vm_area_struct *vma, 99 unsigned long start, unsigned long end); 100 101 static inline bool subpool_is_free(struct hugepage_subpool *spool) 102 { 103 if (spool->count) 104 return false; 105 if (spool->max_hpages != -1) 106 return spool->used_hpages == 0; 107 if (spool->min_hpages != -1) 108 return spool->rsv_hpages == spool->min_hpages; 109 110 return true; 111 } 112 113 static inline void unlock_or_release_subpool(struct hugepage_subpool *spool, 114 unsigned long irq_flags) 115 { 116 spin_unlock_irqrestore(&spool->lock, irq_flags); 117 118 /* If no pages are used, and no other handles to the subpool 119 * remain, give up any reservations based on minimum size and 120 * free the subpool */ 121 if (subpool_is_free(spool)) { 122 if (spool->min_hpages != -1) 123 hugetlb_acct_memory(spool->hstate, 124 -spool->min_hpages); 125 kfree(spool); 126 } 127 } 128 129 struct hugepage_subpool *hugepage_new_subpool(struct hstate *h, long max_hpages, 130 long min_hpages) 131 { 132 struct hugepage_subpool *spool; 133 134 spool = kzalloc(sizeof(*spool), GFP_KERNEL); 135 if (!spool) 136 return NULL; 137 138 spin_lock_init(&spool->lock); 139 spool->count = 1; 140 spool->max_hpages = max_hpages; 141 spool->hstate = h; 142 spool->min_hpages = min_hpages; 143 144 if (min_hpages != -1 && hugetlb_acct_memory(h, min_hpages)) { 145 kfree(spool); 146 return NULL; 147 } 148 spool->rsv_hpages = min_hpages; 149 150 return spool; 151 } 152 153 void hugepage_put_subpool(struct hugepage_subpool *spool) 154 { 155 unsigned long flags; 156 157 spin_lock_irqsave(&spool->lock, flags); 158 BUG_ON(!spool->count); 159 spool->count--; 160 unlock_or_release_subpool(spool, flags); 161 } 162 163 /* 164 * Subpool accounting for allocating and reserving pages. 165 * Return -ENOMEM if there are not enough resources to satisfy the 166 * request. Otherwise, return the number of pages by which the 167 * global pools must be adjusted (upward). The returned value may 168 * only be different than the passed value (delta) in the case where 169 * a subpool minimum size must be maintained. 170 */ 171 static long hugepage_subpool_get_pages(struct hugepage_subpool *spool, 172 long delta) 173 { 174 long ret = delta; 175 176 if (!spool) 177 return ret; 178 179 spin_lock_irq(&spool->lock); 180 181 if (spool->max_hpages != -1) { /* maximum size accounting */ 182 if ((spool->used_hpages + delta) <= spool->max_hpages) 183 spool->used_hpages += delta; 184 else { 185 ret = -ENOMEM; 186 goto unlock_ret; 187 } 188 } 189 190 /* minimum size accounting */ 191 if (spool->min_hpages != -1 && spool->rsv_hpages) { 192 if (delta > spool->rsv_hpages) { 193 /* 194 * Asking for more reserves than those already taken on 195 * behalf of subpool. Return difference. 196 */ 197 ret = delta - spool->rsv_hpages; 198 spool->rsv_hpages = 0; 199 } else { 200 ret = 0; /* reserves already accounted for */ 201 spool->rsv_hpages -= delta; 202 } 203 } 204 205 unlock_ret: 206 spin_unlock_irq(&spool->lock); 207 return ret; 208 } 209 210 /* 211 * Subpool accounting for freeing and unreserving pages. 212 * Return the number of global page reservations that must be dropped. 213 * The return value may only be different than the passed value (delta) 214 * in the case where a subpool minimum size must be maintained. 215 */ 216 static long hugepage_subpool_put_pages(struct hugepage_subpool *spool, 217 long delta) 218 { 219 long ret = delta; 220 unsigned long flags; 221 222 if (!spool) 223 return delta; 224 225 spin_lock_irqsave(&spool->lock, flags); 226 227 if (spool->max_hpages != -1) /* maximum size accounting */ 228 spool->used_hpages -= delta; 229 230 /* minimum size accounting */ 231 if (spool->min_hpages != -1 && spool->used_hpages < spool->min_hpages) { 232 if (spool->rsv_hpages + delta <= spool->min_hpages) 233 ret = 0; 234 else 235 ret = spool->rsv_hpages + delta - spool->min_hpages; 236 237 spool->rsv_hpages += delta; 238 if (spool->rsv_hpages > spool->min_hpages) 239 spool->rsv_hpages = spool->min_hpages; 240 } 241 242 /* 243 * If hugetlbfs_put_super couldn't free spool due to an outstanding 244 * quota reference, free it now. 245 */ 246 unlock_or_release_subpool(spool, flags); 247 248 return ret; 249 } 250 251 static inline struct hugepage_subpool *subpool_inode(struct inode *inode) 252 { 253 return HUGETLBFS_SB(inode->i_sb)->spool; 254 } 255 256 static inline struct hugepage_subpool *subpool_vma(struct vm_area_struct *vma) 257 { 258 return subpool_inode(file_inode(vma->vm_file)); 259 } 260 261 /* 262 * hugetlb vma_lock helper routines 263 */ 264 void hugetlb_vma_lock_read(struct vm_area_struct *vma) 265 { 266 if (__vma_shareable_lock(vma)) { 267 struct hugetlb_vma_lock *vma_lock = vma->vm_private_data; 268 269 down_read(&vma_lock->rw_sema); 270 } 271 } 272 273 void hugetlb_vma_unlock_read(struct vm_area_struct *vma) 274 { 275 if (__vma_shareable_lock(vma)) { 276 struct hugetlb_vma_lock *vma_lock = vma->vm_private_data; 277 278 up_read(&vma_lock->rw_sema); 279 } 280 } 281 282 void hugetlb_vma_lock_write(struct vm_area_struct *vma) 283 { 284 if (__vma_shareable_lock(vma)) { 285 struct hugetlb_vma_lock *vma_lock = vma->vm_private_data; 286 287 down_write(&vma_lock->rw_sema); 288 } 289 } 290 291 void hugetlb_vma_unlock_write(struct vm_area_struct *vma) 292 { 293 if (__vma_shareable_lock(vma)) { 294 struct hugetlb_vma_lock *vma_lock = vma->vm_private_data; 295 296 up_write(&vma_lock->rw_sema); 297 } 298 } 299 300 int hugetlb_vma_trylock_write(struct vm_area_struct *vma) 301 { 302 struct hugetlb_vma_lock *vma_lock = vma->vm_private_data; 303 304 if (!__vma_shareable_lock(vma)) 305 return 1; 306 307 return down_write_trylock(&vma_lock->rw_sema); 308 } 309 310 void hugetlb_vma_assert_locked(struct vm_area_struct *vma) 311 { 312 if (__vma_shareable_lock(vma)) { 313 struct hugetlb_vma_lock *vma_lock = vma->vm_private_data; 314 315 lockdep_assert_held(&vma_lock->rw_sema); 316 } 317 } 318 319 void hugetlb_vma_lock_release(struct kref *kref) 320 { 321 struct hugetlb_vma_lock *vma_lock = container_of(kref, 322 struct hugetlb_vma_lock, refs); 323 324 kfree(vma_lock); 325 } 326 327 static void __hugetlb_vma_unlock_write_put(struct hugetlb_vma_lock *vma_lock) 328 { 329 struct vm_area_struct *vma = vma_lock->vma; 330 331 /* 332 * vma_lock structure may or not be released as a result of put, 333 * it certainly will no longer be attached to vma so clear pointer. 334 * Semaphore synchronizes access to vma_lock->vma field. 335 */ 336 vma_lock->vma = NULL; 337 vma->vm_private_data = NULL; 338 up_write(&vma_lock->rw_sema); 339 kref_put(&vma_lock->refs, hugetlb_vma_lock_release); 340 } 341 342 static void __hugetlb_vma_unlock_write_free(struct vm_area_struct *vma) 343 { 344 if (__vma_shareable_lock(vma)) { 345 struct hugetlb_vma_lock *vma_lock = vma->vm_private_data; 346 347 __hugetlb_vma_unlock_write_put(vma_lock); 348 } 349 } 350 351 static void hugetlb_vma_lock_free(struct vm_area_struct *vma) 352 { 353 /* 354 * Only present in sharable vmas. 355 */ 356 if (!vma || !__vma_shareable_lock(vma)) 357 return; 358 359 if (vma->vm_private_data) { 360 struct hugetlb_vma_lock *vma_lock = vma->vm_private_data; 361 362 down_write(&vma_lock->rw_sema); 363 __hugetlb_vma_unlock_write_put(vma_lock); 364 } 365 } 366 367 static void hugetlb_vma_lock_alloc(struct vm_area_struct *vma) 368 { 369 struct hugetlb_vma_lock *vma_lock; 370 371 /* Only establish in (flags) sharable vmas */ 372 if (!vma || !(vma->vm_flags & VM_MAYSHARE)) 373 return; 374 375 /* Should never get here with non-NULL vm_private_data */ 376 if (vma->vm_private_data) 377 return; 378 379 vma_lock = kmalloc(sizeof(*vma_lock), GFP_KERNEL); 380 if (!vma_lock) { 381 /* 382 * If we can not allocate structure, then vma can not 383 * participate in pmd sharing. This is only a possible 384 * performance enhancement and memory saving issue. 385 * However, the lock is also used to synchronize page 386 * faults with truncation. If the lock is not present, 387 * unlikely races could leave pages in a file past i_size 388 * until the file is removed. Warn in the unlikely case of 389 * allocation failure. 390 */ 391 pr_warn_once("HugeTLB: unable to allocate vma specific lock\n"); 392 return; 393 } 394 395 kref_init(&vma_lock->refs); 396 init_rwsem(&vma_lock->rw_sema); 397 vma_lock->vma = vma; 398 vma->vm_private_data = vma_lock; 399 } 400 401 /* Helper that removes a struct file_region from the resv_map cache and returns 402 * it for use. 403 */ 404 static struct file_region * 405 get_file_region_entry_from_cache(struct resv_map *resv, long from, long to) 406 { 407 struct file_region *nrg; 408 409 VM_BUG_ON(resv->region_cache_count <= 0); 410 411 resv->region_cache_count--; 412 nrg = list_first_entry(&resv->region_cache, struct file_region, link); 413 list_del(&nrg->link); 414 415 nrg->from = from; 416 nrg->to = to; 417 418 return nrg; 419 } 420 421 static void copy_hugetlb_cgroup_uncharge_info(struct file_region *nrg, 422 struct file_region *rg) 423 { 424 #ifdef CONFIG_CGROUP_HUGETLB 425 nrg->reservation_counter = rg->reservation_counter; 426 nrg->css = rg->css; 427 if (rg->css) 428 css_get(rg->css); 429 #endif 430 } 431 432 /* Helper that records hugetlb_cgroup uncharge info. */ 433 static void record_hugetlb_cgroup_uncharge_info(struct hugetlb_cgroup *h_cg, 434 struct hstate *h, 435 struct resv_map *resv, 436 struct file_region *nrg) 437 { 438 #ifdef CONFIG_CGROUP_HUGETLB 439 if (h_cg) { 440 nrg->reservation_counter = 441 &h_cg->rsvd_hugepage[hstate_index(h)]; 442 nrg->css = &h_cg->css; 443 /* 444 * The caller will hold exactly one h_cg->css reference for the 445 * whole contiguous reservation region. But this area might be 446 * scattered when there are already some file_regions reside in 447 * it. As a result, many file_regions may share only one css 448 * reference. In order to ensure that one file_region must hold 449 * exactly one h_cg->css reference, we should do css_get for 450 * each file_region and leave the reference held by caller 451 * untouched. 452 */ 453 css_get(&h_cg->css); 454 if (!resv->pages_per_hpage) 455 resv->pages_per_hpage = pages_per_huge_page(h); 456 /* pages_per_hpage should be the same for all entries in 457 * a resv_map. 458 */ 459 VM_BUG_ON(resv->pages_per_hpage != pages_per_huge_page(h)); 460 } else { 461 nrg->reservation_counter = NULL; 462 nrg->css = NULL; 463 } 464 #endif 465 } 466 467 static void put_uncharge_info(struct file_region *rg) 468 { 469 #ifdef CONFIG_CGROUP_HUGETLB 470 if (rg->css) 471 css_put(rg->css); 472 #endif 473 } 474 475 static bool has_same_uncharge_info(struct file_region *rg, 476 struct file_region *org) 477 { 478 #ifdef CONFIG_CGROUP_HUGETLB 479 return rg->reservation_counter == org->reservation_counter && 480 rg->css == org->css; 481 482 #else 483 return true; 484 #endif 485 } 486 487 static void coalesce_file_region(struct resv_map *resv, struct file_region *rg) 488 { 489 struct file_region *nrg, *prg; 490 491 prg = list_prev_entry(rg, link); 492 if (&prg->link != &resv->regions && prg->to == rg->from && 493 has_same_uncharge_info(prg, rg)) { 494 prg->to = rg->to; 495 496 list_del(&rg->link); 497 put_uncharge_info(rg); 498 kfree(rg); 499 500 rg = prg; 501 } 502 503 nrg = list_next_entry(rg, link); 504 if (&nrg->link != &resv->regions && nrg->from == rg->to && 505 has_same_uncharge_info(nrg, rg)) { 506 nrg->from = rg->from; 507 508 list_del(&rg->link); 509 put_uncharge_info(rg); 510 kfree(rg); 511 } 512 } 513 514 static inline long 515 hugetlb_resv_map_add(struct resv_map *map, struct list_head *rg, long from, 516 long to, struct hstate *h, struct hugetlb_cgroup *cg, 517 long *regions_needed) 518 { 519 struct file_region *nrg; 520 521 if (!regions_needed) { 522 nrg = get_file_region_entry_from_cache(map, from, to); 523 record_hugetlb_cgroup_uncharge_info(cg, h, map, nrg); 524 list_add(&nrg->link, rg); 525 coalesce_file_region(map, nrg); 526 } else 527 *regions_needed += 1; 528 529 return to - from; 530 } 531 532 /* 533 * Must be called with resv->lock held. 534 * 535 * Calling this with regions_needed != NULL will count the number of pages 536 * to be added but will not modify the linked list. And regions_needed will 537 * indicate the number of file_regions needed in the cache to carry out to add 538 * the regions for this range. 539 */ 540 static long add_reservation_in_range(struct resv_map *resv, long f, long t, 541 struct hugetlb_cgroup *h_cg, 542 struct hstate *h, long *regions_needed) 543 { 544 long add = 0; 545 struct list_head *head = &resv->regions; 546 long last_accounted_offset = f; 547 struct file_region *iter, *trg = NULL; 548 struct list_head *rg = NULL; 549 550 if (regions_needed) 551 *regions_needed = 0; 552 553 /* In this loop, we essentially handle an entry for the range 554 * [last_accounted_offset, iter->from), at every iteration, with some 555 * bounds checking. 556 */ 557 list_for_each_entry_safe(iter, trg, head, link) { 558 /* Skip irrelevant regions that start before our range. */ 559 if (iter->from < f) { 560 /* If this region ends after the last accounted offset, 561 * then we need to update last_accounted_offset. 562 */ 563 if (iter->to > last_accounted_offset) 564 last_accounted_offset = iter->to; 565 continue; 566 } 567 568 /* When we find a region that starts beyond our range, we've 569 * finished. 570 */ 571 if (iter->from >= t) { 572 rg = iter->link.prev; 573 break; 574 } 575 576 /* Add an entry for last_accounted_offset -> iter->from, and 577 * update last_accounted_offset. 578 */ 579 if (iter->from > last_accounted_offset) 580 add += hugetlb_resv_map_add(resv, iter->link.prev, 581 last_accounted_offset, 582 iter->from, h, h_cg, 583 regions_needed); 584 585 last_accounted_offset = iter->to; 586 } 587 588 /* Handle the case where our range extends beyond 589 * last_accounted_offset. 590 */ 591 if (!rg) 592 rg = head->prev; 593 if (last_accounted_offset < t) 594 add += hugetlb_resv_map_add(resv, rg, last_accounted_offset, 595 t, h, h_cg, regions_needed); 596 597 return add; 598 } 599 600 /* Must be called with resv->lock acquired. Will drop lock to allocate entries. 601 */ 602 static int allocate_file_region_entries(struct resv_map *resv, 603 int regions_needed) 604 __must_hold(&resv->lock) 605 { 606 LIST_HEAD(allocated_regions); 607 int to_allocate = 0, i = 0; 608 struct file_region *trg = NULL, *rg = NULL; 609 610 VM_BUG_ON(regions_needed < 0); 611 612 /* 613 * Check for sufficient descriptors in the cache to accommodate 614 * the number of in progress add operations plus regions_needed. 615 * 616 * This is a while loop because when we drop the lock, some other call 617 * to region_add or region_del may have consumed some region_entries, 618 * so we keep looping here until we finally have enough entries for 619 * (adds_in_progress + regions_needed). 620 */ 621 while (resv->region_cache_count < 622 (resv->adds_in_progress + regions_needed)) { 623 to_allocate = resv->adds_in_progress + regions_needed - 624 resv->region_cache_count; 625 626 /* At this point, we should have enough entries in the cache 627 * for all the existing adds_in_progress. We should only be 628 * needing to allocate for regions_needed. 629 */ 630 VM_BUG_ON(resv->region_cache_count < resv->adds_in_progress); 631 632 spin_unlock(&resv->lock); 633 for (i = 0; i < to_allocate; i++) { 634 trg = kmalloc(sizeof(*trg), GFP_KERNEL); 635 if (!trg) 636 goto out_of_memory; 637 list_add(&trg->link, &allocated_regions); 638 } 639 640 spin_lock(&resv->lock); 641 642 list_splice(&allocated_regions, &resv->region_cache); 643 resv->region_cache_count += to_allocate; 644 } 645 646 return 0; 647 648 out_of_memory: 649 list_for_each_entry_safe(rg, trg, &allocated_regions, link) { 650 list_del(&rg->link); 651 kfree(rg); 652 } 653 return -ENOMEM; 654 } 655 656 /* 657 * Add the huge page range represented by [f, t) to the reserve 658 * map. Regions will be taken from the cache to fill in this range. 659 * Sufficient regions should exist in the cache due to the previous 660 * call to region_chg with the same range, but in some cases the cache will not 661 * have sufficient entries due to races with other code doing region_add or 662 * region_del. The extra needed entries will be allocated. 663 * 664 * regions_needed is the out value provided by a previous call to region_chg. 665 * 666 * Return the number of new huge pages added to the map. This number is greater 667 * than or equal to zero. If file_region entries needed to be allocated for 668 * this operation and we were not able to allocate, it returns -ENOMEM. 669 * region_add of regions of length 1 never allocate file_regions and cannot 670 * fail; region_chg will always allocate at least 1 entry and a region_add for 671 * 1 page will only require at most 1 entry. 672 */ 673 static long region_add(struct resv_map *resv, long f, long t, 674 long in_regions_needed, struct hstate *h, 675 struct hugetlb_cgroup *h_cg) 676 { 677 long add = 0, actual_regions_needed = 0; 678 679 spin_lock(&resv->lock); 680 retry: 681 682 /* Count how many regions are actually needed to execute this add. */ 683 add_reservation_in_range(resv, f, t, NULL, NULL, 684 &actual_regions_needed); 685 686 /* 687 * Check for sufficient descriptors in the cache to accommodate 688 * this add operation. Note that actual_regions_needed may be greater 689 * than in_regions_needed, as the resv_map may have been modified since 690 * the region_chg call. In this case, we need to make sure that we 691 * allocate extra entries, such that we have enough for all the 692 * existing adds_in_progress, plus the excess needed for this 693 * operation. 694 */ 695 if (actual_regions_needed > in_regions_needed && 696 resv->region_cache_count < 697 resv->adds_in_progress + 698 (actual_regions_needed - in_regions_needed)) { 699 /* region_add operation of range 1 should never need to 700 * allocate file_region entries. 701 */ 702 VM_BUG_ON(t - f <= 1); 703 704 if (allocate_file_region_entries( 705 resv, actual_regions_needed - in_regions_needed)) { 706 return -ENOMEM; 707 } 708 709 goto retry; 710 } 711 712 add = add_reservation_in_range(resv, f, t, h_cg, h, NULL); 713 714 resv->adds_in_progress -= in_regions_needed; 715 716 spin_unlock(&resv->lock); 717 return add; 718 } 719 720 /* 721 * Examine the existing reserve map and determine how many 722 * huge pages in the specified range [f, t) are NOT currently 723 * represented. This routine is called before a subsequent 724 * call to region_add that will actually modify the reserve 725 * map to add the specified range [f, t). region_chg does 726 * not change the number of huge pages represented by the 727 * map. A number of new file_region structures is added to the cache as a 728 * placeholder, for the subsequent region_add call to use. At least 1 729 * file_region structure is added. 730 * 731 * out_regions_needed is the number of regions added to the 732 * resv->adds_in_progress. This value needs to be provided to a follow up call 733 * to region_add or region_abort for proper accounting. 734 * 735 * Returns the number of huge pages that need to be added to the existing 736 * reservation map for the range [f, t). This number is greater or equal to 737 * zero. -ENOMEM is returned if a new file_region structure or cache entry 738 * is needed and can not be allocated. 739 */ 740 static long region_chg(struct resv_map *resv, long f, long t, 741 long *out_regions_needed) 742 { 743 long chg = 0; 744 745 spin_lock(&resv->lock); 746 747 /* Count how many hugepages in this range are NOT represented. */ 748 chg = add_reservation_in_range(resv, f, t, NULL, NULL, 749 out_regions_needed); 750 751 if (*out_regions_needed == 0) 752 *out_regions_needed = 1; 753 754 if (allocate_file_region_entries(resv, *out_regions_needed)) 755 return -ENOMEM; 756 757 resv->adds_in_progress += *out_regions_needed; 758 759 spin_unlock(&resv->lock); 760 return chg; 761 } 762 763 /* 764 * Abort the in progress add operation. The adds_in_progress field 765 * of the resv_map keeps track of the operations in progress between 766 * calls to region_chg and region_add. Operations are sometimes 767 * aborted after the call to region_chg. In such cases, region_abort 768 * is called to decrement the adds_in_progress counter. regions_needed 769 * is the value returned by the region_chg call, it is used to decrement 770 * the adds_in_progress counter. 771 * 772 * NOTE: The range arguments [f, t) are not needed or used in this 773 * routine. They are kept to make reading the calling code easier as 774 * arguments will match the associated region_chg call. 775 */ 776 static void region_abort(struct resv_map *resv, long f, long t, 777 long regions_needed) 778 { 779 spin_lock(&resv->lock); 780 VM_BUG_ON(!resv->region_cache_count); 781 resv->adds_in_progress -= regions_needed; 782 spin_unlock(&resv->lock); 783 } 784 785 /* 786 * Delete the specified range [f, t) from the reserve map. If the 787 * t parameter is LONG_MAX, this indicates that ALL regions after f 788 * should be deleted. Locate the regions which intersect [f, t) 789 * and either trim, delete or split the existing regions. 790 * 791 * Returns the number of huge pages deleted from the reserve map. 792 * In the normal case, the return value is zero or more. In the 793 * case where a region must be split, a new region descriptor must 794 * be allocated. If the allocation fails, -ENOMEM will be returned. 795 * NOTE: If the parameter t == LONG_MAX, then we will never split 796 * a region and possibly return -ENOMEM. Callers specifying 797 * t == LONG_MAX do not need to check for -ENOMEM error. 798 */ 799 static long region_del(struct resv_map *resv, long f, long t) 800 { 801 struct list_head *head = &resv->regions; 802 struct file_region *rg, *trg; 803 struct file_region *nrg = NULL; 804 long del = 0; 805 806 retry: 807 spin_lock(&resv->lock); 808 list_for_each_entry_safe(rg, trg, head, link) { 809 /* 810 * Skip regions before the range to be deleted. file_region 811 * ranges are normally of the form [from, to). However, there 812 * may be a "placeholder" entry in the map which is of the form 813 * (from, to) with from == to. Check for placeholder entries 814 * at the beginning of the range to be deleted. 815 */ 816 if (rg->to <= f && (rg->to != rg->from || rg->to != f)) 817 continue; 818 819 if (rg->from >= t) 820 break; 821 822 if (f > rg->from && t < rg->to) { /* Must split region */ 823 /* 824 * Check for an entry in the cache before dropping 825 * lock and attempting allocation. 826 */ 827 if (!nrg && 828 resv->region_cache_count > resv->adds_in_progress) { 829 nrg = list_first_entry(&resv->region_cache, 830 struct file_region, 831 link); 832 list_del(&nrg->link); 833 resv->region_cache_count--; 834 } 835 836 if (!nrg) { 837 spin_unlock(&resv->lock); 838 nrg = kmalloc(sizeof(*nrg), GFP_KERNEL); 839 if (!nrg) 840 return -ENOMEM; 841 goto retry; 842 } 843 844 del += t - f; 845 hugetlb_cgroup_uncharge_file_region( 846 resv, rg, t - f, false); 847 848 /* New entry for end of split region */ 849 nrg->from = t; 850 nrg->to = rg->to; 851 852 copy_hugetlb_cgroup_uncharge_info(nrg, rg); 853 854 INIT_LIST_HEAD(&nrg->link); 855 856 /* Original entry is trimmed */ 857 rg->to = f; 858 859 list_add(&nrg->link, &rg->link); 860 nrg = NULL; 861 break; 862 } 863 864 if (f <= rg->from && t >= rg->to) { /* Remove entire region */ 865 del += rg->to - rg->from; 866 hugetlb_cgroup_uncharge_file_region(resv, rg, 867 rg->to - rg->from, true); 868 list_del(&rg->link); 869 kfree(rg); 870 continue; 871 } 872 873 if (f <= rg->from) { /* Trim beginning of region */ 874 hugetlb_cgroup_uncharge_file_region(resv, rg, 875 t - rg->from, false); 876 877 del += t - rg->from; 878 rg->from = t; 879 } else { /* Trim end of region */ 880 hugetlb_cgroup_uncharge_file_region(resv, rg, 881 rg->to - f, false); 882 883 del += rg->to - f; 884 rg->to = f; 885 } 886 } 887 888 spin_unlock(&resv->lock); 889 kfree(nrg); 890 return del; 891 } 892 893 /* 894 * A rare out of memory error was encountered which prevented removal of 895 * the reserve map region for a page. The huge page itself was free'ed 896 * and removed from the page cache. This routine will adjust the subpool 897 * usage count, and the global reserve count if needed. By incrementing 898 * these counts, the reserve map entry which could not be deleted will 899 * appear as a "reserved" entry instead of simply dangling with incorrect 900 * counts. 901 */ 902 void hugetlb_fix_reserve_counts(struct inode *inode) 903 { 904 struct hugepage_subpool *spool = subpool_inode(inode); 905 long rsv_adjust; 906 bool reserved = false; 907 908 rsv_adjust = hugepage_subpool_get_pages(spool, 1); 909 if (rsv_adjust > 0) { 910 struct hstate *h = hstate_inode(inode); 911 912 if (!hugetlb_acct_memory(h, 1)) 913 reserved = true; 914 } else if (!rsv_adjust) { 915 reserved = true; 916 } 917 918 if (!reserved) 919 pr_warn("hugetlb: Huge Page Reserved count may go negative.\n"); 920 } 921 922 /* 923 * Count and return the number of huge pages in the reserve map 924 * that intersect with the range [f, t). 925 */ 926 static long region_count(struct resv_map *resv, long f, long t) 927 { 928 struct list_head *head = &resv->regions; 929 struct file_region *rg; 930 long chg = 0; 931 932 spin_lock(&resv->lock); 933 /* Locate each segment we overlap with, and count that overlap. */ 934 list_for_each_entry(rg, head, link) { 935 long seg_from; 936 long seg_to; 937 938 if (rg->to <= f) 939 continue; 940 if (rg->from >= t) 941 break; 942 943 seg_from = max(rg->from, f); 944 seg_to = min(rg->to, t); 945 946 chg += seg_to - seg_from; 947 } 948 spin_unlock(&resv->lock); 949 950 return chg; 951 } 952 953 /* 954 * Convert the address within this vma to the page offset within 955 * the mapping, in pagecache page units; huge pages here. 956 */ 957 static pgoff_t vma_hugecache_offset(struct hstate *h, 958 struct vm_area_struct *vma, unsigned long address) 959 { 960 return ((address - vma->vm_start) >> huge_page_shift(h)) + 961 (vma->vm_pgoff >> huge_page_order(h)); 962 } 963 964 pgoff_t linear_hugepage_index(struct vm_area_struct *vma, 965 unsigned long address) 966 { 967 return vma_hugecache_offset(hstate_vma(vma), vma, address); 968 } 969 EXPORT_SYMBOL_GPL(linear_hugepage_index); 970 971 /* 972 * Return the size of the pages allocated when backing a VMA. In the majority 973 * cases this will be same size as used by the page table entries. 974 */ 975 unsigned long vma_kernel_pagesize(struct vm_area_struct *vma) 976 { 977 if (vma->vm_ops && vma->vm_ops->pagesize) 978 return vma->vm_ops->pagesize(vma); 979 return PAGE_SIZE; 980 } 981 EXPORT_SYMBOL_GPL(vma_kernel_pagesize); 982 983 /* 984 * Return the page size being used by the MMU to back a VMA. In the majority 985 * of cases, the page size used by the kernel matches the MMU size. On 986 * architectures where it differs, an architecture-specific 'strong' 987 * version of this symbol is required. 988 */ 989 __weak unsigned long vma_mmu_pagesize(struct vm_area_struct *vma) 990 { 991 return vma_kernel_pagesize(vma); 992 } 993 994 /* 995 * Flags for MAP_PRIVATE reservations. These are stored in the bottom 996 * bits of the reservation map pointer, which are always clear due to 997 * alignment. 998 */ 999 #define HPAGE_RESV_OWNER (1UL << 0) 1000 #define HPAGE_RESV_UNMAPPED (1UL << 1) 1001 #define HPAGE_RESV_MASK (HPAGE_RESV_OWNER | HPAGE_RESV_UNMAPPED) 1002 1003 /* 1004 * These helpers are used to track how many pages are reserved for 1005 * faults in a MAP_PRIVATE mapping. Only the process that called mmap() 1006 * is guaranteed to have their future faults succeed. 1007 * 1008 * With the exception of hugetlb_dup_vma_private() which is called at fork(), 1009 * the reserve counters are updated with the hugetlb_lock held. It is safe 1010 * to reset the VMA at fork() time as it is not in use yet and there is no 1011 * chance of the global counters getting corrupted as a result of the values. 1012 * 1013 * The private mapping reservation is represented in a subtly different 1014 * manner to a shared mapping. A shared mapping has a region map associated 1015 * with the underlying file, this region map represents the backing file 1016 * pages which have ever had a reservation assigned which this persists even 1017 * after the page is instantiated. A private mapping has a region map 1018 * associated with the original mmap which is attached to all VMAs which 1019 * reference it, this region map represents those offsets which have consumed 1020 * reservation ie. where pages have been instantiated. 1021 */ 1022 static unsigned long get_vma_private_data(struct vm_area_struct *vma) 1023 { 1024 return (unsigned long)vma->vm_private_data; 1025 } 1026 1027 static void set_vma_private_data(struct vm_area_struct *vma, 1028 unsigned long value) 1029 { 1030 vma->vm_private_data = (void *)value; 1031 } 1032 1033 static void 1034 resv_map_set_hugetlb_cgroup_uncharge_info(struct resv_map *resv_map, 1035 struct hugetlb_cgroup *h_cg, 1036 struct hstate *h) 1037 { 1038 #ifdef CONFIG_CGROUP_HUGETLB 1039 if (!h_cg || !h) { 1040 resv_map->reservation_counter = NULL; 1041 resv_map->pages_per_hpage = 0; 1042 resv_map->css = NULL; 1043 } else { 1044 resv_map->reservation_counter = 1045 &h_cg->rsvd_hugepage[hstate_index(h)]; 1046 resv_map->pages_per_hpage = pages_per_huge_page(h); 1047 resv_map->css = &h_cg->css; 1048 } 1049 #endif 1050 } 1051 1052 struct resv_map *resv_map_alloc(void) 1053 { 1054 struct resv_map *resv_map = kmalloc(sizeof(*resv_map), GFP_KERNEL); 1055 struct file_region *rg = kmalloc(sizeof(*rg), GFP_KERNEL); 1056 1057 if (!resv_map || !rg) { 1058 kfree(resv_map); 1059 kfree(rg); 1060 return NULL; 1061 } 1062 1063 kref_init(&resv_map->refs); 1064 spin_lock_init(&resv_map->lock); 1065 INIT_LIST_HEAD(&resv_map->regions); 1066 1067 resv_map->adds_in_progress = 0; 1068 /* 1069 * Initialize these to 0. On shared mappings, 0's here indicate these 1070 * fields don't do cgroup accounting. On private mappings, these will be 1071 * re-initialized to the proper values, to indicate that hugetlb cgroup 1072 * reservations are to be un-charged from here. 1073 */ 1074 resv_map_set_hugetlb_cgroup_uncharge_info(resv_map, NULL, NULL); 1075 1076 INIT_LIST_HEAD(&resv_map->region_cache); 1077 list_add(&rg->link, &resv_map->region_cache); 1078 resv_map->region_cache_count = 1; 1079 1080 return resv_map; 1081 } 1082 1083 void resv_map_release(struct kref *ref) 1084 { 1085 struct resv_map *resv_map = container_of(ref, struct resv_map, refs); 1086 struct list_head *head = &resv_map->region_cache; 1087 struct file_region *rg, *trg; 1088 1089 /* Clear out any active regions before we release the map. */ 1090 region_del(resv_map, 0, LONG_MAX); 1091 1092 /* ... and any entries left in the cache */ 1093 list_for_each_entry_safe(rg, trg, head, link) { 1094 list_del(&rg->link); 1095 kfree(rg); 1096 } 1097 1098 VM_BUG_ON(resv_map->adds_in_progress); 1099 1100 kfree(resv_map); 1101 } 1102 1103 static inline struct resv_map *inode_resv_map(struct inode *inode) 1104 { 1105 /* 1106 * At inode evict time, i_mapping may not point to the original 1107 * address space within the inode. This original address space 1108 * contains the pointer to the resv_map. So, always use the 1109 * address space embedded within the inode. 1110 * The VERY common case is inode->mapping == &inode->i_data but, 1111 * this may not be true for device special inodes. 1112 */ 1113 return (struct resv_map *)(&inode->i_data)->private_data; 1114 } 1115 1116 static struct resv_map *vma_resv_map(struct vm_area_struct *vma) 1117 { 1118 VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma); 1119 if (vma->vm_flags & VM_MAYSHARE) { 1120 struct address_space *mapping = vma->vm_file->f_mapping; 1121 struct inode *inode = mapping->host; 1122 1123 return inode_resv_map(inode); 1124 1125 } else { 1126 return (struct resv_map *)(get_vma_private_data(vma) & 1127 ~HPAGE_RESV_MASK); 1128 } 1129 } 1130 1131 static void set_vma_resv_map(struct vm_area_struct *vma, struct resv_map *map) 1132 { 1133 VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma); 1134 VM_BUG_ON_VMA(vma->vm_flags & VM_MAYSHARE, vma); 1135 1136 set_vma_private_data(vma, (get_vma_private_data(vma) & 1137 HPAGE_RESV_MASK) | (unsigned long)map); 1138 } 1139 1140 static void set_vma_resv_flags(struct vm_area_struct *vma, unsigned long flags) 1141 { 1142 VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma); 1143 VM_BUG_ON_VMA(vma->vm_flags & VM_MAYSHARE, vma); 1144 1145 set_vma_private_data(vma, get_vma_private_data(vma) | flags); 1146 } 1147 1148 static int is_vma_resv_set(struct vm_area_struct *vma, unsigned long flag) 1149 { 1150 VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma); 1151 1152 return (get_vma_private_data(vma) & flag) != 0; 1153 } 1154 1155 void hugetlb_dup_vma_private(struct vm_area_struct *vma) 1156 { 1157 VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma); 1158 /* 1159 * Clear vm_private_data 1160 * - For shared mappings this is a per-vma semaphore that may be 1161 * allocated in a subsequent call to hugetlb_vm_op_open. 1162 * Before clearing, make sure pointer is not associated with vma 1163 * as this will leak the structure. This is the case when called 1164 * via clear_vma_resv_huge_pages() and hugetlb_vm_op_open has already 1165 * been called to allocate a new structure. 1166 * - For MAP_PRIVATE mappings, this is the reserve map which does 1167 * not apply to children. Faults generated by the children are 1168 * not guaranteed to succeed, even if read-only. 1169 */ 1170 if (vma->vm_flags & VM_MAYSHARE) { 1171 struct hugetlb_vma_lock *vma_lock = vma->vm_private_data; 1172 1173 if (vma_lock && vma_lock->vma != vma) 1174 vma->vm_private_data = NULL; 1175 } else 1176 vma->vm_private_data = NULL; 1177 } 1178 1179 /* 1180 * Reset and decrement one ref on hugepage private reservation. 1181 * Called with mm->mmap_lock writer semaphore held. 1182 * This function should be only used by move_vma() and operate on 1183 * same sized vma. It should never come here with last ref on the 1184 * reservation. 1185 */ 1186 void clear_vma_resv_huge_pages(struct vm_area_struct *vma) 1187 { 1188 /* 1189 * Clear the old hugetlb private page reservation. 1190 * It has already been transferred to new_vma. 1191 * 1192 * During a mremap() operation of a hugetlb vma we call move_vma() 1193 * which copies vma into new_vma and unmaps vma. After the copy 1194 * operation both new_vma and vma share a reference to the resv_map 1195 * struct, and at that point vma is about to be unmapped. We don't 1196 * want to return the reservation to the pool at unmap of vma because 1197 * the reservation still lives on in new_vma, so simply decrement the 1198 * ref here and remove the resv_map reference from this vma. 1199 */ 1200 struct resv_map *reservations = vma_resv_map(vma); 1201 1202 if (reservations && is_vma_resv_set(vma, HPAGE_RESV_OWNER)) { 1203 resv_map_put_hugetlb_cgroup_uncharge_info(reservations); 1204 kref_put(&reservations->refs, resv_map_release); 1205 } 1206 1207 hugetlb_dup_vma_private(vma); 1208 } 1209 1210 /* Returns true if the VMA has associated reserve pages */ 1211 static bool vma_has_reserves(struct vm_area_struct *vma, long chg) 1212 { 1213 if (vma->vm_flags & VM_NORESERVE) { 1214 /* 1215 * This address is already reserved by other process(chg == 0), 1216 * so, we should decrement reserved count. Without decrementing, 1217 * reserve count remains after releasing inode, because this 1218 * allocated page will go into page cache and is regarded as 1219 * coming from reserved pool in releasing step. Currently, we 1220 * don't have any other solution to deal with this situation 1221 * properly, so add work-around here. 1222 */ 1223 if (vma->vm_flags & VM_MAYSHARE && chg == 0) 1224 return true; 1225 else 1226 return false; 1227 } 1228 1229 /* Shared mappings always use reserves */ 1230 if (vma->vm_flags & VM_MAYSHARE) { 1231 /* 1232 * We know VM_NORESERVE is not set. Therefore, there SHOULD 1233 * be a region map for all pages. The only situation where 1234 * there is no region map is if a hole was punched via 1235 * fallocate. In this case, there really are no reserves to 1236 * use. This situation is indicated if chg != 0. 1237 */ 1238 if (chg) 1239 return false; 1240 else 1241 return true; 1242 } 1243 1244 /* 1245 * Only the process that called mmap() has reserves for 1246 * private mappings. 1247 */ 1248 if (is_vma_resv_set(vma, HPAGE_RESV_OWNER)) { 1249 /* 1250 * Like the shared case above, a hole punch or truncate 1251 * could have been performed on the private mapping. 1252 * Examine the value of chg to determine if reserves 1253 * actually exist or were previously consumed. 1254 * Very Subtle - The value of chg comes from a previous 1255 * call to vma_needs_reserves(). The reserve map for 1256 * private mappings has different (opposite) semantics 1257 * than that of shared mappings. vma_needs_reserves() 1258 * has already taken this difference in semantics into 1259 * account. Therefore, the meaning of chg is the same 1260 * as in the shared case above. Code could easily be 1261 * combined, but keeping it separate draws attention to 1262 * subtle differences. 1263 */ 1264 if (chg) 1265 return false; 1266 else 1267 return true; 1268 } 1269 1270 return false; 1271 } 1272 1273 static void enqueue_hugetlb_folio(struct hstate *h, struct folio *folio) 1274 { 1275 int nid = folio_nid(folio); 1276 1277 lockdep_assert_held(&hugetlb_lock); 1278 VM_BUG_ON_FOLIO(folio_ref_count(folio), folio); 1279 1280 list_move(&folio->lru, &h->hugepage_freelists[nid]); 1281 h->free_huge_pages++; 1282 h->free_huge_pages_node[nid]++; 1283 folio_set_hugetlb_freed(folio); 1284 } 1285 1286 static struct folio *dequeue_hugetlb_folio_node_exact(struct hstate *h, 1287 int nid) 1288 { 1289 struct folio *folio; 1290 bool pin = !!(current->flags & PF_MEMALLOC_PIN); 1291 1292 lockdep_assert_held(&hugetlb_lock); 1293 list_for_each_entry(folio, &h->hugepage_freelists[nid], lru) { 1294 if (pin && !folio_is_longterm_pinnable(folio)) 1295 continue; 1296 1297 if (folio_test_hwpoison(folio)) 1298 continue; 1299 1300 list_move(&folio->lru, &h->hugepage_activelist); 1301 folio_ref_unfreeze(folio, 1); 1302 folio_clear_hugetlb_freed(folio); 1303 h->free_huge_pages--; 1304 h->free_huge_pages_node[nid]--; 1305 return folio; 1306 } 1307 1308 return NULL; 1309 } 1310 1311 static struct folio *dequeue_hugetlb_folio_nodemask(struct hstate *h, gfp_t gfp_mask, 1312 int nid, nodemask_t *nmask) 1313 { 1314 unsigned int cpuset_mems_cookie; 1315 struct zonelist *zonelist; 1316 struct zone *zone; 1317 struct zoneref *z; 1318 int node = NUMA_NO_NODE; 1319 1320 zonelist = node_zonelist(nid, gfp_mask); 1321 1322 retry_cpuset: 1323 cpuset_mems_cookie = read_mems_allowed_begin(); 1324 for_each_zone_zonelist_nodemask(zone, z, zonelist, gfp_zone(gfp_mask), nmask) { 1325 struct folio *folio; 1326 1327 if (!cpuset_zone_allowed(zone, gfp_mask)) 1328 continue; 1329 /* 1330 * no need to ask again on the same node. Pool is node rather than 1331 * zone aware 1332 */ 1333 if (zone_to_nid(zone) == node) 1334 continue; 1335 node = zone_to_nid(zone); 1336 1337 folio = dequeue_hugetlb_folio_node_exact(h, node); 1338 if (folio) 1339 return folio; 1340 } 1341 if (unlikely(read_mems_allowed_retry(cpuset_mems_cookie))) 1342 goto retry_cpuset; 1343 1344 return NULL; 1345 } 1346 1347 static unsigned long available_huge_pages(struct hstate *h) 1348 { 1349 return h->free_huge_pages - h->resv_huge_pages; 1350 } 1351 1352 static struct folio *dequeue_hugetlb_folio_vma(struct hstate *h, 1353 struct vm_area_struct *vma, 1354 unsigned long address, int avoid_reserve, 1355 long chg) 1356 { 1357 struct folio *folio = NULL; 1358 struct mempolicy *mpol; 1359 gfp_t gfp_mask; 1360 nodemask_t *nodemask; 1361 int nid; 1362 1363 /* 1364 * A child process with MAP_PRIVATE mappings created by their parent 1365 * have no page reserves. This check ensures that reservations are 1366 * not "stolen". The child may still get SIGKILLed 1367 */ 1368 if (!vma_has_reserves(vma, chg) && !available_huge_pages(h)) 1369 goto err; 1370 1371 /* If reserves cannot be used, ensure enough pages are in the pool */ 1372 if (avoid_reserve && !available_huge_pages(h)) 1373 goto err; 1374 1375 gfp_mask = htlb_alloc_mask(h); 1376 nid = huge_node(vma, address, gfp_mask, &mpol, &nodemask); 1377 1378 if (mpol_is_preferred_many(mpol)) { 1379 folio = dequeue_hugetlb_folio_nodemask(h, gfp_mask, 1380 nid, nodemask); 1381 1382 /* Fallback to all nodes if page==NULL */ 1383 nodemask = NULL; 1384 } 1385 1386 if (!folio) 1387 folio = dequeue_hugetlb_folio_nodemask(h, gfp_mask, 1388 nid, nodemask); 1389 1390 if (folio && !avoid_reserve && vma_has_reserves(vma, chg)) { 1391 folio_set_hugetlb_restore_reserve(folio); 1392 h->resv_huge_pages--; 1393 } 1394 1395 mpol_cond_put(mpol); 1396 return folio; 1397 1398 err: 1399 return NULL; 1400 } 1401 1402 /* 1403 * common helper functions for hstate_next_node_to_{alloc|free}. 1404 * We may have allocated or freed a huge page based on a different 1405 * nodes_allowed previously, so h->next_node_to_{alloc|free} might 1406 * be outside of *nodes_allowed. Ensure that we use an allowed 1407 * node for alloc or free. 1408 */ 1409 static int next_node_allowed(int nid, nodemask_t *nodes_allowed) 1410 { 1411 nid = next_node_in(nid, *nodes_allowed); 1412 VM_BUG_ON(nid >= MAX_NUMNODES); 1413 1414 return nid; 1415 } 1416 1417 static int get_valid_node_allowed(int nid, nodemask_t *nodes_allowed) 1418 { 1419 if (!node_isset(nid, *nodes_allowed)) 1420 nid = next_node_allowed(nid, nodes_allowed); 1421 return nid; 1422 } 1423 1424 /* 1425 * returns the previously saved node ["this node"] from which to 1426 * allocate a persistent huge page for the pool and advance the 1427 * next node from which to allocate, handling wrap at end of node 1428 * mask. 1429 */ 1430 static int hstate_next_node_to_alloc(struct hstate *h, 1431 nodemask_t *nodes_allowed) 1432 { 1433 int nid; 1434 1435 VM_BUG_ON(!nodes_allowed); 1436 1437 nid = get_valid_node_allowed(h->next_nid_to_alloc, nodes_allowed); 1438 h->next_nid_to_alloc = next_node_allowed(nid, nodes_allowed); 1439 1440 return nid; 1441 } 1442 1443 /* 1444 * helper for remove_pool_huge_page() - return the previously saved 1445 * node ["this node"] from which to free a huge page. Advance the 1446 * next node id whether or not we find a free huge page to free so 1447 * that the next attempt to free addresses the next node. 1448 */ 1449 static int hstate_next_node_to_free(struct hstate *h, nodemask_t *nodes_allowed) 1450 { 1451 int nid; 1452 1453 VM_BUG_ON(!nodes_allowed); 1454 1455 nid = get_valid_node_allowed(h->next_nid_to_free, nodes_allowed); 1456 h->next_nid_to_free = next_node_allowed(nid, nodes_allowed); 1457 1458 return nid; 1459 } 1460 1461 #define for_each_node_mask_to_alloc(hs, nr_nodes, node, mask) \ 1462 for (nr_nodes = nodes_weight(*mask); \ 1463 nr_nodes > 0 && \ 1464 ((node = hstate_next_node_to_alloc(hs, mask)) || 1); \ 1465 nr_nodes--) 1466 1467 #define for_each_node_mask_to_free(hs, nr_nodes, node, mask) \ 1468 for (nr_nodes = nodes_weight(*mask); \ 1469 nr_nodes > 0 && \ 1470 ((node = hstate_next_node_to_free(hs, mask)) || 1); \ 1471 nr_nodes--) 1472 1473 /* used to demote non-gigantic_huge pages as well */ 1474 static void __destroy_compound_gigantic_folio(struct folio *folio, 1475 unsigned int order, bool demote) 1476 { 1477 int i; 1478 int nr_pages = 1 << order; 1479 struct page *p; 1480 1481 atomic_set(&folio->_entire_mapcount, 0); 1482 atomic_set(&folio->_nr_pages_mapped, 0); 1483 atomic_set(&folio->_pincount, 0); 1484 1485 for (i = 1; i < nr_pages; i++) { 1486 p = folio_page(folio, i); 1487 p->flags &= ~PAGE_FLAGS_CHECK_AT_FREE; 1488 p->mapping = NULL; 1489 clear_compound_head(p); 1490 if (!demote) 1491 set_page_refcounted(p); 1492 } 1493 1494 __folio_clear_head(folio); 1495 } 1496 1497 static void destroy_compound_hugetlb_folio_for_demote(struct folio *folio, 1498 unsigned int order) 1499 { 1500 __destroy_compound_gigantic_folio(folio, order, true); 1501 } 1502 1503 #ifdef CONFIG_ARCH_HAS_GIGANTIC_PAGE 1504 static void destroy_compound_gigantic_folio(struct folio *folio, 1505 unsigned int order) 1506 { 1507 __destroy_compound_gigantic_folio(folio, order, false); 1508 } 1509 1510 static void free_gigantic_folio(struct folio *folio, unsigned int order) 1511 { 1512 /* 1513 * If the page isn't allocated using the cma allocator, 1514 * cma_release() returns false. 1515 */ 1516 #ifdef CONFIG_CMA 1517 int nid = folio_nid(folio); 1518 1519 if (cma_release(hugetlb_cma[nid], &folio->page, 1 << order)) 1520 return; 1521 #endif 1522 1523 free_contig_range(folio_pfn(folio), 1 << order); 1524 } 1525 1526 #ifdef CONFIG_CONTIG_ALLOC 1527 static struct folio *alloc_gigantic_folio(struct hstate *h, gfp_t gfp_mask, 1528 int nid, nodemask_t *nodemask) 1529 { 1530 struct page *page; 1531 unsigned long nr_pages = pages_per_huge_page(h); 1532 if (nid == NUMA_NO_NODE) 1533 nid = numa_mem_id(); 1534 1535 #ifdef CONFIG_CMA 1536 { 1537 int node; 1538 1539 if (hugetlb_cma[nid]) { 1540 page = cma_alloc(hugetlb_cma[nid], nr_pages, 1541 huge_page_order(h), true); 1542 if (page) 1543 return page_folio(page); 1544 } 1545 1546 if (!(gfp_mask & __GFP_THISNODE)) { 1547 for_each_node_mask(node, *nodemask) { 1548 if (node == nid || !hugetlb_cma[node]) 1549 continue; 1550 1551 page = cma_alloc(hugetlb_cma[node], nr_pages, 1552 huge_page_order(h), true); 1553 if (page) 1554 return page_folio(page); 1555 } 1556 } 1557 } 1558 #endif 1559 1560 page = alloc_contig_pages(nr_pages, gfp_mask, nid, nodemask); 1561 return page ? page_folio(page) : NULL; 1562 } 1563 1564 #else /* !CONFIG_CONTIG_ALLOC */ 1565 static struct folio *alloc_gigantic_folio(struct hstate *h, gfp_t gfp_mask, 1566 int nid, nodemask_t *nodemask) 1567 { 1568 return NULL; 1569 } 1570 #endif /* CONFIG_CONTIG_ALLOC */ 1571 1572 #else /* !CONFIG_ARCH_HAS_GIGANTIC_PAGE */ 1573 static struct folio *alloc_gigantic_folio(struct hstate *h, gfp_t gfp_mask, 1574 int nid, nodemask_t *nodemask) 1575 { 1576 return NULL; 1577 } 1578 static inline void free_gigantic_folio(struct folio *folio, 1579 unsigned int order) { } 1580 static inline void destroy_compound_gigantic_folio(struct folio *folio, 1581 unsigned int order) { } 1582 #endif 1583 1584 static inline void __clear_hugetlb_destructor(struct hstate *h, 1585 struct folio *folio) 1586 { 1587 lockdep_assert_held(&hugetlb_lock); 1588 1589 folio_clear_hugetlb(folio); 1590 } 1591 1592 /* 1593 * Remove hugetlb folio from lists. 1594 * If vmemmap exists for the folio, update dtor so that the folio appears 1595 * as just a compound page. Otherwise, wait until after allocating vmemmap 1596 * to update dtor. 1597 * 1598 * A reference is held on the folio, except in the case of demote. 1599 * 1600 * Must be called with hugetlb lock held. 1601 */ 1602 static void __remove_hugetlb_folio(struct hstate *h, struct folio *folio, 1603 bool adjust_surplus, 1604 bool demote) 1605 { 1606 int nid = folio_nid(folio); 1607 1608 VM_BUG_ON_FOLIO(hugetlb_cgroup_from_folio(folio), folio); 1609 VM_BUG_ON_FOLIO(hugetlb_cgroup_from_folio_rsvd(folio), folio); 1610 1611 lockdep_assert_held(&hugetlb_lock); 1612 if (hstate_is_gigantic(h) && !gigantic_page_runtime_supported()) 1613 return; 1614 1615 list_del(&folio->lru); 1616 1617 if (folio_test_hugetlb_freed(folio)) { 1618 h->free_huge_pages--; 1619 h->free_huge_pages_node[nid]--; 1620 } 1621 if (adjust_surplus) { 1622 h->surplus_huge_pages--; 1623 h->surplus_huge_pages_node[nid]--; 1624 } 1625 1626 /* 1627 * We can only clear the hugetlb destructor after allocating vmemmap 1628 * pages. Otherwise, someone (memory error handling) may try to write 1629 * to tail struct pages. 1630 */ 1631 if (!folio_test_hugetlb_vmemmap_optimized(folio)) 1632 __clear_hugetlb_destructor(h, folio); 1633 1634 /* 1635 * In the case of demote we do not ref count the page as it will soon 1636 * be turned into a page of smaller size. 1637 */ 1638 if (!demote) 1639 folio_ref_unfreeze(folio, 1); 1640 1641 h->nr_huge_pages--; 1642 h->nr_huge_pages_node[nid]--; 1643 } 1644 1645 static void remove_hugetlb_folio(struct hstate *h, struct folio *folio, 1646 bool adjust_surplus) 1647 { 1648 __remove_hugetlb_folio(h, folio, adjust_surplus, false); 1649 } 1650 1651 static void remove_hugetlb_folio_for_demote(struct hstate *h, struct folio *folio, 1652 bool adjust_surplus) 1653 { 1654 __remove_hugetlb_folio(h, folio, adjust_surplus, true); 1655 } 1656 1657 static void add_hugetlb_folio(struct hstate *h, struct folio *folio, 1658 bool adjust_surplus) 1659 { 1660 int zeroed; 1661 int nid = folio_nid(folio); 1662 1663 VM_BUG_ON_FOLIO(!folio_test_hugetlb_vmemmap_optimized(folio), folio); 1664 1665 lockdep_assert_held(&hugetlb_lock); 1666 1667 INIT_LIST_HEAD(&folio->lru); 1668 h->nr_huge_pages++; 1669 h->nr_huge_pages_node[nid]++; 1670 1671 if (adjust_surplus) { 1672 h->surplus_huge_pages++; 1673 h->surplus_huge_pages_node[nid]++; 1674 } 1675 1676 folio_set_hugetlb(folio); 1677 folio_change_private(folio, NULL); 1678 /* 1679 * We have to set hugetlb_vmemmap_optimized again as above 1680 * folio_change_private(folio, NULL) cleared it. 1681 */ 1682 folio_set_hugetlb_vmemmap_optimized(folio); 1683 1684 /* 1685 * This folio is about to be managed by the hugetlb allocator and 1686 * should have no users. Drop our reference, and check for others 1687 * just in case. 1688 */ 1689 zeroed = folio_put_testzero(folio); 1690 if (unlikely(!zeroed)) 1691 /* 1692 * It is VERY unlikely soneone else has taken a ref 1693 * on the folio. In this case, we simply return as 1694 * free_huge_folio() will be called when this other ref 1695 * is dropped. 1696 */ 1697 return; 1698 1699 arch_clear_hugepage_flags(&folio->page); 1700 enqueue_hugetlb_folio(h, folio); 1701 } 1702 1703 static void __update_and_free_hugetlb_folio(struct hstate *h, 1704 struct folio *folio) 1705 { 1706 bool clear_dtor = folio_test_hugetlb_vmemmap_optimized(folio); 1707 1708 if (hstate_is_gigantic(h) && !gigantic_page_runtime_supported()) 1709 return; 1710 1711 /* 1712 * If we don't know which subpages are hwpoisoned, we can't free 1713 * the hugepage, so it's leaked intentionally. 1714 */ 1715 if (folio_test_hugetlb_raw_hwp_unreliable(folio)) 1716 return; 1717 1718 if (hugetlb_vmemmap_restore(h, &folio->page)) { 1719 spin_lock_irq(&hugetlb_lock); 1720 /* 1721 * If we cannot allocate vmemmap pages, just refuse to free the 1722 * page and put the page back on the hugetlb free list and treat 1723 * as a surplus page. 1724 */ 1725 add_hugetlb_folio(h, folio, true); 1726 spin_unlock_irq(&hugetlb_lock); 1727 return; 1728 } 1729 1730 /* 1731 * Move PageHWPoison flag from head page to the raw error pages, 1732 * which makes any healthy subpages reusable. 1733 */ 1734 if (unlikely(folio_test_hwpoison(folio))) 1735 folio_clear_hugetlb_hwpoison(folio); 1736 1737 /* 1738 * If vmemmap pages were allocated above, then we need to clear the 1739 * hugetlb destructor under the hugetlb lock. 1740 */ 1741 if (clear_dtor) { 1742 spin_lock_irq(&hugetlb_lock); 1743 __clear_hugetlb_destructor(h, folio); 1744 spin_unlock_irq(&hugetlb_lock); 1745 } 1746 1747 /* 1748 * Non-gigantic pages demoted from CMA allocated gigantic pages 1749 * need to be given back to CMA in free_gigantic_folio. 1750 */ 1751 if (hstate_is_gigantic(h) || 1752 hugetlb_cma_folio(folio, huge_page_order(h))) { 1753 destroy_compound_gigantic_folio(folio, huge_page_order(h)); 1754 free_gigantic_folio(folio, huge_page_order(h)); 1755 } else { 1756 __free_pages(&folio->page, huge_page_order(h)); 1757 } 1758 } 1759 1760 /* 1761 * As update_and_free_hugetlb_folio() can be called under any context, so we cannot 1762 * use GFP_KERNEL to allocate vmemmap pages. However, we can defer the 1763 * actual freeing in a workqueue to prevent from using GFP_ATOMIC to allocate 1764 * the vmemmap pages. 1765 * 1766 * free_hpage_workfn() locklessly retrieves the linked list of pages to be 1767 * freed and frees them one-by-one. As the page->mapping pointer is going 1768 * to be cleared in free_hpage_workfn() anyway, it is reused as the llist_node 1769 * structure of a lockless linked list of huge pages to be freed. 1770 */ 1771 static LLIST_HEAD(hpage_freelist); 1772 1773 static void free_hpage_workfn(struct work_struct *work) 1774 { 1775 struct llist_node *node; 1776 1777 node = llist_del_all(&hpage_freelist); 1778 1779 while (node) { 1780 struct page *page; 1781 struct hstate *h; 1782 1783 page = container_of((struct address_space **)node, 1784 struct page, mapping); 1785 node = node->next; 1786 page->mapping = NULL; 1787 /* 1788 * The VM_BUG_ON_FOLIO(!folio_test_hugetlb(folio), folio) in 1789 * folio_hstate() is going to trigger because a previous call to 1790 * remove_hugetlb_folio() will clear the hugetlb bit, so do 1791 * not use folio_hstate() directly. 1792 */ 1793 h = size_to_hstate(page_size(page)); 1794 1795 __update_and_free_hugetlb_folio(h, page_folio(page)); 1796 1797 cond_resched(); 1798 } 1799 } 1800 static DECLARE_WORK(free_hpage_work, free_hpage_workfn); 1801 1802 static inline void flush_free_hpage_work(struct hstate *h) 1803 { 1804 if (hugetlb_vmemmap_optimizable(h)) 1805 flush_work(&free_hpage_work); 1806 } 1807 1808 static void update_and_free_hugetlb_folio(struct hstate *h, struct folio *folio, 1809 bool atomic) 1810 { 1811 if (!folio_test_hugetlb_vmemmap_optimized(folio) || !atomic) { 1812 __update_and_free_hugetlb_folio(h, folio); 1813 return; 1814 } 1815 1816 /* 1817 * Defer freeing to avoid using GFP_ATOMIC to allocate vmemmap pages. 1818 * 1819 * Only call schedule_work() if hpage_freelist is previously 1820 * empty. Otherwise, schedule_work() had been called but the workfn 1821 * hasn't retrieved the list yet. 1822 */ 1823 if (llist_add((struct llist_node *)&folio->mapping, &hpage_freelist)) 1824 schedule_work(&free_hpage_work); 1825 } 1826 1827 static void update_and_free_pages_bulk(struct hstate *h, struct list_head *list) 1828 { 1829 struct page *page, *t_page; 1830 struct folio *folio; 1831 1832 list_for_each_entry_safe(page, t_page, list, lru) { 1833 folio = page_folio(page); 1834 update_and_free_hugetlb_folio(h, folio, false); 1835 cond_resched(); 1836 } 1837 } 1838 1839 struct hstate *size_to_hstate(unsigned long size) 1840 { 1841 struct hstate *h; 1842 1843 for_each_hstate(h) { 1844 if (huge_page_size(h) == size) 1845 return h; 1846 } 1847 return NULL; 1848 } 1849 1850 void free_huge_folio(struct folio *folio) 1851 { 1852 /* 1853 * Can't pass hstate in here because it is called from the 1854 * compound page destructor. 1855 */ 1856 struct hstate *h = folio_hstate(folio); 1857 int nid = folio_nid(folio); 1858 struct hugepage_subpool *spool = hugetlb_folio_subpool(folio); 1859 bool restore_reserve; 1860 unsigned long flags; 1861 1862 VM_BUG_ON_FOLIO(folio_ref_count(folio), folio); 1863 VM_BUG_ON_FOLIO(folio_mapcount(folio), folio); 1864 1865 hugetlb_set_folio_subpool(folio, NULL); 1866 if (folio_test_anon(folio)) 1867 __ClearPageAnonExclusive(&folio->page); 1868 folio->mapping = NULL; 1869 restore_reserve = folio_test_hugetlb_restore_reserve(folio); 1870 folio_clear_hugetlb_restore_reserve(folio); 1871 1872 /* 1873 * If HPageRestoreReserve was set on page, page allocation consumed a 1874 * reservation. If the page was associated with a subpool, there 1875 * would have been a page reserved in the subpool before allocation 1876 * via hugepage_subpool_get_pages(). Since we are 'restoring' the 1877 * reservation, do not call hugepage_subpool_put_pages() as this will 1878 * remove the reserved page from the subpool. 1879 */ 1880 if (!restore_reserve) { 1881 /* 1882 * A return code of zero implies that the subpool will be 1883 * under its minimum size if the reservation is not restored 1884 * after page is free. Therefore, force restore_reserve 1885 * operation. 1886 */ 1887 if (hugepage_subpool_put_pages(spool, 1) == 0) 1888 restore_reserve = true; 1889 } 1890 1891 spin_lock_irqsave(&hugetlb_lock, flags); 1892 folio_clear_hugetlb_migratable(folio); 1893 hugetlb_cgroup_uncharge_folio(hstate_index(h), 1894 pages_per_huge_page(h), folio); 1895 hugetlb_cgroup_uncharge_folio_rsvd(hstate_index(h), 1896 pages_per_huge_page(h), folio); 1897 if (restore_reserve) 1898 h->resv_huge_pages++; 1899 1900 if (folio_test_hugetlb_temporary(folio)) { 1901 remove_hugetlb_folio(h, folio, false); 1902 spin_unlock_irqrestore(&hugetlb_lock, flags); 1903 update_and_free_hugetlb_folio(h, folio, true); 1904 } else if (h->surplus_huge_pages_node[nid]) { 1905 /* remove the page from active list */ 1906 remove_hugetlb_folio(h, folio, true); 1907 spin_unlock_irqrestore(&hugetlb_lock, flags); 1908 update_and_free_hugetlb_folio(h, folio, true); 1909 } else { 1910 arch_clear_hugepage_flags(&folio->page); 1911 enqueue_hugetlb_folio(h, folio); 1912 spin_unlock_irqrestore(&hugetlb_lock, flags); 1913 } 1914 } 1915 1916 /* 1917 * Must be called with the hugetlb lock held 1918 */ 1919 static void __prep_account_new_huge_page(struct hstate *h, int nid) 1920 { 1921 lockdep_assert_held(&hugetlb_lock); 1922 h->nr_huge_pages++; 1923 h->nr_huge_pages_node[nid]++; 1924 } 1925 1926 static void __prep_new_hugetlb_folio(struct hstate *h, struct folio *folio) 1927 { 1928 hugetlb_vmemmap_optimize(h, &folio->page); 1929 INIT_LIST_HEAD(&folio->lru); 1930 folio_set_hugetlb(folio); 1931 hugetlb_set_folio_subpool(folio, NULL); 1932 set_hugetlb_cgroup(folio, NULL); 1933 set_hugetlb_cgroup_rsvd(folio, NULL); 1934 } 1935 1936 static void prep_new_hugetlb_folio(struct hstate *h, struct folio *folio, int nid) 1937 { 1938 __prep_new_hugetlb_folio(h, folio); 1939 spin_lock_irq(&hugetlb_lock); 1940 __prep_account_new_huge_page(h, nid); 1941 spin_unlock_irq(&hugetlb_lock); 1942 } 1943 1944 static bool __prep_compound_gigantic_folio(struct folio *folio, 1945 unsigned int order, bool demote) 1946 { 1947 int i, j; 1948 int nr_pages = 1 << order; 1949 struct page *p; 1950 1951 __folio_clear_reserved(folio); 1952 for (i = 0; i < nr_pages; i++) { 1953 p = folio_page(folio, i); 1954 1955 /* 1956 * For gigantic hugepages allocated through bootmem at 1957 * boot, it's safer to be consistent with the not-gigantic 1958 * hugepages and clear the PG_reserved bit from all tail pages 1959 * too. Otherwise drivers using get_user_pages() to access tail 1960 * pages may get the reference counting wrong if they see 1961 * PG_reserved set on a tail page (despite the head page not 1962 * having PG_reserved set). Enforcing this consistency between 1963 * head and tail pages allows drivers to optimize away a check 1964 * on the head page when they need know if put_page() is needed 1965 * after get_user_pages(). 1966 */ 1967 if (i != 0) /* head page cleared above */ 1968 __ClearPageReserved(p); 1969 /* 1970 * Subtle and very unlikely 1971 * 1972 * Gigantic 'page allocators' such as memblock or cma will 1973 * return a set of pages with each page ref counted. We need 1974 * to turn this set of pages into a compound page with tail 1975 * page ref counts set to zero. Code such as speculative page 1976 * cache adding could take a ref on a 'to be' tail page. 1977 * We need to respect any increased ref count, and only set 1978 * the ref count to zero if count is currently 1. If count 1979 * is not 1, we return an error. An error return indicates 1980 * the set of pages can not be converted to a gigantic page. 1981 * The caller who allocated the pages should then discard the 1982 * pages using the appropriate free interface. 1983 * 1984 * In the case of demote, the ref count will be zero. 1985 */ 1986 if (!demote) { 1987 if (!page_ref_freeze(p, 1)) { 1988 pr_warn("HugeTLB page can not be used due to unexpected inflated ref count\n"); 1989 goto out_error; 1990 } 1991 } else { 1992 VM_BUG_ON_PAGE(page_count(p), p); 1993 } 1994 if (i != 0) 1995 set_compound_head(p, &folio->page); 1996 } 1997 __folio_set_head(folio); 1998 /* we rely on prep_new_hugetlb_folio to set the destructor */ 1999 folio_set_order(folio, order); 2000 atomic_set(&folio->_entire_mapcount, -1); 2001 atomic_set(&folio->_nr_pages_mapped, 0); 2002 atomic_set(&folio->_pincount, 0); 2003 return true; 2004 2005 out_error: 2006 /* undo page modifications made above */ 2007 for (j = 0; j < i; j++) { 2008 p = folio_page(folio, j); 2009 if (j != 0) 2010 clear_compound_head(p); 2011 set_page_refcounted(p); 2012 } 2013 /* need to clear PG_reserved on remaining tail pages */ 2014 for (; j < nr_pages; j++) { 2015 p = folio_page(folio, j); 2016 __ClearPageReserved(p); 2017 } 2018 return false; 2019 } 2020 2021 static bool prep_compound_gigantic_folio(struct folio *folio, 2022 unsigned int order) 2023 { 2024 return __prep_compound_gigantic_folio(folio, order, false); 2025 } 2026 2027 static bool prep_compound_gigantic_folio_for_demote(struct folio *folio, 2028 unsigned int order) 2029 { 2030 return __prep_compound_gigantic_folio(folio, order, true); 2031 } 2032 2033 /* 2034 * PageHuge() only returns true for hugetlbfs pages, but not for normal or 2035 * transparent huge pages. See the PageTransHuge() documentation for more 2036 * details. 2037 */ 2038 int PageHuge(struct page *page) 2039 { 2040 struct folio *folio; 2041 2042 if (!PageCompound(page)) 2043 return 0; 2044 folio = page_folio(page); 2045 return folio_test_hugetlb(folio); 2046 } 2047 EXPORT_SYMBOL_GPL(PageHuge); 2048 2049 /* 2050 * Find and lock address space (mapping) in write mode. 2051 * 2052 * Upon entry, the page is locked which means that page_mapping() is 2053 * stable. Due to locking order, we can only trylock_write. If we can 2054 * not get the lock, simply return NULL to caller. 2055 */ 2056 struct address_space *hugetlb_page_mapping_lock_write(struct page *hpage) 2057 { 2058 struct address_space *mapping = page_mapping(hpage); 2059 2060 if (!mapping) 2061 return mapping; 2062 2063 if (i_mmap_trylock_write(mapping)) 2064 return mapping; 2065 2066 return NULL; 2067 } 2068 2069 pgoff_t hugetlb_basepage_index(struct page *page) 2070 { 2071 struct page *page_head = compound_head(page); 2072 pgoff_t index = page_index(page_head); 2073 unsigned long compound_idx; 2074 2075 if (compound_order(page_head) > MAX_ORDER) 2076 compound_idx = page_to_pfn(page) - page_to_pfn(page_head); 2077 else 2078 compound_idx = page - page_head; 2079 2080 return (index << compound_order(page_head)) + compound_idx; 2081 } 2082 2083 static struct folio *alloc_buddy_hugetlb_folio(struct hstate *h, 2084 gfp_t gfp_mask, int nid, nodemask_t *nmask, 2085 nodemask_t *node_alloc_noretry) 2086 { 2087 int order = huge_page_order(h); 2088 struct page *page; 2089 bool alloc_try_hard = true; 2090 bool retry = true; 2091 2092 /* 2093 * By default we always try hard to allocate the page with 2094 * __GFP_RETRY_MAYFAIL flag. However, if we are allocating pages in 2095 * a loop (to adjust global huge page counts) and previous allocation 2096 * failed, do not continue to try hard on the same node. Use the 2097 * node_alloc_noretry bitmap to manage this state information. 2098 */ 2099 if (node_alloc_noretry && node_isset(nid, *node_alloc_noretry)) 2100 alloc_try_hard = false; 2101 gfp_mask |= __GFP_COMP|__GFP_NOWARN; 2102 if (alloc_try_hard) 2103 gfp_mask |= __GFP_RETRY_MAYFAIL; 2104 if (nid == NUMA_NO_NODE) 2105 nid = numa_mem_id(); 2106 retry: 2107 page = __alloc_pages(gfp_mask, order, nid, nmask); 2108 2109 /* Freeze head page */ 2110 if (page && !page_ref_freeze(page, 1)) { 2111 __free_pages(page, order); 2112 if (retry) { /* retry once */ 2113 retry = false; 2114 goto retry; 2115 } 2116 /* WOW! twice in a row. */ 2117 pr_warn("HugeTLB head page unexpected inflated ref count\n"); 2118 page = NULL; 2119 } 2120 2121 /* 2122 * If we did not specify __GFP_RETRY_MAYFAIL, but still got a page this 2123 * indicates an overall state change. Clear bit so that we resume 2124 * normal 'try hard' allocations. 2125 */ 2126 if (node_alloc_noretry && page && !alloc_try_hard) 2127 node_clear(nid, *node_alloc_noretry); 2128 2129 /* 2130 * If we tried hard to get a page but failed, set bit so that 2131 * subsequent attempts will not try as hard until there is an 2132 * overall state change. 2133 */ 2134 if (node_alloc_noretry && !page && alloc_try_hard) 2135 node_set(nid, *node_alloc_noretry); 2136 2137 if (!page) { 2138 __count_vm_event(HTLB_BUDDY_PGALLOC_FAIL); 2139 return NULL; 2140 } 2141 2142 __count_vm_event(HTLB_BUDDY_PGALLOC); 2143 return page_folio(page); 2144 } 2145 2146 /* 2147 * Common helper to allocate a fresh hugetlb page. All specific allocators 2148 * should use this function to get new hugetlb pages 2149 * 2150 * Note that returned page is 'frozen': ref count of head page and all tail 2151 * pages is zero. 2152 */ 2153 static struct folio *alloc_fresh_hugetlb_folio(struct hstate *h, 2154 gfp_t gfp_mask, int nid, nodemask_t *nmask, 2155 nodemask_t *node_alloc_noretry) 2156 { 2157 struct folio *folio; 2158 bool retry = false; 2159 2160 retry: 2161 if (hstate_is_gigantic(h)) 2162 folio = alloc_gigantic_folio(h, gfp_mask, nid, nmask); 2163 else 2164 folio = alloc_buddy_hugetlb_folio(h, gfp_mask, 2165 nid, nmask, node_alloc_noretry); 2166 if (!folio) 2167 return NULL; 2168 if (hstate_is_gigantic(h)) { 2169 if (!prep_compound_gigantic_folio(folio, huge_page_order(h))) { 2170 /* 2171 * Rare failure to convert pages to compound page. 2172 * Free pages and try again - ONCE! 2173 */ 2174 free_gigantic_folio(folio, huge_page_order(h)); 2175 if (!retry) { 2176 retry = true; 2177 goto retry; 2178 } 2179 return NULL; 2180 } 2181 } 2182 prep_new_hugetlb_folio(h, folio, folio_nid(folio)); 2183 2184 return folio; 2185 } 2186 2187 /* 2188 * Allocates a fresh page to the hugetlb allocator pool in the node interleaved 2189 * manner. 2190 */ 2191 static int alloc_pool_huge_page(struct hstate *h, nodemask_t *nodes_allowed, 2192 nodemask_t *node_alloc_noretry) 2193 { 2194 struct folio *folio; 2195 int nr_nodes, node; 2196 gfp_t gfp_mask = htlb_alloc_mask(h) | __GFP_THISNODE; 2197 2198 for_each_node_mask_to_alloc(h, nr_nodes, node, nodes_allowed) { 2199 folio = alloc_fresh_hugetlb_folio(h, gfp_mask, node, 2200 nodes_allowed, node_alloc_noretry); 2201 if (folio) { 2202 free_huge_folio(folio); /* free it into the hugepage allocator */ 2203 return 1; 2204 } 2205 } 2206 2207 return 0; 2208 } 2209 2210 /* 2211 * Remove huge page from pool from next node to free. Attempt to keep 2212 * persistent huge pages more or less balanced over allowed nodes. 2213 * This routine only 'removes' the hugetlb page. The caller must make 2214 * an additional call to free the page to low level allocators. 2215 * Called with hugetlb_lock locked. 2216 */ 2217 static struct page *remove_pool_huge_page(struct hstate *h, 2218 nodemask_t *nodes_allowed, 2219 bool acct_surplus) 2220 { 2221 int nr_nodes, node; 2222 struct page *page = NULL; 2223 struct folio *folio; 2224 2225 lockdep_assert_held(&hugetlb_lock); 2226 for_each_node_mask_to_free(h, nr_nodes, node, nodes_allowed) { 2227 /* 2228 * If we're returning unused surplus pages, only examine 2229 * nodes with surplus pages. 2230 */ 2231 if ((!acct_surplus || h->surplus_huge_pages_node[node]) && 2232 !list_empty(&h->hugepage_freelists[node])) { 2233 page = list_entry(h->hugepage_freelists[node].next, 2234 struct page, lru); 2235 folio = page_folio(page); 2236 remove_hugetlb_folio(h, folio, acct_surplus); 2237 break; 2238 } 2239 } 2240 2241 return page; 2242 } 2243 2244 /* 2245 * Dissolve a given free hugepage into free buddy pages. This function does 2246 * nothing for in-use hugepages and non-hugepages. 2247 * This function returns values like below: 2248 * 2249 * -ENOMEM: failed to allocate vmemmap pages to free the freed hugepages 2250 * when the system is under memory pressure and the feature of 2251 * freeing unused vmemmap pages associated with each hugetlb page 2252 * is enabled. 2253 * -EBUSY: failed to dissolved free hugepages or the hugepage is in-use 2254 * (allocated or reserved.) 2255 * 0: successfully dissolved free hugepages or the page is not a 2256 * hugepage (considered as already dissolved) 2257 */ 2258 int dissolve_free_huge_page(struct page *page) 2259 { 2260 int rc = -EBUSY; 2261 struct folio *folio = page_folio(page); 2262 2263 retry: 2264 /* Not to disrupt normal path by vainly holding hugetlb_lock */ 2265 if (!folio_test_hugetlb(folio)) 2266 return 0; 2267 2268 spin_lock_irq(&hugetlb_lock); 2269 if (!folio_test_hugetlb(folio)) { 2270 rc = 0; 2271 goto out; 2272 } 2273 2274 if (!folio_ref_count(folio)) { 2275 struct hstate *h = folio_hstate(folio); 2276 if (!available_huge_pages(h)) 2277 goto out; 2278 2279 /* 2280 * We should make sure that the page is already on the free list 2281 * when it is dissolved. 2282 */ 2283 if (unlikely(!folio_test_hugetlb_freed(folio))) { 2284 spin_unlock_irq(&hugetlb_lock); 2285 cond_resched(); 2286 2287 /* 2288 * Theoretically, we should return -EBUSY when we 2289 * encounter this race. In fact, we have a chance 2290 * to successfully dissolve the page if we do a 2291 * retry. Because the race window is quite small. 2292 * If we seize this opportunity, it is an optimization 2293 * for increasing the success rate of dissolving page. 2294 */ 2295 goto retry; 2296 } 2297 2298 remove_hugetlb_folio(h, folio, false); 2299 h->max_huge_pages--; 2300 spin_unlock_irq(&hugetlb_lock); 2301 2302 /* 2303 * Normally update_and_free_hugtlb_folio will allocate required vmemmmap 2304 * before freeing the page. update_and_free_hugtlb_folio will fail to 2305 * free the page if it can not allocate required vmemmap. We 2306 * need to adjust max_huge_pages if the page is not freed. 2307 * Attempt to allocate vmemmmap here so that we can take 2308 * appropriate action on failure. 2309 */ 2310 rc = hugetlb_vmemmap_restore(h, &folio->page); 2311 if (!rc) { 2312 update_and_free_hugetlb_folio(h, folio, false); 2313 } else { 2314 spin_lock_irq(&hugetlb_lock); 2315 add_hugetlb_folio(h, folio, false); 2316 h->max_huge_pages++; 2317 spin_unlock_irq(&hugetlb_lock); 2318 } 2319 2320 return rc; 2321 } 2322 out: 2323 spin_unlock_irq(&hugetlb_lock); 2324 return rc; 2325 } 2326 2327 /* 2328 * Dissolve free hugepages in a given pfn range. Used by memory hotplug to 2329 * make specified memory blocks removable from the system. 2330 * Note that this will dissolve a free gigantic hugepage completely, if any 2331 * part of it lies within the given range. 2332 * Also note that if dissolve_free_huge_page() returns with an error, all 2333 * free hugepages that were dissolved before that error are lost. 2334 */ 2335 int dissolve_free_huge_pages(unsigned long start_pfn, unsigned long end_pfn) 2336 { 2337 unsigned long pfn; 2338 struct page *page; 2339 int rc = 0; 2340 unsigned int order; 2341 struct hstate *h; 2342 2343 if (!hugepages_supported()) 2344 return rc; 2345 2346 order = huge_page_order(&default_hstate); 2347 for_each_hstate(h) 2348 order = min(order, huge_page_order(h)); 2349 2350 for (pfn = start_pfn; pfn < end_pfn; pfn += 1 << order) { 2351 page = pfn_to_page(pfn); 2352 rc = dissolve_free_huge_page(page); 2353 if (rc) 2354 break; 2355 } 2356 2357 return rc; 2358 } 2359 2360 /* 2361 * Allocates a fresh surplus page from the page allocator. 2362 */ 2363 static struct folio *alloc_surplus_hugetlb_folio(struct hstate *h, 2364 gfp_t gfp_mask, int nid, nodemask_t *nmask) 2365 { 2366 struct folio *folio = NULL; 2367 2368 if (hstate_is_gigantic(h)) 2369 return NULL; 2370 2371 spin_lock_irq(&hugetlb_lock); 2372 if (h->surplus_huge_pages >= h->nr_overcommit_huge_pages) 2373 goto out_unlock; 2374 spin_unlock_irq(&hugetlb_lock); 2375 2376 folio = alloc_fresh_hugetlb_folio(h, gfp_mask, nid, nmask, NULL); 2377 if (!folio) 2378 return NULL; 2379 2380 spin_lock_irq(&hugetlb_lock); 2381 /* 2382 * We could have raced with the pool size change. 2383 * Double check that and simply deallocate the new page 2384 * if we would end up overcommiting the surpluses. Abuse 2385 * temporary page to workaround the nasty free_huge_folio 2386 * codeflow 2387 */ 2388 if (h->surplus_huge_pages >= h->nr_overcommit_huge_pages) { 2389 folio_set_hugetlb_temporary(folio); 2390 spin_unlock_irq(&hugetlb_lock); 2391 free_huge_folio(folio); 2392 return NULL; 2393 } 2394 2395 h->surplus_huge_pages++; 2396 h->surplus_huge_pages_node[folio_nid(folio)]++; 2397 2398 out_unlock: 2399 spin_unlock_irq(&hugetlb_lock); 2400 2401 return folio; 2402 } 2403 2404 static struct folio *alloc_migrate_hugetlb_folio(struct hstate *h, gfp_t gfp_mask, 2405 int nid, nodemask_t *nmask) 2406 { 2407 struct folio *folio; 2408 2409 if (hstate_is_gigantic(h)) 2410 return NULL; 2411 2412 folio = alloc_fresh_hugetlb_folio(h, gfp_mask, nid, nmask, NULL); 2413 if (!folio) 2414 return NULL; 2415 2416 /* fresh huge pages are frozen */ 2417 folio_ref_unfreeze(folio, 1); 2418 /* 2419 * We do not account these pages as surplus because they are only 2420 * temporary and will be released properly on the last reference 2421 */ 2422 folio_set_hugetlb_temporary(folio); 2423 2424 return folio; 2425 } 2426 2427 /* 2428 * Use the VMA's mpolicy to allocate a huge page from the buddy. 2429 */ 2430 static 2431 struct folio *alloc_buddy_hugetlb_folio_with_mpol(struct hstate *h, 2432 struct vm_area_struct *vma, unsigned long addr) 2433 { 2434 struct folio *folio = NULL; 2435 struct mempolicy *mpol; 2436 gfp_t gfp_mask = htlb_alloc_mask(h); 2437 int nid; 2438 nodemask_t *nodemask; 2439 2440 nid = huge_node(vma, addr, gfp_mask, &mpol, &nodemask); 2441 if (mpol_is_preferred_many(mpol)) { 2442 gfp_t gfp = gfp_mask | __GFP_NOWARN; 2443 2444 gfp &= ~(__GFP_DIRECT_RECLAIM | __GFP_NOFAIL); 2445 folio = alloc_surplus_hugetlb_folio(h, gfp, nid, nodemask); 2446 2447 /* Fallback to all nodes if page==NULL */ 2448 nodemask = NULL; 2449 } 2450 2451 if (!folio) 2452 folio = alloc_surplus_hugetlb_folio(h, gfp_mask, nid, nodemask); 2453 mpol_cond_put(mpol); 2454 return folio; 2455 } 2456 2457 /* folio migration callback function */ 2458 struct folio *alloc_hugetlb_folio_nodemask(struct hstate *h, int preferred_nid, 2459 nodemask_t *nmask, gfp_t gfp_mask) 2460 { 2461 spin_lock_irq(&hugetlb_lock); 2462 if (available_huge_pages(h)) { 2463 struct folio *folio; 2464 2465 folio = dequeue_hugetlb_folio_nodemask(h, gfp_mask, 2466 preferred_nid, nmask); 2467 if (folio) { 2468 spin_unlock_irq(&hugetlb_lock); 2469 return folio; 2470 } 2471 } 2472 spin_unlock_irq(&hugetlb_lock); 2473 2474 return alloc_migrate_hugetlb_folio(h, gfp_mask, preferred_nid, nmask); 2475 } 2476 2477 /* mempolicy aware migration callback */ 2478 struct folio *alloc_hugetlb_folio_vma(struct hstate *h, struct vm_area_struct *vma, 2479 unsigned long address) 2480 { 2481 struct mempolicy *mpol; 2482 nodemask_t *nodemask; 2483 struct folio *folio; 2484 gfp_t gfp_mask; 2485 int node; 2486 2487 gfp_mask = htlb_alloc_mask(h); 2488 node = huge_node(vma, address, gfp_mask, &mpol, &nodemask); 2489 folio = alloc_hugetlb_folio_nodemask(h, node, nodemask, gfp_mask); 2490 mpol_cond_put(mpol); 2491 2492 return folio; 2493 } 2494 2495 /* 2496 * Increase the hugetlb pool such that it can accommodate a reservation 2497 * of size 'delta'. 2498 */ 2499 static int gather_surplus_pages(struct hstate *h, long delta) 2500 __must_hold(&hugetlb_lock) 2501 { 2502 LIST_HEAD(surplus_list); 2503 struct folio *folio, *tmp; 2504 int ret; 2505 long i; 2506 long needed, allocated; 2507 bool alloc_ok = true; 2508 2509 lockdep_assert_held(&hugetlb_lock); 2510 needed = (h->resv_huge_pages + delta) - h->free_huge_pages; 2511 if (needed <= 0) { 2512 h->resv_huge_pages += delta; 2513 return 0; 2514 } 2515 2516 allocated = 0; 2517 2518 ret = -ENOMEM; 2519 retry: 2520 spin_unlock_irq(&hugetlb_lock); 2521 for (i = 0; i < needed; i++) { 2522 folio = alloc_surplus_hugetlb_folio(h, htlb_alloc_mask(h), 2523 NUMA_NO_NODE, NULL); 2524 if (!folio) { 2525 alloc_ok = false; 2526 break; 2527 } 2528 list_add(&folio->lru, &surplus_list); 2529 cond_resched(); 2530 } 2531 allocated += i; 2532 2533 /* 2534 * After retaking hugetlb_lock, we need to recalculate 'needed' 2535 * because either resv_huge_pages or free_huge_pages may have changed. 2536 */ 2537 spin_lock_irq(&hugetlb_lock); 2538 needed = (h->resv_huge_pages + delta) - 2539 (h->free_huge_pages + allocated); 2540 if (needed > 0) { 2541 if (alloc_ok) 2542 goto retry; 2543 /* 2544 * We were not able to allocate enough pages to 2545 * satisfy the entire reservation so we free what 2546 * we've allocated so far. 2547 */ 2548 goto free; 2549 } 2550 /* 2551 * The surplus_list now contains _at_least_ the number of extra pages 2552 * needed to accommodate the reservation. Add the appropriate number 2553 * of pages to the hugetlb pool and free the extras back to the buddy 2554 * allocator. Commit the entire reservation here to prevent another 2555 * process from stealing the pages as they are added to the pool but 2556 * before they are reserved. 2557 */ 2558 needed += allocated; 2559 h->resv_huge_pages += delta; 2560 ret = 0; 2561 2562 /* Free the needed pages to the hugetlb pool */ 2563 list_for_each_entry_safe(folio, tmp, &surplus_list, lru) { 2564 if ((--needed) < 0) 2565 break; 2566 /* Add the page to the hugetlb allocator */ 2567 enqueue_hugetlb_folio(h, folio); 2568 } 2569 free: 2570 spin_unlock_irq(&hugetlb_lock); 2571 2572 /* 2573 * Free unnecessary surplus pages to the buddy allocator. 2574 * Pages have no ref count, call free_huge_folio directly. 2575 */ 2576 list_for_each_entry_safe(folio, tmp, &surplus_list, lru) 2577 free_huge_folio(folio); 2578 spin_lock_irq(&hugetlb_lock); 2579 2580 return ret; 2581 } 2582 2583 /* 2584 * This routine has two main purposes: 2585 * 1) Decrement the reservation count (resv_huge_pages) by the value passed 2586 * in unused_resv_pages. This corresponds to the prior adjustments made 2587 * to the associated reservation map. 2588 * 2) Free any unused surplus pages that may have been allocated to satisfy 2589 * the reservation. As many as unused_resv_pages may be freed. 2590 */ 2591 static void return_unused_surplus_pages(struct hstate *h, 2592 unsigned long unused_resv_pages) 2593 { 2594 unsigned long nr_pages; 2595 struct page *page; 2596 LIST_HEAD(page_list); 2597 2598 lockdep_assert_held(&hugetlb_lock); 2599 /* Uncommit the reservation */ 2600 h->resv_huge_pages -= unused_resv_pages; 2601 2602 if (hstate_is_gigantic(h) && !gigantic_page_runtime_supported()) 2603 goto out; 2604 2605 /* 2606 * Part (or even all) of the reservation could have been backed 2607 * by pre-allocated pages. Only free surplus pages. 2608 */ 2609 nr_pages = min(unused_resv_pages, h->surplus_huge_pages); 2610 2611 /* 2612 * We want to release as many surplus pages as possible, spread 2613 * evenly across all nodes with memory. Iterate across these nodes 2614 * until we can no longer free unreserved surplus pages. This occurs 2615 * when the nodes with surplus pages have no free pages. 2616 * remove_pool_huge_page() will balance the freed pages across the 2617 * on-line nodes with memory and will handle the hstate accounting. 2618 */ 2619 while (nr_pages--) { 2620 page = remove_pool_huge_page(h, &node_states[N_MEMORY], 1); 2621 if (!page) 2622 goto out; 2623 2624 list_add(&page->lru, &page_list); 2625 } 2626 2627 out: 2628 spin_unlock_irq(&hugetlb_lock); 2629 update_and_free_pages_bulk(h, &page_list); 2630 spin_lock_irq(&hugetlb_lock); 2631 } 2632 2633 2634 /* 2635 * vma_needs_reservation, vma_commit_reservation and vma_end_reservation 2636 * are used by the huge page allocation routines to manage reservations. 2637 * 2638 * vma_needs_reservation is called to determine if the huge page at addr 2639 * within the vma has an associated reservation. If a reservation is 2640 * needed, the value 1 is returned. The caller is then responsible for 2641 * managing the global reservation and subpool usage counts. After 2642 * the huge page has been allocated, vma_commit_reservation is called 2643 * to add the page to the reservation map. If the page allocation fails, 2644 * the reservation must be ended instead of committed. vma_end_reservation 2645 * is called in such cases. 2646 * 2647 * In the normal case, vma_commit_reservation returns the same value 2648 * as the preceding vma_needs_reservation call. The only time this 2649 * is not the case is if a reserve map was changed between calls. It 2650 * is the responsibility of the caller to notice the difference and 2651 * take appropriate action. 2652 * 2653 * vma_add_reservation is used in error paths where a reservation must 2654 * be restored when a newly allocated huge page must be freed. It is 2655 * to be called after calling vma_needs_reservation to determine if a 2656 * reservation exists. 2657 * 2658 * vma_del_reservation is used in error paths where an entry in the reserve 2659 * map was created during huge page allocation and must be removed. It is to 2660 * be called after calling vma_needs_reservation to determine if a reservation 2661 * exists. 2662 */ 2663 enum vma_resv_mode { 2664 VMA_NEEDS_RESV, 2665 VMA_COMMIT_RESV, 2666 VMA_END_RESV, 2667 VMA_ADD_RESV, 2668 VMA_DEL_RESV, 2669 }; 2670 static long __vma_reservation_common(struct hstate *h, 2671 struct vm_area_struct *vma, unsigned long addr, 2672 enum vma_resv_mode mode) 2673 { 2674 struct resv_map *resv; 2675 pgoff_t idx; 2676 long ret; 2677 long dummy_out_regions_needed; 2678 2679 resv = vma_resv_map(vma); 2680 if (!resv) 2681 return 1; 2682 2683 idx = vma_hugecache_offset(h, vma, addr); 2684 switch (mode) { 2685 case VMA_NEEDS_RESV: 2686 ret = region_chg(resv, idx, idx + 1, &dummy_out_regions_needed); 2687 /* We assume that vma_reservation_* routines always operate on 2688 * 1 page, and that adding to resv map a 1 page entry can only 2689 * ever require 1 region. 2690 */ 2691 VM_BUG_ON(dummy_out_regions_needed != 1); 2692 break; 2693 case VMA_COMMIT_RESV: 2694 ret = region_add(resv, idx, idx + 1, 1, NULL, NULL); 2695 /* region_add calls of range 1 should never fail. */ 2696 VM_BUG_ON(ret < 0); 2697 break; 2698 case VMA_END_RESV: 2699 region_abort(resv, idx, idx + 1, 1); 2700 ret = 0; 2701 break; 2702 case VMA_ADD_RESV: 2703 if (vma->vm_flags & VM_MAYSHARE) { 2704 ret = region_add(resv, idx, idx + 1, 1, NULL, NULL); 2705 /* region_add calls of range 1 should never fail. */ 2706 VM_BUG_ON(ret < 0); 2707 } else { 2708 region_abort(resv, idx, idx + 1, 1); 2709 ret = region_del(resv, idx, idx + 1); 2710 } 2711 break; 2712 case VMA_DEL_RESV: 2713 if (vma->vm_flags & VM_MAYSHARE) { 2714 region_abort(resv, idx, idx + 1, 1); 2715 ret = region_del(resv, idx, idx + 1); 2716 } else { 2717 ret = region_add(resv, idx, idx + 1, 1, NULL, NULL); 2718 /* region_add calls of range 1 should never fail. */ 2719 VM_BUG_ON(ret < 0); 2720 } 2721 break; 2722 default: 2723 BUG(); 2724 } 2725 2726 if (vma->vm_flags & VM_MAYSHARE || mode == VMA_DEL_RESV) 2727 return ret; 2728 /* 2729 * We know private mapping must have HPAGE_RESV_OWNER set. 2730 * 2731 * In most cases, reserves always exist for private mappings. 2732 * However, a file associated with mapping could have been 2733 * hole punched or truncated after reserves were consumed. 2734 * As subsequent fault on such a range will not use reserves. 2735 * Subtle - The reserve map for private mappings has the 2736 * opposite meaning than that of shared mappings. If NO 2737 * entry is in the reserve map, it means a reservation exists. 2738 * If an entry exists in the reserve map, it means the 2739 * reservation has already been consumed. As a result, the 2740 * return value of this routine is the opposite of the 2741 * value returned from reserve map manipulation routines above. 2742 */ 2743 if (ret > 0) 2744 return 0; 2745 if (ret == 0) 2746 return 1; 2747 return ret; 2748 } 2749 2750 static long vma_needs_reservation(struct hstate *h, 2751 struct vm_area_struct *vma, unsigned long addr) 2752 { 2753 return __vma_reservation_common(h, vma, addr, VMA_NEEDS_RESV); 2754 } 2755 2756 static long vma_commit_reservation(struct hstate *h, 2757 struct vm_area_struct *vma, unsigned long addr) 2758 { 2759 return __vma_reservation_common(h, vma, addr, VMA_COMMIT_RESV); 2760 } 2761 2762 static void vma_end_reservation(struct hstate *h, 2763 struct vm_area_struct *vma, unsigned long addr) 2764 { 2765 (void)__vma_reservation_common(h, vma, addr, VMA_END_RESV); 2766 } 2767 2768 static long vma_add_reservation(struct hstate *h, 2769 struct vm_area_struct *vma, unsigned long addr) 2770 { 2771 return __vma_reservation_common(h, vma, addr, VMA_ADD_RESV); 2772 } 2773 2774 static long vma_del_reservation(struct hstate *h, 2775 struct vm_area_struct *vma, unsigned long addr) 2776 { 2777 return __vma_reservation_common(h, vma, addr, VMA_DEL_RESV); 2778 } 2779 2780 /* 2781 * This routine is called to restore reservation information on error paths. 2782 * It should ONLY be called for folios allocated via alloc_hugetlb_folio(), 2783 * and the hugetlb mutex should remain held when calling this routine. 2784 * 2785 * It handles two specific cases: 2786 * 1) A reservation was in place and the folio consumed the reservation. 2787 * hugetlb_restore_reserve is set in the folio. 2788 * 2) No reservation was in place for the page, so hugetlb_restore_reserve is 2789 * not set. However, alloc_hugetlb_folio always updates the reserve map. 2790 * 2791 * In case 1, free_huge_folio later in the error path will increment the 2792 * global reserve count. But, free_huge_folio does not have enough context 2793 * to adjust the reservation map. This case deals primarily with private 2794 * mappings. Adjust the reserve map here to be consistent with global 2795 * reserve count adjustments to be made by free_huge_folio. Make sure the 2796 * reserve map indicates there is a reservation present. 2797 * 2798 * In case 2, simply undo reserve map modifications done by alloc_hugetlb_folio. 2799 */ 2800 void restore_reserve_on_error(struct hstate *h, struct vm_area_struct *vma, 2801 unsigned long address, struct folio *folio) 2802 { 2803 long rc = vma_needs_reservation(h, vma, address); 2804 2805 if (folio_test_hugetlb_restore_reserve(folio)) { 2806 if (unlikely(rc < 0)) 2807 /* 2808 * Rare out of memory condition in reserve map 2809 * manipulation. Clear hugetlb_restore_reserve so 2810 * that global reserve count will not be incremented 2811 * by free_huge_folio. This will make it appear 2812 * as though the reservation for this folio was 2813 * consumed. This may prevent the task from 2814 * faulting in the folio at a later time. This 2815 * is better than inconsistent global huge page 2816 * accounting of reserve counts. 2817 */ 2818 folio_clear_hugetlb_restore_reserve(folio); 2819 else if (rc) 2820 (void)vma_add_reservation(h, vma, address); 2821 else 2822 vma_end_reservation(h, vma, address); 2823 } else { 2824 if (!rc) { 2825 /* 2826 * This indicates there is an entry in the reserve map 2827 * not added by alloc_hugetlb_folio. We know it was added 2828 * before the alloc_hugetlb_folio call, otherwise 2829 * hugetlb_restore_reserve would be set on the folio. 2830 * Remove the entry so that a subsequent allocation 2831 * does not consume a reservation. 2832 */ 2833 rc = vma_del_reservation(h, vma, address); 2834 if (rc < 0) 2835 /* 2836 * VERY rare out of memory condition. Since 2837 * we can not delete the entry, set 2838 * hugetlb_restore_reserve so that the reserve 2839 * count will be incremented when the folio 2840 * is freed. This reserve will be consumed 2841 * on a subsequent allocation. 2842 */ 2843 folio_set_hugetlb_restore_reserve(folio); 2844 } else if (rc < 0) { 2845 /* 2846 * Rare out of memory condition from 2847 * vma_needs_reservation call. Memory allocation is 2848 * only attempted if a new entry is needed. Therefore, 2849 * this implies there is not an entry in the 2850 * reserve map. 2851 * 2852 * For shared mappings, no entry in the map indicates 2853 * no reservation. We are done. 2854 */ 2855 if (!(vma->vm_flags & VM_MAYSHARE)) 2856 /* 2857 * For private mappings, no entry indicates 2858 * a reservation is present. Since we can 2859 * not add an entry, set hugetlb_restore_reserve 2860 * on the folio so reserve count will be 2861 * incremented when freed. This reserve will 2862 * be consumed on a subsequent allocation. 2863 */ 2864 folio_set_hugetlb_restore_reserve(folio); 2865 } else 2866 /* 2867 * No reservation present, do nothing 2868 */ 2869 vma_end_reservation(h, vma, address); 2870 } 2871 } 2872 2873 /* 2874 * alloc_and_dissolve_hugetlb_folio - Allocate a new folio and dissolve 2875 * the old one 2876 * @h: struct hstate old page belongs to 2877 * @old_folio: Old folio to dissolve 2878 * @list: List to isolate the page in case we need to 2879 * Returns 0 on success, otherwise negated error. 2880 */ 2881 static int alloc_and_dissolve_hugetlb_folio(struct hstate *h, 2882 struct folio *old_folio, struct list_head *list) 2883 { 2884 gfp_t gfp_mask = htlb_alloc_mask(h) | __GFP_THISNODE; 2885 int nid = folio_nid(old_folio); 2886 struct folio *new_folio; 2887 int ret = 0; 2888 2889 /* 2890 * Before dissolving the folio, we need to allocate a new one for the 2891 * pool to remain stable. Here, we allocate the folio and 'prep' it 2892 * by doing everything but actually updating counters and adding to 2893 * the pool. This simplifies and let us do most of the processing 2894 * under the lock. 2895 */ 2896 new_folio = alloc_buddy_hugetlb_folio(h, gfp_mask, nid, NULL, NULL); 2897 if (!new_folio) 2898 return -ENOMEM; 2899 __prep_new_hugetlb_folio(h, new_folio); 2900 2901 retry: 2902 spin_lock_irq(&hugetlb_lock); 2903 if (!folio_test_hugetlb(old_folio)) { 2904 /* 2905 * Freed from under us. Drop new_folio too. 2906 */ 2907 goto free_new; 2908 } else if (folio_ref_count(old_folio)) { 2909 bool isolated; 2910 2911 /* 2912 * Someone has grabbed the folio, try to isolate it here. 2913 * Fail with -EBUSY if not possible. 2914 */ 2915 spin_unlock_irq(&hugetlb_lock); 2916 isolated = isolate_hugetlb(old_folio, list); 2917 ret = isolated ? 0 : -EBUSY; 2918 spin_lock_irq(&hugetlb_lock); 2919 goto free_new; 2920 } else if (!folio_test_hugetlb_freed(old_folio)) { 2921 /* 2922 * Folio's refcount is 0 but it has not been enqueued in the 2923 * freelist yet. Race window is small, so we can succeed here if 2924 * we retry. 2925 */ 2926 spin_unlock_irq(&hugetlb_lock); 2927 cond_resched(); 2928 goto retry; 2929 } else { 2930 /* 2931 * Ok, old_folio is still a genuine free hugepage. Remove it from 2932 * the freelist and decrease the counters. These will be 2933 * incremented again when calling __prep_account_new_huge_page() 2934 * and enqueue_hugetlb_folio() for new_folio. The counters will 2935 * remain stable since this happens under the lock. 2936 */ 2937 remove_hugetlb_folio(h, old_folio, false); 2938 2939 /* 2940 * Ref count on new_folio is already zero as it was dropped 2941 * earlier. It can be directly added to the pool free list. 2942 */ 2943 __prep_account_new_huge_page(h, nid); 2944 enqueue_hugetlb_folio(h, new_folio); 2945 2946 /* 2947 * Folio has been replaced, we can safely free the old one. 2948 */ 2949 spin_unlock_irq(&hugetlb_lock); 2950 update_and_free_hugetlb_folio(h, old_folio, false); 2951 } 2952 2953 return ret; 2954 2955 free_new: 2956 spin_unlock_irq(&hugetlb_lock); 2957 /* Folio has a zero ref count, but needs a ref to be freed */ 2958 folio_ref_unfreeze(new_folio, 1); 2959 update_and_free_hugetlb_folio(h, new_folio, false); 2960 2961 return ret; 2962 } 2963 2964 int isolate_or_dissolve_huge_page(struct page *page, struct list_head *list) 2965 { 2966 struct hstate *h; 2967 struct folio *folio = page_folio(page); 2968 int ret = -EBUSY; 2969 2970 /* 2971 * The page might have been dissolved from under our feet, so make sure 2972 * to carefully check the state under the lock. 2973 * Return success when racing as if we dissolved the page ourselves. 2974 */ 2975 spin_lock_irq(&hugetlb_lock); 2976 if (folio_test_hugetlb(folio)) { 2977 h = folio_hstate(folio); 2978 } else { 2979 spin_unlock_irq(&hugetlb_lock); 2980 return 0; 2981 } 2982 spin_unlock_irq(&hugetlb_lock); 2983 2984 /* 2985 * Fence off gigantic pages as there is a cyclic dependency between 2986 * alloc_contig_range and them. Return -ENOMEM as this has the effect 2987 * of bailing out right away without further retrying. 2988 */ 2989 if (hstate_is_gigantic(h)) 2990 return -ENOMEM; 2991 2992 if (folio_ref_count(folio) && isolate_hugetlb(folio, list)) 2993 ret = 0; 2994 else if (!folio_ref_count(folio)) 2995 ret = alloc_and_dissolve_hugetlb_folio(h, folio, list); 2996 2997 return ret; 2998 } 2999 3000 struct folio *alloc_hugetlb_folio(struct vm_area_struct *vma, 3001 unsigned long addr, int avoid_reserve) 3002 { 3003 struct hugepage_subpool *spool = subpool_vma(vma); 3004 struct hstate *h = hstate_vma(vma); 3005 struct folio *folio; 3006 long map_chg, map_commit; 3007 long gbl_chg; 3008 int ret, idx; 3009 struct hugetlb_cgroup *h_cg = NULL; 3010 bool deferred_reserve; 3011 3012 idx = hstate_index(h); 3013 /* 3014 * Examine the region/reserve map to determine if the process 3015 * has a reservation for the page to be allocated. A return 3016 * code of zero indicates a reservation exists (no change). 3017 */ 3018 map_chg = gbl_chg = vma_needs_reservation(h, vma, addr); 3019 if (map_chg < 0) 3020 return ERR_PTR(-ENOMEM); 3021 3022 /* 3023 * Processes that did not create the mapping will have no 3024 * reserves as indicated by the region/reserve map. Check 3025 * that the allocation will not exceed the subpool limit. 3026 * Allocations for MAP_NORESERVE mappings also need to be 3027 * checked against any subpool limit. 3028 */ 3029 if (map_chg || avoid_reserve) { 3030 gbl_chg = hugepage_subpool_get_pages(spool, 1); 3031 if (gbl_chg < 0) { 3032 vma_end_reservation(h, vma, addr); 3033 return ERR_PTR(-ENOSPC); 3034 } 3035 3036 /* 3037 * Even though there was no reservation in the region/reserve 3038 * map, there could be reservations associated with the 3039 * subpool that can be used. This would be indicated if the 3040 * return value of hugepage_subpool_get_pages() is zero. 3041 * However, if avoid_reserve is specified we still avoid even 3042 * the subpool reservations. 3043 */ 3044 if (avoid_reserve) 3045 gbl_chg = 1; 3046 } 3047 3048 /* If this allocation is not consuming a reservation, charge it now. 3049 */ 3050 deferred_reserve = map_chg || avoid_reserve; 3051 if (deferred_reserve) { 3052 ret = hugetlb_cgroup_charge_cgroup_rsvd( 3053 idx, pages_per_huge_page(h), &h_cg); 3054 if (ret) 3055 goto out_subpool_put; 3056 } 3057 3058 ret = hugetlb_cgroup_charge_cgroup(idx, pages_per_huge_page(h), &h_cg); 3059 if (ret) 3060 goto out_uncharge_cgroup_reservation; 3061 3062 spin_lock_irq(&hugetlb_lock); 3063 /* 3064 * glb_chg is passed to indicate whether or not a page must be taken 3065 * from the global free pool (global change). gbl_chg == 0 indicates 3066 * a reservation exists for the allocation. 3067 */ 3068 folio = dequeue_hugetlb_folio_vma(h, vma, addr, avoid_reserve, gbl_chg); 3069 if (!folio) { 3070 spin_unlock_irq(&hugetlb_lock); 3071 folio = alloc_buddy_hugetlb_folio_with_mpol(h, vma, addr); 3072 if (!folio) 3073 goto out_uncharge_cgroup; 3074 spin_lock_irq(&hugetlb_lock); 3075 if (!avoid_reserve && vma_has_reserves(vma, gbl_chg)) { 3076 folio_set_hugetlb_restore_reserve(folio); 3077 h->resv_huge_pages--; 3078 } 3079 list_add(&folio->lru, &h->hugepage_activelist); 3080 folio_ref_unfreeze(folio, 1); 3081 /* Fall through */ 3082 } 3083 3084 hugetlb_cgroup_commit_charge(idx, pages_per_huge_page(h), h_cg, folio); 3085 /* If allocation is not consuming a reservation, also store the 3086 * hugetlb_cgroup pointer on the page. 3087 */ 3088 if (deferred_reserve) { 3089 hugetlb_cgroup_commit_charge_rsvd(idx, pages_per_huge_page(h), 3090 h_cg, folio); 3091 } 3092 3093 spin_unlock_irq(&hugetlb_lock); 3094 3095 hugetlb_set_folio_subpool(folio, spool); 3096 3097 map_commit = vma_commit_reservation(h, vma, addr); 3098 if (unlikely(map_chg > map_commit)) { 3099 /* 3100 * The page was added to the reservation map between 3101 * vma_needs_reservation and vma_commit_reservation. 3102 * This indicates a race with hugetlb_reserve_pages. 3103 * Adjust for the subpool count incremented above AND 3104 * in hugetlb_reserve_pages for the same page. Also, 3105 * the reservation count added in hugetlb_reserve_pages 3106 * no longer applies. 3107 */ 3108 long rsv_adjust; 3109 3110 rsv_adjust = hugepage_subpool_put_pages(spool, 1); 3111 hugetlb_acct_memory(h, -rsv_adjust); 3112 if (deferred_reserve) 3113 hugetlb_cgroup_uncharge_folio_rsvd(hstate_index(h), 3114 pages_per_huge_page(h), folio); 3115 } 3116 return folio; 3117 3118 out_uncharge_cgroup: 3119 hugetlb_cgroup_uncharge_cgroup(idx, pages_per_huge_page(h), h_cg); 3120 out_uncharge_cgroup_reservation: 3121 if (deferred_reserve) 3122 hugetlb_cgroup_uncharge_cgroup_rsvd(idx, pages_per_huge_page(h), 3123 h_cg); 3124 out_subpool_put: 3125 if (map_chg || avoid_reserve) 3126 hugepage_subpool_put_pages(spool, 1); 3127 vma_end_reservation(h, vma, addr); 3128 return ERR_PTR(-ENOSPC); 3129 } 3130 3131 int alloc_bootmem_huge_page(struct hstate *h, int nid) 3132 __attribute__ ((weak, alias("__alloc_bootmem_huge_page"))); 3133 int __alloc_bootmem_huge_page(struct hstate *h, int nid) 3134 { 3135 struct huge_bootmem_page *m = NULL; /* initialize for clang */ 3136 int nr_nodes, node; 3137 3138 /* do node specific alloc */ 3139 if (nid != NUMA_NO_NODE) { 3140 m = memblock_alloc_try_nid_raw(huge_page_size(h), huge_page_size(h), 3141 0, MEMBLOCK_ALLOC_ACCESSIBLE, nid); 3142 if (!m) 3143 return 0; 3144 goto found; 3145 } 3146 /* allocate from next node when distributing huge pages */ 3147 for_each_node_mask_to_alloc(h, nr_nodes, node, &node_states[N_MEMORY]) { 3148 m = memblock_alloc_try_nid_raw( 3149 huge_page_size(h), huge_page_size(h), 3150 0, MEMBLOCK_ALLOC_ACCESSIBLE, node); 3151 /* 3152 * Use the beginning of the huge page to store the 3153 * huge_bootmem_page struct (until gather_bootmem 3154 * puts them into the mem_map). 3155 */ 3156 if (!m) 3157 return 0; 3158 goto found; 3159 } 3160 3161 found: 3162 /* Put them into a private list first because mem_map is not up yet */ 3163 INIT_LIST_HEAD(&m->list); 3164 list_add(&m->list, &huge_boot_pages); 3165 m->hstate = h; 3166 return 1; 3167 } 3168 3169 /* 3170 * Put bootmem huge pages into the standard lists after mem_map is up. 3171 * Note: This only applies to gigantic (order > MAX_ORDER) pages. 3172 */ 3173 static void __init gather_bootmem_prealloc(void) 3174 { 3175 struct huge_bootmem_page *m; 3176 3177 list_for_each_entry(m, &huge_boot_pages, list) { 3178 struct page *page = virt_to_page(m); 3179 struct folio *folio = page_folio(page); 3180 struct hstate *h = m->hstate; 3181 3182 VM_BUG_ON(!hstate_is_gigantic(h)); 3183 WARN_ON(folio_ref_count(folio) != 1); 3184 if (prep_compound_gigantic_folio(folio, huge_page_order(h))) { 3185 WARN_ON(folio_test_reserved(folio)); 3186 prep_new_hugetlb_folio(h, folio, folio_nid(folio)); 3187 free_huge_folio(folio); /* add to the hugepage allocator */ 3188 } else { 3189 /* VERY unlikely inflated ref count on a tail page */ 3190 free_gigantic_folio(folio, huge_page_order(h)); 3191 } 3192 3193 /* 3194 * We need to restore the 'stolen' pages to totalram_pages 3195 * in order to fix confusing memory reports from free(1) and 3196 * other side-effects, like CommitLimit going negative. 3197 */ 3198 adjust_managed_page_count(page, pages_per_huge_page(h)); 3199 cond_resched(); 3200 } 3201 } 3202 static void __init hugetlb_hstate_alloc_pages_onenode(struct hstate *h, int nid) 3203 { 3204 unsigned long i; 3205 char buf[32]; 3206 3207 for (i = 0; i < h->max_huge_pages_node[nid]; ++i) { 3208 if (hstate_is_gigantic(h)) { 3209 if (!alloc_bootmem_huge_page(h, nid)) 3210 break; 3211 } else { 3212 struct folio *folio; 3213 gfp_t gfp_mask = htlb_alloc_mask(h) | __GFP_THISNODE; 3214 3215 folio = alloc_fresh_hugetlb_folio(h, gfp_mask, nid, 3216 &node_states[N_MEMORY], NULL); 3217 if (!folio) 3218 break; 3219 free_huge_folio(folio); /* free it into the hugepage allocator */ 3220 } 3221 cond_resched(); 3222 } 3223 if (i == h->max_huge_pages_node[nid]) 3224 return; 3225 3226 string_get_size(huge_page_size(h), 1, STRING_UNITS_2, buf, 32); 3227 pr_warn("HugeTLB: allocating %u of page size %s failed node%d. Only allocated %lu hugepages.\n", 3228 h->max_huge_pages_node[nid], buf, nid, i); 3229 h->max_huge_pages -= (h->max_huge_pages_node[nid] - i); 3230 h->max_huge_pages_node[nid] = i; 3231 } 3232 3233 static void __init hugetlb_hstate_alloc_pages(struct hstate *h) 3234 { 3235 unsigned long i; 3236 nodemask_t *node_alloc_noretry; 3237 bool node_specific_alloc = false; 3238 3239 /* skip gigantic hugepages allocation if hugetlb_cma enabled */ 3240 if (hstate_is_gigantic(h) && hugetlb_cma_size) { 3241 pr_warn_once("HugeTLB: hugetlb_cma is enabled, skip boot time allocation\n"); 3242 return; 3243 } 3244 3245 /* do node specific alloc */ 3246 for_each_online_node(i) { 3247 if (h->max_huge_pages_node[i] > 0) { 3248 hugetlb_hstate_alloc_pages_onenode(h, i); 3249 node_specific_alloc = true; 3250 } 3251 } 3252 3253 if (node_specific_alloc) 3254 return; 3255 3256 /* below will do all node balanced alloc */ 3257 if (!hstate_is_gigantic(h)) { 3258 /* 3259 * Bit mask controlling how hard we retry per-node allocations. 3260 * Ignore errors as lower level routines can deal with 3261 * node_alloc_noretry == NULL. If this kmalloc fails at boot 3262 * time, we are likely in bigger trouble. 3263 */ 3264 node_alloc_noretry = kmalloc(sizeof(*node_alloc_noretry), 3265 GFP_KERNEL); 3266 } else { 3267 /* allocations done at boot time */ 3268 node_alloc_noretry = NULL; 3269 } 3270 3271 /* bit mask controlling how hard we retry per-node allocations */ 3272 if (node_alloc_noretry) 3273 nodes_clear(*node_alloc_noretry); 3274 3275 for (i = 0; i < h->max_huge_pages; ++i) { 3276 if (hstate_is_gigantic(h)) { 3277 if (!alloc_bootmem_huge_page(h, NUMA_NO_NODE)) 3278 break; 3279 } else if (!alloc_pool_huge_page(h, 3280 &node_states[N_MEMORY], 3281 node_alloc_noretry)) 3282 break; 3283 cond_resched(); 3284 } 3285 if (i < h->max_huge_pages) { 3286 char buf[32]; 3287 3288 string_get_size(huge_page_size(h), 1, STRING_UNITS_2, buf, 32); 3289 pr_warn("HugeTLB: allocating %lu of page size %s failed. Only allocated %lu hugepages.\n", 3290 h->max_huge_pages, buf, i); 3291 h->max_huge_pages = i; 3292 } 3293 kfree(node_alloc_noretry); 3294 } 3295 3296 static void __init hugetlb_init_hstates(void) 3297 { 3298 struct hstate *h, *h2; 3299 3300 for_each_hstate(h) { 3301 /* oversize hugepages were init'ed in early boot */ 3302 if (!hstate_is_gigantic(h)) 3303 hugetlb_hstate_alloc_pages(h); 3304 3305 /* 3306 * Set demote order for each hstate. Note that 3307 * h->demote_order is initially 0. 3308 * - We can not demote gigantic pages if runtime freeing 3309 * is not supported, so skip this. 3310 * - If CMA allocation is possible, we can not demote 3311 * HUGETLB_PAGE_ORDER or smaller size pages. 3312 */ 3313 if (hstate_is_gigantic(h) && !gigantic_page_runtime_supported()) 3314 continue; 3315 if (hugetlb_cma_size && h->order <= HUGETLB_PAGE_ORDER) 3316 continue; 3317 for_each_hstate(h2) { 3318 if (h2 == h) 3319 continue; 3320 if (h2->order < h->order && 3321 h2->order > h->demote_order) 3322 h->demote_order = h2->order; 3323 } 3324 } 3325 } 3326 3327 static void __init report_hugepages(void) 3328 { 3329 struct hstate *h; 3330 3331 for_each_hstate(h) { 3332 char buf[32]; 3333 3334 string_get_size(huge_page_size(h), 1, STRING_UNITS_2, buf, 32); 3335 pr_info("HugeTLB: registered %s page size, pre-allocated %ld pages\n", 3336 buf, h->free_huge_pages); 3337 pr_info("HugeTLB: %d KiB vmemmap can be freed for a %s page\n", 3338 hugetlb_vmemmap_optimizable_size(h) / SZ_1K, buf); 3339 } 3340 } 3341 3342 #ifdef CONFIG_HIGHMEM 3343 static void try_to_free_low(struct hstate *h, unsigned long count, 3344 nodemask_t *nodes_allowed) 3345 { 3346 int i; 3347 LIST_HEAD(page_list); 3348 3349 lockdep_assert_held(&hugetlb_lock); 3350 if (hstate_is_gigantic(h)) 3351 return; 3352 3353 /* 3354 * Collect pages to be freed on a list, and free after dropping lock 3355 */ 3356 for_each_node_mask(i, *nodes_allowed) { 3357 struct page *page, *next; 3358 struct list_head *freel = &h->hugepage_freelists[i]; 3359 list_for_each_entry_safe(page, next, freel, lru) { 3360 if (count >= h->nr_huge_pages) 3361 goto out; 3362 if (PageHighMem(page)) 3363 continue; 3364 remove_hugetlb_folio(h, page_folio(page), false); 3365 list_add(&page->lru, &page_list); 3366 } 3367 } 3368 3369 out: 3370 spin_unlock_irq(&hugetlb_lock); 3371 update_and_free_pages_bulk(h, &page_list); 3372 spin_lock_irq(&hugetlb_lock); 3373 } 3374 #else 3375 static inline void try_to_free_low(struct hstate *h, unsigned long count, 3376 nodemask_t *nodes_allowed) 3377 { 3378 } 3379 #endif 3380 3381 /* 3382 * Increment or decrement surplus_huge_pages. Keep node-specific counters 3383 * balanced by operating on them in a round-robin fashion. 3384 * Returns 1 if an adjustment was made. 3385 */ 3386 static int adjust_pool_surplus(struct hstate *h, nodemask_t *nodes_allowed, 3387 int delta) 3388 { 3389 int nr_nodes, node; 3390 3391 lockdep_assert_held(&hugetlb_lock); 3392 VM_BUG_ON(delta != -1 && delta != 1); 3393 3394 if (delta < 0) { 3395 for_each_node_mask_to_alloc(h, nr_nodes, node, nodes_allowed) { 3396 if (h->surplus_huge_pages_node[node]) 3397 goto found; 3398 } 3399 } else { 3400 for_each_node_mask_to_free(h, nr_nodes, node, nodes_allowed) { 3401 if (h->surplus_huge_pages_node[node] < 3402 h->nr_huge_pages_node[node]) 3403 goto found; 3404 } 3405 } 3406 return 0; 3407 3408 found: 3409 h->surplus_huge_pages += delta; 3410 h->surplus_huge_pages_node[node] += delta; 3411 return 1; 3412 } 3413 3414 #define persistent_huge_pages(h) (h->nr_huge_pages - h->surplus_huge_pages) 3415 static int set_max_huge_pages(struct hstate *h, unsigned long count, int nid, 3416 nodemask_t *nodes_allowed) 3417 { 3418 unsigned long min_count, ret; 3419 struct page *page; 3420 LIST_HEAD(page_list); 3421 NODEMASK_ALLOC(nodemask_t, node_alloc_noretry, GFP_KERNEL); 3422 3423 /* 3424 * Bit mask controlling how hard we retry per-node allocations. 3425 * If we can not allocate the bit mask, do not attempt to allocate 3426 * the requested huge pages. 3427 */ 3428 if (node_alloc_noretry) 3429 nodes_clear(*node_alloc_noretry); 3430 else 3431 return -ENOMEM; 3432 3433 /* 3434 * resize_lock mutex prevents concurrent adjustments to number of 3435 * pages in hstate via the proc/sysfs interfaces. 3436 */ 3437 mutex_lock(&h->resize_lock); 3438 flush_free_hpage_work(h); 3439 spin_lock_irq(&hugetlb_lock); 3440 3441 /* 3442 * Check for a node specific request. 3443 * Changing node specific huge page count may require a corresponding 3444 * change to the global count. In any case, the passed node mask 3445 * (nodes_allowed) will restrict alloc/free to the specified node. 3446 */ 3447 if (nid != NUMA_NO_NODE) { 3448 unsigned long old_count = count; 3449 3450 count += h->nr_huge_pages - h->nr_huge_pages_node[nid]; 3451 /* 3452 * User may have specified a large count value which caused the 3453 * above calculation to overflow. In this case, they wanted 3454 * to allocate as many huge pages as possible. Set count to 3455 * largest possible value to align with their intention. 3456 */ 3457 if (count < old_count) 3458 count = ULONG_MAX; 3459 } 3460 3461 /* 3462 * Gigantic pages runtime allocation depend on the capability for large 3463 * page range allocation. 3464 * If the system does not provide this feature, return an error when 3465 * the user tries to allocate gigantic pages but let the user free the 3466 * boottime allocated gigantic pages. 3467 */ 3468 if (hstate_is_gigantic(h) && !IS_ENABLED(CONFIG_CONTIG_ALLOC)) { 3469 if (count > persistent_huge_pages(h)) { 3470 spin_unlock_irq(&hugetlb_lock); 3471 mutex_unlock(&h->resize_lock); 3472 NODEMASK_FREE(node_alloc_noretry); 3473 return -EINVAL; 3474 } 3475 /* Fall through to decrease pool */ 3476 } 3477 3478 /* 3479 * Increase the pool size 3480 * First take pages out of surplus state. Then make up the 3481 * remaining difference by allocating fresh huge pages. 3482 * 3483 * We might race with alloc_surplus_hugetlb_folio() here and be unable 3484 * to convert a surplus huge page to a normal huge page. That is 3485 * not critical, though, it just means the overall size of the 3486 * pool might be one hugepage larger than it needs to be, but 3487 * within all the constraints specified by the sysctls. 3488 */ 3489 while (h->surplus_huge_pages && count > persistent_huge_pages(h)) { 3490 if (!adjust_pool_surplus(h, nodes_allowed, -1)) 3491 break; 3492 } 3493 3494 while (count > persistent_huge_pages(h)) { 3495 /* 3496 * If this allocation races such that we no longer need the 3497 * page, free_huge_folio will handle it by freeing the page 3498 * and reducing the surplus. 3499 */ 3500 spin_unlock_irq(&hugetlb_lock); 3501 3502 /* yield cpu to avoid soft lockup */ 3503 cond_resched(); 3504 3505 ret = alloc_pool_huge_page(h, nodes_allowed, 3506 node_alloc_noretry); 3507 spin_lock_irq(&hugetlb_lock); 3508 if (!ret) 3509 goto out; 3510 3511 /* Bail for signals. Probably ctrl-c from user */ 3512 if (signal_pending(current)) 3513 goto out; 3514 } 3515 3516 /* 3517 * Decrease the pool size 3518 * First return free pages to the buddy allocator (being careful 3519 * to keep enough around to satisfy reservations). Then place 3520 * pages into surplus state as needed so the pool will shrink 3521 * to the desired size as pages become free. 3522 * 3523 * By placing pages into the surplus state independent of the 3524 * overcommit value, we are allowing the surplus pool size to 3525 * exceed overcommit. There are few sane options here. Since 3526 * alloc_surplus_hugetlb_folio() is checking the global counter, 3527 * though, we'll note that we're not allowed to exceed surplus 3528 * and won't grow the pool anywhere else. Not until one of the 3529 * sysctls are changed, or the surplus pages go out of use. 3530 */ 3531 min_count = h->resv_huge_pages + h->nr_huge_pages - h->free_huge_pages; 3532 min_count = max(count, min_count); 3533 try_to_free_low(h, min_count, nodes_allowed); 3534 3535 /* 3536 * Collect pages to be removed on list without dropping lock 3537 */ 3538 while (min_count < persistent_huge_pages(h)) { 3539 page = remove_pool_huge_page(h, nodes_allowed, 0); 3540 if (!page) 3541 break; 3542 3543 list_add(&page->lru, &page_list); 3544 } 3545 /* free the pages after dropping lock */ 3546 spin_unlock_irq(&hugetlb_lock); 3547 update_and_free_pages_bulk(h, &page_list); 3548 flush_free_hpage_work(h); 3549 spin_lock_irq(&hugetlb_lock); 3550 3551 while (count < persistent_huge_pages(h)) { 3552 if (!adjust_pool_surplus(h, nodes_allowed, 1)) 3553 break; 3554 } 3555 out: 3556 h->max_huge_pages = persistent_huge_pages(h); 3557 spin_unlock_irq(&hugetlb_lock); 3558 mutex_unlock(&h->resize_lock); 3559 3560 NODEMASK_FREE(node_alloc_noretry); 3561 3562 return 0; 3563 } 3564 3565 static int demote_free_hugetlb_folio(struct hstate *h, struct folio *folio) 3566 { 3567 int i, nid = folio_nid(folio); 3568 struct hstate *target_hstate; 3569 struct page *subpage; 3570 struct folio *inner_folio; 3571 int rc = 0; 3572 3573 target_hstate = size_to_hstate(PAGE_SIZE << h->demote_order); 3574 3575 remove_hugetlb_folio_for_demote(h, folio, false); 3576 spin_unlock_irq(&hugetlb_lock); 3577 3578 rc = hugetlb_vmemmap_restore(h, &folio->page); 3579 if (rc) { 3580 /* Allocation of vmemmmap failed, we can not demote folio */ 3581 spin_lock_irq(&hugetlb_lock); 3582 folio_ref_unfreeze(folio, 1); 3583 add_hugetlb_folio(h, folio, false); 3584 return rc; 3585 } 3586 3587 /* 3588 * Use destroy_compound_hugetlb_folio_for_demote for all huge page 3589 * sizes as it will not ref count folios. 3590 */ 3591 destroy_compound_hugetlb_folio_for_demote(folio, huge_page_order(h)); 3592 3593 /* 3594 * Taking target hstate mutex synchronizes with set_max_huge_pages. 3595 * Without the mutex, pages added to target hstate could be marked 3596 * as surplus. 3597 * 3598 * Note that we already hold h->resize_lock. To prevent deadlock, 3599 * use the convention of always taking larger size hstate mutex first. 3600 */ 3601 mutex_lock(&target_hstate->resize_lock); 3602 for (i = 0; i < pages_per_huge_page(h); 3603 i += pages_per_huge_page(target_hstate)) { 3604 subpage = folio_page(folio, i); 3605 inner_folio = page_folio(subpage); 3606 if (hstate_is_gigantic(target_hstate)) 3607 prep_compound_gigantic_folio_for_demote(inner_folio, 3608 target_hstate->order); 3609 else 3610 prep_compound_page(subpage, target_hstate->order); 3611 folio_change_private(inner_folio, NULL); 3612 prep_new_hugetlb_folio(target_hstate, inner_folio, nid); 3613 free_huge_folio(inner_folio); 3614 } 3615 mutex_unlock(&target_hstate->resize_lock); 3616 3617 spin_lock_irq(&hugetlb_lock); 3618 3619 /* 3620 * Not absolutely necessary, but for consistency update max_huge_pages 3621 * based on pool changes for the demoted page. 3622 */ 3623 h->max_huge_pages--; 3624 target_hstate->max_huge_pages += 3625 pages_per_huge_page(h) / pages_per_huge_page(target_hstate); 3626 3627 return rc; 3628 } 3629 3630 static int demote_pool_huge_page(struct hstate *h, nodemask_t *nodes_allowed) 3631 __must_hold(&hugetlb_lock) 3632 { 3633 int nr_nodes, node; 3634 struct folio *folio; 3635 3636 lockdep_assert_held(&hugetlb_lock); 3637 3638 /* We should never get here if no demote order */ 3639 if (!h->demote_order) { 3640 pr_warn("HugeTLB: NULL demote order passed to demote_pool_huge_page.\n"); 3641 return -EINVAL; /* internal error */ 3642 } 3643 3644 for_each_node_mask_to_free(h, nr_nodes, node, nodes_allowed) { 3645 list_for_each_entry(folio, &h->hugepage_freelists[node], lru) { 3646 if (folio_test_hwpoison(folio)) 3647 continue; 3648 return demote_free_hugetlb_folio(h, folio); 3649 } 3650 } 3651 3652 /* 3653 * Only way to get here is if all pages on free lists are poisoned. 3654 * Return -EBUSY so that caller will not retry. 3655 */ 3656 return -EBUSY; 3657 } 3658 3659 #define HSTATE_ATTR_RO(_name) \ 3660 static struct kobj_attribute _name##_attr = __ATTR_RO(_name) 3661 3662 #define HSTATE_ATTR_WO(_name) \ 3663 static struct kobj_attribute _name##_attr = __ATTR_WO(_name) 3664 3665 #define HSTATE_ATTR(_name) \ 3666 static struct kobj_attribute _name##_attr = __ATTR_RW(_name) 3667 3668 static struct kobject *hugepages_kobj; 3669 static struct kobject *hstate_kobjs[HUGE_MAX_HSTATE]; 3670 3671 static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp); 3672 3673 static struct hstate *kobj_to_hstate(struct kobject *kobj, int *nidp) 3674 { 3675 int i; 3676 3677 for (i = 0; i < HUGE_MAX_HSTATE; i++) 3678 if (hstate_kobjs[i] == kobj) { 3679 if (nidp) 3680 *nidp = NUMA_NO_NODE; 3681 return &hstates[i]; 3682 } 3683 3684 return kobj_to_node_hstate(kobj, nidp); 3685 } 3686 3687 static ssize_t nr_hugepages_show_common(struct kobject *kobj, 3688 struct kobj_attribute *attr, char *buf) 3689 { 3690 struct hstate *h; 3691 unsigned long nr_huge_pages; 3692 int nid; 3693 3694 h = kobj_to_hstate(kobj, &nid); 3695 if (nid == NUMA_NO_NODE) 3696 nr_huge_pages = h->nr_huge_pages; 3697 else 3698 nr_huge_pages = h->nr_huge_pages_node[nid]; 3699 3700 return sysfs_emit(buf, "%lu\n", nr_huge_pages); 3701 } 3702 3703 static ssize_t __nr_hugepages_store_common(bool obey_mempolicy, 3704 struct hstate *h, int nid, 3705 unsigned long count, size_t len) 3706 { 3707 int err; 3708 nodemask_t nodes_allowed, *n_mask; 3709 3710 if (hstate_is_gigantic(h) && !gigantic_page_runtime_supported()) 3711 return -EINVAL; 3712 3713 if (nid == NUMA_NO_NODE) { 3714 /* 3715 * global hstate attribute 3716 */ 3717 if (!(obey_mempolicy && 3718 init_nodemask_of_mempolicy(&nodes_allowed))) 3719 n_mask = &node_states[N_MEMORY]; 3720 else 3721 n_mask = &nodes_allowed; 3722 } else { 3723 /* 3724 * Node specific request. count adjustment happens in 3725 * set_max_huge_pages() after acquiring hugetlb_lock. 3726 */ 3727 init_nodemask_of_node(&nodes_allowed, nid); 3728 n_mask = &nodes_allowed; 3729 } 3730 3731 err = set_max_huge_pages(h, count, nid, n_mask); 3732 3733 return err ? err : len; 3734 } 3735 3736 static ssize_t nr_hugepages_store_common(bool obey_mempolicy, 3737 struct kobject *kobj, const char *buf, 3738 size_t len) 3739 { 3740 struct hstate *h; 3741 unsigned long count; 3742 int nid; 3743 int err; 3744 3745 err = kstrtoul(buf, 10, &count); 3746 if (err) 3747 return err; 3748 3749 h = kobj_to_hstate(kobj, &nid); 3750 return __nr_hugepages_store_common(obey_mempolicy, h, nid, count, len); 3751 } 3752 3753 static ssize_t nr_hugepages_show(struct kobject *kobj, 3754 struct kobj_attribute *attr, char *buf) 3755 { 3756 return nr_hugepages_show_common(kobj, attr, buf); 3757 } 3758 3759 static ssize_t nr_hugepages_store(struct kobject *kobj, 3760 struct kobj_attribute *attr, const char *buf, size_t len) 3761 { 3762 return nr_hugepages_store_common(false, kobj, buf, len); 3763 } 3764 HSTATE_ATTR(nr_hugepages); 3765 3766 #ifdef CONFIG_NUMA 3767 3768 /* 3769 * hstate attribute for optionally mempolicy-based constraint on persistent 3770 * huge page alloc/free. 3771 */ 3772 static ssize_t nr_hugepages_mempolicy_show(struct kobject *kobj, 3773 struct kobj_attribute *attr, 3774 char *buf) 3775 { 3776 return nr_hugepages_show_common(kobj, attr, buf); 3777 } 3778 3779 static ssize_t nr_hugepages_mempolicy_store(struct kobject *kobj, 3780 struct kobj_attribute *attr, const char *buf, size_t len) 3781 { 3782 return nr_hugepages_store_common(true, kobj, buf, len); 3783 } 3784 HSTATE_ATTR(nr_hugepages_mempolicy); 3785 #endif 3786 3787 3788 static ssize_t nr_overcommit_hugepages_show(struct kobject *kobj, 3789 struct kobj_attribute *attr, char *buf) 3790 { 3791 struct hstate *h = kobj_to_hstate(kobj, NULL); 3792 return sysfs_emit(buf, "%lu\n", h->nr_overcommit_huge_pages); 3793 } 3794 3795 static ssize_t nr_overcommit_hugepages_store(struct kobject *kobj, 3796 struct kobj_attribute *attr, const char *buf, size_t count) 3797 { 3798 int err; 3799 unsigned long input; 3800 struct hstate *h = kobj_to_hstate(kobj, NULL); 3801 3802 if (hstate_is_gigantic(h)) 3803 return -EINVAL; 3804 3805 err = kstrtoul(buf, 10, &input); 3806 if (err) 3807 return err; 3808 3809 spin_lock_irq(&hugetlb_lock); 3810 h->nr_overcommit_huge_pages = input; 3811 spin_unlock_irq(&hugetlb_lock); 3812 3813 return count; 3814 } 3815 HSTATE_ATTR(nr_overcommit_hugepages); 3816 3817 static ssize_t free_hugepages_show(struct kobject *kobj, 3818 struct kobj_attribute *attr, char *buf) 3819 { 3820 struct hstate *h; 3821 unsigned long free_huge_pages; 3822 int nid; 3823 3824 h = kobj_to_hstate(kobj, &nid); 3825 if (nid == NUMA_NO_NODE) 3826 free_huge_pages = h->free_huge_pages; 3827 else 3828 free_huge_pages = h->free_huge_pages_node[nid]; 3829 3830 return sysfs_emit(buf, "%lu\n", free_huge_pages); 3831 } 3832 HSTATE_ATTR_RO(free_hugepages); 3833 3834 static ssize_t resv_hugepages_show(struct kobject *kobj, 3835 struct kobj_attribute *attr, char *buf) 3836 { 3837 struct hstate *h = kobj_to_hstate(kobj, NULL); 3838 return sysfs_emit(buf, "%lu\n", h->resv_huge_pages); 3839 } 3840 HSTATE_ATTR_RO(resv_hugepages); 3841 3842 static ssize_t surplus_hugepages_show(struct kobject *kobj, 3843 struct kobj_attribute *attr, char *buf) 3844 { 3845 struct hstate *h; 3846 unsigned long surplus_huge_pages; 3847 int nid; 3848 3849 h = kobj_to_hstate(kobj, &nid); 3850 if (nid == NUMA_NO_NODE) 3851 surplus_huge_pages = h->surplus_huge_pages; 3852 else 3853 surplus_huge_pages = h->surplus_huge_pages_node[nid]; 3854 3855 return sysfs_emit(buf, "%lu\n", surplus_huge_pages); 3856 } 3857 HSTATE_ATTR_RO(surplus_hugepages); 3858 3859 static ssize_t demote_store(struct kobject *kobj, 3860 struct kobj_attribute *attr, const char *buf, size_t len) 3861 { 3862 unsigned long nr_demote; 3863 unsigned long nr_available; 3864 nodemask_t nodes_allowed, *n_mask; 3865 struct hstate *h; 3866 int err; 3867 int nid; 3868 3869 err = kstrtoul(buf, 10, &nr_demote); 3870 if (err) 3871 return err; 3872 h = kobj_to_hstate(kobj, &nid); 3873 3874 if (nid != NUMA_NO_NODE) { 3875 init_nodemask_of_node(&nodes_allowed, nid); 3876 n_mask = &nodes_allowed; 3877 } else { 3878 n_mask = &node_states[N_MEMORY]; 3879 } 3880 3881 /* Synchronize with other sysfs operations modifying huge pages */ 3882 mutex_lock(&h->resize_lock); 3883 spin_lock_irq(&hugetlb_lock); 3884 3885 while (nr_demote) { 3886 /* 3887 * Check for available pages to demote each time thorough the 3888 * loop as demote_pool_huge_page will drop hugetlb_lock. 3889 */ 3890 if (nid != NUMA_NO_NODE) 3891 nr_available = h->free_huge_pages_node[nid]; 3892 else 3893 nr_available = h->free_huge_pages; 3894 nr_available -= h->resv_huge_pages; 3895 if (!nr_available) 3896 break; 3897 3898 err = demote_pool_huge_page(h, n_mask); 3899 if (err) 3900 break; 3901 3902 nr_demote--; 3903 } 3904 3905 spin_unlock_irq(&hugetlb_lock); 3906 mutex_unlock(&h->resize_lock); 3907 3908 if (err) 3909 return err; 3910 return len; 3911 } 3912 HSTATE_ATTR_WO(demote); 3913 3914 static ssize_t demote_size_show(struct kobject *kobj, 3915 struct kobj_attribute *attr, char *buf) 3916 { 3917 struct hstate *h = kobj_to_hstate(kobj, NULL); 3918 unsigned long demote_size = (PAGE_SIZE << h->demote_order) / SZ_1K; 3919 3920 return sysfs_emit(buf, "%lukB\n", demote_size); 3921 } 3922 3923 static ssize_t demote_size_store(struct kobject *kobj, 3924 struct kobj_attribute *attr, 3925 const char *buf, size_t count) 3926 { 3927 struct hstate *h, *demote_hstate; 3928 unsigned long demote_size; 3929 unsigned int demote_order; 3930 3931 demote_size = (unsigned long)memparse(buf, NULL); 3932 3933 demote_hstate = size_to_hstate(demote_size); 3934 if (!demote_hstate) 3935 return -EINVAL; 3936 demote_order = demote_hstate->order; 3937 if (demote_order < HUGETLB_PAGE_ORDER) 3938 return -EINVAL; 3939 3940 /* demote order must be smaller than hstate order */ 3941 h = kobj_to_hstate(kobj, NULL); 3942 if (demote_order >= h->order) 3943 return -EINVAL; 3944 3945 /* resize_lock synchronizes access to demote size and writes */ 3946 mutex_lock(&h->resize_lock); 3947 h->demote_order = demote_order; 3948 mutex_unlock(&h->resize_lock); 3949 3950 return count; 3951 } 3952 HSTATE_ATTR(demote_size); 3953 3954 static struct attribute *hstate_attrs[] = { 3955 &nr_hugepages_attr.attr, 3956 &nr_overcommit_hugepages_attr.attr, 3957 &free_hugepages_attr.attr, 3958 &resv_hugepages_attr.attr, 3959 &surplus_hugepages_attr.attr, 3960 #ifdef CONFIG_NUMA 3961 &nr_hugepages_mempolicy_attr.attr, 3962 #endif 3963 NULL, 3964 }; 3965 3966 static const struct attribute_group hstate_attr_group = { 3967 .attrs = hstate_attrs, 3968 }; 3969 3970 static struct attribute *hstate_demote_attrs[] = { 3971 &demote_size_attr.attr, 3972 &demote_attr.attr, 3973 NULL, 3974 }; 3975 3976 static const struct attribute_group hstate_demote_attr_group = { 3977 .attrs = hstate_demote_attrs, 3978 }; 3979 3980 static int hugetlb_sysfs_add_hstate(struct hstate *h, struct kobject *parent, 3981 struct kobject **hstate_kobjs, 3982 const struct attribute_group *hstate_attr_group) 3983 { 3984 int retval; 3985 int hi = hstate_index(h); 3986 3987 hstate_kobjs[hi] = kobject_create_and_add(h->name, parent); 3988 if (!hstate_kobjs[hi]) 3989 return -ENOMEM; 3990 3991 retval = sysfs_create_group(hstate_kobjs[hi], hstate_attr_group); 3992 if (retval) { 3993 kobject_put(hstate_kobjs[hi]); 3994 hstate_kobjs[hi] = NULL; 3995 return retval; 3996 } 3997 3998 if (h->demote_order) { 3999 retval = sysfs_create_group(hstate_kobjs[hi], 4000 &hstate_demote_attr_group); 4001 if (retval) { 4002 pr_warn("HugeTLB unable to create demote interfaces for %s\n", h->name); 4003 sysfs_remove_group(hstate_kobjs[hi], hstate_attr_group); 4004 kobject_put(hstate_kobjs[hi]); 4005 hstate_kobjs[hi] = NULL; 4006 return retval; 4007 } 4008 } 4009 4010 return 0; 4011 } 4012 4013 #ifdef CONFIG_NUMA 4014 static bool hugetlb_sysfs_initialized __ro_after_init; 4015 4016 /* 4017 * node_hstate/s - associate per node hstate attributes, via their kobjects, 4018 * with node devices in node_devices[] using a parallel array. The array 4019 * index of a node device or _hstate == node id. 4020 * This is here to avoid any static dependency of the node device driver, in 4021 * the base kernel, on the hugetlb module. 4022 */ 4023 struct node_hstate { 4024 struct kobject *hugepages_kobj; 4025 struct kobject *hstate_kobjs[HUGE_MAX_HSTATE]; 4026 }; 4027 static struct node_hstate node_hstates[MAX_NUMNODES]; 4028 4029 /* 4030 * A subset of global hstate attributes for node devices 4031 */ 4032 static struct attribute *per_node_hstate_attrs[] = { 4033 &nr_hugepages_attr.attr, 4034 &free_hugepages_attr.attr, 4035 &surplus_hugepages_attr.attr, 4036 NULL, 4037 }; 4038 4039 static const struct attribute_group per_node_hstate_attr_group = { 4040 .attrs = per_node_hstate_attrs, 4041 }; 4042 4043 /* 4044 * kobj_to_node_hstate - lookup global hstate for node device hstate attr kobj. 4045 * Returns node id via non-NULL nidp. 4046 */ 4047 static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp) 4048 { 4049 int nid; 4050 4051 for (nid = 0; nid < nr_node_ids; nid++) { 4052 struct node_hstate *nhs = &node_hstates[nid]; 4053 int i; 4054 for (i = 0; i < HUGE_MAX_HSTATE; i++) 4055 if (nhs->hstate_kobjs[i] == kobj) { 4056 if (nidp) 4057 *nidp = nid; 4058 return &hstates[i]; 4059 } 4060 } 4061 4062 BUG(); 4063 return NULL; 4064 } 4065 4066 /* 4067 * Unregister hstate attributes from a single node device. 4068 * No-op if no hstate attributes attached. 4069 */ 4070 void hugetlb_unregister_node(struct node *node) 4071 { 4072 struct hstate *h; 4073 struct node_hstate *nhs = &node_hstates[node->dev.id]; 4074 4075 if (!nhs->hugepages_kobj) 4076 return; /* no hstate attributes */ 4077 4078 for_each_hstate(h) { 4079 int idx = hstate_index(h); 4080 struct kobject *hstate_kobj = nhs->hstate_kobjs[idx]; 4081 4082 if (!hstate_kobj) 4083 continue; 4084 if (h->demote_order) 4085 sysfs_remove_group(hstate_kobj, &hstate_demote_attr_group); 4086 sysfs_remove_group(hstate_kobj, &per_node_hstate_attr_group); 4087 kobject_put(hstate_kobj); 4088 nhs->hstate_kobjs[idx] = NULL; 4089 } 4090 4091 kobject_put(nhs->hugepages_kobj); 4092 nhs->hugepages_kobj = NULL; 4093 } 4094 4095 4096 /* 4097 * Register hstate attributes for a single node device. 4098 * No-op if attributes already registered. 4099 */ 4100 void hugetlb_register_node(struct node *node) 4101 { 4102 struct hstate *h; 4103 struct node_hstate *nhs = &node_hstates[node->dev.id]; 4104 int err; 4105 4106 if (!hugetlb_sysfs_initialized) 4107 return; 4108 4109 if (nhs->hugepages_kobj) 4110 return; /* already allocated */ 4111 4112 nhs->hugepages_kobj = kobject_create_and_add("hugepages", 4113 &node->dev.kobj); 4114 if (!nhs->hugepages_kobj) 4115 return; 4116 4117 for_each_hstate(h) { 4118 err = hugetlb_sysfs_add_hstate(h, nhs->hugepages_kobj, 4119 nhs->hstate_kobjs, 4120 &per_node_hstate_attr_group); 4121 if (err) { 4122 pr_err("HugeTLB: Unable to add hstate %s for node %d\n", 4123 h->name, node->dev.id); 4124 hugetlb_unregister_node(node); 4125 break; 4126 } 4127 } 4128 } 4129 4130 /* 4131 * hugetlb init time: register hstate attributes for all registered node 4132 * devices of nodes that have memory. All on-line nodes should have 4133 * registered their associated device by this time. 4134 */ 4135 static void __init hugetlb_register_all_nodes(void) 4136 { 4137 int nid; 4138 4139 for_each_online_node(nid) 4140 hugetlb_register_node(node_devices[nid]); 4141 } 4142 #else /* !CONFIG_NUMA */ 4143 4144 static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp) 4145 { 4146 BUG(); 4147 if (nidp) 4148 *nidp = -1; 4149 return NULL; 4150 } 4151 4152 static void hugetlb_register_all_nodes(void) { } 4153 4154 #endif 4155 4156 #ifdef CONFIG_CMA 4157 static void __init hugetlb_cma_check(void); 4158 #else 4159 static inline __init void hugetlb_cma_check(void) 4160 { 4161 } 4162 #endif 4163 4164 static void __init hugetlb_sysfs_init(void) 4165 { 4166 struct hstate *h; 4167 int err; 4168 4169 hugepages_kobj = kobject_create_and_add("hugepages", mm_kobj); 4170 if (!hugepages_kobj) 4171 return; 4172 4173 for_each_hstate(h) { 4174 err = hugetlb_sysfs_add_hstate(h, hugepages_kobj, 4175 hstate_kobjs, &hstate_attr_group); 4176 if (err) 4177 pr_err("HugeTLB: Unable to add hstate %s", h->name); 4178 } 4179 4180 #ifdef CONFIG_NUMA 4181 hugetlb_sysfs_initialized = true; 4182 #endif 4183 hugetlb_register_all_nodes(); 4184 } 4185 4186 #ifdef CONFIG_SYSCTL 4187 static void hugetlb_sysctl_init(void); 4188 #else 4189 static inline void hugetlb_sysctl_init(void) { } 4190 #endif 4191 4192 static int __init hugetlb_init(void) 4193 { 4194 int i; 4195 4196 BUILD_BUG_ON(sizeof_field(struct page, private) * BITS_PER_BYTE < 4197 __NR_HPAGEFLAGS); 4198 4199 if (!hugepages_supported()) { 4200 if (hugetlb_max_hstate || default_hstate_max_huge_pages) 4201 pr_warn("HugeTLB: huge pages not supported, ignoring associated command-line parameters\n"); 4202 return 0; 4203 } 4204 4205 /* 4206 * Make sure HPAGE_SIZE (HUGETLB_PAGE_ORDER) hstate exists. Some 4207 * architectures depend on setup being done here. 4208 */ 4209 hugetlb_add_hstate(HUGETLB_PAGE_ORDER); 4210 if (!parsed_default_hugepagesz) { 4211 /* 4212 * If we did not parse a default huge page size, set 4213 * default_hstate_idx to HPAGE_SIZE hstate. And, if the 4214 * number of huge pages for this default size was implicitly 4215 * specified, set that here as well. 4216 * Note that the implicit setting will overwrite an explicit 4217 * setting. A warning will be printed in this case. 4218 */ 4219 default_hstate_idx = hstate_index(size_to_hstate(HPAGE_SIZE)); 4220 if (default_hstate_max_huge_pages) { 4221 if (default_hstate.max_huge_pages) { 4222 char buf[32]; 4223 4224 string_get_size(huge_page_size(&default_hstate), 4225 1, STRING_UNITS_2, buf, 32); 4226 pr_warn("HugeTLB: Ignoring hugepages=%lu associated with %s page size\n", 4227 default_hstate.max_huge_pages, buf); 4228 pr_warn("HugeTLB: Using hugepages=%lu for number of default huge pages\n", 4229 default_hstate_max_huge_pages); 4230 } 4231 default_hstate.max_huge_pages = 4232 default_hstate_max_huge_pages; 4233 4234 for_each_online_node(i) 4235 default_hstate.max_huge_pages_node[i] = 4236 default_hugepages_in_node[i]; 4237 } 4238 } 4239 4240 hugetlb_cma_check(); 4241 hugetlb_init_hstates(); 4242 gather_bootmem_prealloc(); 4243 report_hugepages(); 4244 4245 hugetlb_sysfs_init(); 4246 hugetlb_cgroup_file_init(); 4247 hugetlb_sysctl_init(); 4248 4249 #ifdef CONFIG_SMP 4250 num_fault_mutexes = roundup_pow_of_two(8 * num_possible_cpus()); 4251 #else 4252 num_fault_mutexes = 1; 4253 #endif 4254 hugetlb_fault_mutex_table = 4255 kmalloc_array(num_fault_mutexes, sizeof(struct mutex), 4256 GFP_KERNEL); 4257 BUG_ON(!hugetlb_fault_mutex_table); 4258 4259 for (i = 0; i < num_fault_mutexes; i++) 4260 mutex_init(&hugetlb_fault_mutex_table[i]); 4261 return 0; 4262 } 4263 subsys_initcall(hugetlb_init); 4264 4265 /* Overwritten by architectures with more huge page sizes */ 4266 bool __init __attribute((weak)) arch_hugetlb_valid_size(unsigned long size) 4267 { 4268 return size == HPAGE_SIZE; 4269 } 4270 4271 void __init hugetlb_add_hstate(unsigned int order) 4272 { 4273 struct hstate *h; 4274 unsigned long i; 4275 4276 if (size_to_hstate(PAGE_SIZE << order)) { 4277 return; 4278 } 4279 BUG_ON(hugetlb_max_hstate >= HUGE_MAX_HSTATE); 4280 BUG_ON(order == 0); 4281 h = &hstates[hugetlb_max_hstate++]; 4282 mutex_init(&h->resize_lock); 4283 h->order = order; 4284 h->mask = ~(huge_page_size(h) - 1); 4285 for (i = 0; i < MAX_NUMNODES; ++i) 4286 INIT_LIST_HEAD(&h->hugepage_freelists[i]); 4287 INIT_LIST_HEAD(&h->hugepage_activelist); 4288 h->next_nid_to_alloc = first_memory_node; 4289 h->next_nid_to_free = first_memory_node; 4290 snprintf(h->name, HSTATE_NAME_LEN, "hugepages-%lukB", 4291 huge_page_size(h)/SZ_1K); 4292 4293 parsed_hstate = h; 4294 } 4295 4296 bool __init __weak hugetlb_node_alloc_supported(void) 4297 { 4298 return true; 4299 } 4300 4301 static void __init hugepages_clear_pages_in_node(void) 4302 { 4303 if (!hugetlb_max_hstate) { 4304 default_hstate_max_huge_pages = 0; 4305 memset(default_hugepages_in_node, 0, 4306 sizeof(default_hugepages_in_node)); 4307 } else { 4308 parsed_hstate->max_huge_pages = 0; 4309 memset(parsed_hstate->max_huge_pages_node, 0, 4310 sizeof(parsed_hstate->max_huge_pages_node)); 4311 } 4312 } 4313 4314 /* 4315 * hugepages command line processing 4316 * hugepages normally follows a valid hugepagsz or default_hugepagsz 4317 * specification. If not, ignore the hugepages value. hugepages can also 4318 * be the first huge page command line option in which case it implicitly 4319 * specifies the number of huge pages for the default size. 4320 */ 4321 static int __init hugepages_setup(char *s) 4322 { 4323 unsigned long *mhp; 4324 static unsigned long *last_mhp; 4325 int node = NUMA_NO_NODE; 4326 int count; 4327 unsigned long tmp; 4328 char *p = s; 4329 4330 if (!parsed_valid_hugepagesz) { 4331 pr_warn("HugeTLB: hugepages=%s does not follow a valid hugepagesz, ignoring\n", s); 4332 parsed_valid_hugepagesz = true; 4333 return 1; 4334 } 4335 4336 /* 4337 * !hugetlb_max_hstate means we haven't parsed a hugepagesz= parameter 4338 * yet, so this hugepages= parameter goes to the "default hstate". 4339 * Otherwise, it goes with the previously parsed hugepagesz or 4340 * default_hugepagesz. 4341 */ 4342 else if (!hugetlb_max_hstate) 4343 mhp = &default_hstate_max_huge_pages; 4344 else 4345 mhp = &parsed_hstate->max_huge_pages; 4346 4347 if (mhp == last_mhp) { 4348 pr_warn("HugeTLB: hugepages= specified twice without interleaving hugepagesz=, ignoring hugepages=%s\n", s); 4349 return 1; 4350 } 4351 4352 while (*p) { 4353 count = 0; 4354 if (sscanf(p, "%lu%n", &tmp, &count) != 1) 4355 goto invalid; 4356 /* Parameter is node format */ 4357 if (p[count] == ':') { 4358 if (!hugetlb_node_alloc_supported()) { 4359 pr_warn("HugeTLB: architecture can't support node specific alloc, ignoring!\n"); 4360 return 1; 4361 } 4362 if (tmp >= MAX_NUMNODES || !node_online(tmp)) 4363 goto invalid; 4364 node = array_index_nospec(tmp, MAX_NUMNODES); 4365 p += count + 1; 4366 /* Parse hugepages */ 4367 if (sscanf(p, "%lu%n", &tmp, &count) != 1) 4368 goto invalid; 4369 if (!hugetlb_max_hstate) 4370 default_hugepages_in_node[node] = tmp; 4371 else 4372 parsed_hstate->max_huge_pages_node[node] = tmp; 4373 *mhp += tmp; 4374 /* Go to parse next node*/ 4375 if (p[count] == ',') 4376 p += count + 1; 4377 else 4378 break; 4379 } else { 4380 if (p != s) 4381 goto invalid; 4382 *mhp = tmp; 4383 break; 4384 } 4385 } 4386 4387 /* 4388 * Global state is always initialized later in hugetlb_init. 4389 * But we need to allocate gigantic hstates here early to still 4390 * use the bootmem allocator. 4391 */ 4392 if (hugetlb_max_hstate && hstate_is_gigantic(parsed_hstate)) 4393 hugetlb_hstate_alloc_pages(parsed_hstate); 4394 4395 last_mhp = mhp; 4396 4397 return 1; 4398 4399 invalid: 4400 pr_warn("HugeTLB: Invalid hugepages parameter %s\n", p); 4401 hugepages_clear_pages_in_node(); 4402 return 1; 4403 } 4404 __setup("hugepages=", hugepages_setup); 4405 4406 /* 4407 * hugepagesz command line processing 4408 * A specific huge page size can only be specified once with hugepagesz. 4409 * hugepagesz is followed by hugepages on the command line. The global 4410 * variable 'parsed_valid_hugepagesz' is used to determine if prior 4411 * hugepagesz argument was valid. 4412 */ 4413 static int __init hugepagesz_setup(char *s) 4414 { 4415 unsigned long size; 4416 struct hstate *h; 4417 4418 parsed_valid_hugepagesz = false; 4419 size = (unsigned long)memparse(s, NULL); 4420 4421 if (!arch_hugetlb_valid_size(size)) { 4422 pr_err("HugeTLB: unsupported hugepagesz=%s\n", s); 4423 return 1; 4424 } 4425 4426 h = size_to_hstate(size); 4427 if (h) { 4428 /* 4429 * hstate for this size already exists. This is normally 4430 * an error, but is allowed if the existing hstate is the 4431 * default hstate. More specifically, it is only allowed if 4432 * the number of huge pages for the default hstate was not 4433 * previously specified. 4434 */ 4435 if (!parsed_default_hugepagesz || h != &default_hstate || 4436 default_hstate.max_huge_pages) { 4437 pr_warn("HugeTLB: hugepagesz=%s specified twice, ignoring\n", s); 4438 return 1; 4439 } 4440 4441 /* 4442 * No need to call hugetlb_add_hstate() as hstate already 4443 * exists. But, do set parsed_hstate so that a following 4444 * hugepages= parameter will be applied to this hstate. 4445 */ 4446 parsed_hstate = h; 4447 parsed_valid_hugepagesz = true; 4448 return 1; 4449 } 4450 4451 hugetlb_add_hstate(ilog2(size) - PAGE_SHIFT); 4452 parsed_valid_hugepagesz = true; 4453 return 1; 4454 } 4455 __setup("hugepagesz=", hugepagesz_setup); 4456 4457 /* 4458 * default_hugepagesz command line input 4459 * Only one instance of default_hugepagesz allowed on command line. 4460 */ 4461 static int __init default_hugepagesz_setup(char *s) 4462 { 4463 unsigned long size; 4464 int i; 4465 4466 parsed_valid_hugepagesz = false; 4467 if (parsed_default_hugepagesz) { 4468 pr_err("HugeTLB: default_hugepagesz previously specified, ignoring %s\n", s); 4469 return 1; 4470 } 4471 4472 size = (unsigned long)memparse(s, NULL); 4473 4474 if (!arch_hugetlb_valid_size(size)) { 4475 pr_err("HugeTLB: unsupported default_hugepagesz=%s\n", s); 4476 return 1; 4477 } 4478 4479 hugetlb_add_hstate(ilog2(size) - PAGE_SHIFT); 4480 parsed_valid_hugepagesz = true; 4481 parsed_default_hugepagesz = true; 4482 default_hstate_idx = hstate_index(size_to_hstate(size)); 4483 4484 /* 4485 * The number of default huge pages (for this size) could have been 4486 * specified as the first hugetlb parameter: hugepages=X. If so, 4487 * then default_hstate_max_huge_pages is set. If the default huge 4488 * page size is gigantic (> MAX_ORDER), then the pages must be 4489 * allocated here from bootmem allocator. 4490 */ 4491 if (default_hstate_max_huge_pages) { 4492 default_hstate.max_huge_pages = default_hstate_max_huge_pages; 4493 for_each_online_node(i) 4494 default_hstate.max_huge_pages_node[i] = 4495 default_hugepages_in_node[i]; 4496 if (hstate_is_gigantic(&default_hstate)) 4497 hugetlb_hstate_alloc_pages(&default_hstate); 4498 default_hstate_max_huge_pages = 0; 4499 } 4500 4501 return 1; 4502 } 4503 __setup("default_hugepagesz=", default_hugepagesz_setup); 4504 4505 static nodemask_t *policy_mbind_nodemask(gfp_t gfp) 4506 { 4507 #ifdef CONFIG_NUMA 4508 struct mempolicy *mpol = get_task_policy(current); 4509 4510 /* 4511 * Only enforce MPOL_BIND policy which overlaps with cpuset policy 4512 * (from policy_nodemask) specifically for hugetlb case 4513 */ 4514 if (mpol->mode == MPOL_BIND && 4515 (apply_policy_zone(mpol, gfp_zone(gfp)) && 4516 cpuset_nodemask_valid_mems_allowed(&mpol->nodes))) 4517 return &mpol->nodes; 4518 #endif 4519 return NULL; 4520 } 4521 4522 static unsigned int allowed_mems_nr(struct hstate *h) 4523 { 4524 int node; 4525 unsigned int nr = 0; 4526 nodemask_t *mbind_nodemask; 4527 unsigned int *array = h->free_huge_pages_node; 4528 gfp_t gfp_mask = htlb_alloc_mask(h); 4529 4530 mbind_nodemask = policy_mbind_nodemask(gfp_mask); 4531 for_each_node_mask(node, cpuset_current_mems_allowed) { 4532 if (!mbind_nodemask || node_isset(node, *mbind_nodemask)) 4533 nr += array[node]; 4534 } 4535 4536 return nr; 4537 } 4538 4539 #ifdef CONFIG_SYSCTL 4540 static int proc_hugetlb_doulongvec_minmax(struct ctl_table *table, int write, 4541 void *buffer, size_t *length, 4542 loff_t *ppos, unsigned long *out) 4543 { 4544 struct ctl_table dup_table; 4545 4546 /* 4547 * In order to avoid races with __do_proc_doulongvec_minmax(), we 4548 * can duplicate the @table and alter the duplicate of it. 4549 */ 4550 dup_table = *table; 4551 dup_table.data = out; 4552 4553 return proc_doulongvec_minmax(&dup_table, write, buffer, length, ppos); 4554 } 4555 4556 static int hugetlb_sysctl_handler_common(bool obey_mempolicy, 4557 struct ctl_table *table, int write, 4558 void *buffer, size_t *length, loff_t *ppos) 4559 { 4560 struct hstate *h = &default_hstate; 4561 unsigned long tmp = h->max_huge_pages; 4562 int ret; 4563 4564 if (!hugepages_supported()) 4565 return -EOPNOTSUPP; 4566 4567 ret = proc_hugetlb_doulongvec_minmax(table, write, buffer, length, ppos, 4568 &tmp); 4569 if (ret) 4570 goto out; 4571 4572 if (write) 4573 ret = __nr_hugepages_store_common(obey_mempolicy, h, 4574 NUMA_NO_NODE, tmp, *length); 4575 out: 4576 return ret; 4577 } 4578 4579 static int hugetlb_sysctl_handler(struct ctl_table *table, int write, 4580 void *buffer, size_t *length, loff_t *ppos) 4581 { 4582 4583 return hugetlb_sysctl_handler_common(false, table, write, 4584 buffer, length, ppos); 4585 } 4586 4587 #ifdef CONFIG_NUMA 4588 static int hugetlb_mempolicy_sysctl_handler(struct ctl_table *table, int write, 4589 void *buffer, size_t *length, loff_t *ppos) 4590 { 4591 return hugetlb_sysctl_handler_common(true, table, write, 4592 buffer, length, ppos); 4593 } 4594 #endif /* CONFIG_NUMA */ 4595 4596 static int hugetlb_overcommit_handler(struct ctl_table *table, int write, 4597 void *buffer, size_t *length, loff_t *ppos) 4598 { 4599 struct hstate *h = &default_hstate; 4600 unsigned long tmp; 4601 int ret; 4602 4603 if (!hugepages_supported()) 4604 return -EOPNOTSUPP; 4605 4606 tmp = h->nr_overcommit_huge_pages; 4607 4608 if (write && hstate_is_gigantic(h)) 4609 return -EINVAL; 4610 4611 ret = proc_hugetlb_doulongvec_minmax(table, write, buffer, length, ppos, 4612 &tmp); 4613 if (ret) 4614 goto out; 4615 4616 if (write) { 4617 spin_lock_irq(&hugetlb_lock); 4618 h->nr_overcommit_huge_pages = tmp; 4619 spin_unlock_irq(&hugetlb_lock); 4620 } 4621 out: 4622 return ret; 4623 } 4624 4625 static struct ctl_table hugetlb_table[] = { 4626 { 4627 .procname = "nr_hugepages", 4628 .data = NULL, 4629 .maxlen = sizeof(unsigned long), 4630 .mode = 0644, 4631 .proc_handler = hugetlb_sysctl_handler, 4632 }, 4633 #ifdef CONFIG_NUMA 4634 { 4635 .procname = "nr_hugepages_mempolicy", 4636 .data = NULL, 4637 .maxlen = sizeof(unsigned long), 4638 .mode = 0644, 4639 .proc_handler = &hugetlb_mempolicy_sysctl_handler, 4640 }, 4641 #endif 4642 { 4643 .procname = "hugetlb_shm_group", 4644 .data = &sysctl_hugetlb_shm_group, 4645 .maxlen = sizeof(gid_t), 4646 .mode = 0644, 4647 .proc_handler = proc_dointvec, 4648 }, 4649 { 4650 .procname = "nr_overcommit_hugepages", 4651 .data = NULL, 4652 .maxlen = sizeof(unsigned long), 4653 .mode = 0644, 4654 .proc_handler = hugetlb_overcommit_handler, 4655 }, 4656 { } 4657 }; 4658 4659 static void hugetlb_sysctl_init(void) 4660 { 4661 register_sysctl_init("vm", hugetlb_table); 4662 } 4663 #endif /* CONFIG_SYSCTL */ 4664 4665 void hugetlb_report_meminfo(struct seq_file *m) 4666 { 4667 struct hstate *h; 4668 unsigned long total = 0; 4669 4670 if (!hugepages_supported()) 4671 return; 4672 4673 for_each_hstate(h) { 4674 unsigned long count = h->nr_huge_pages; 4675 4676 total += huge_page_size(h) * count; 4677 4678 if (h == &default_hstate) 4679 seq_printf(m, 4680 "HugePages_Total: %5lu\n" 4681 "HugePages_Free: %5lu\n" 4682 "HugePages_Rsvd: %5lu\n" 4683 "HugePages_Surp: %5lu\n" 4684 "Hugepagesize: %8lu kB\n", 4685 count, 4686 h->free_huge_pages, 4687 h->resv_huge_pages, 4688 h->surplus_huge_pages, 4689 huge_page_size(h) / SZ_1K); 4690 } 4691 4692 seq_printf(m, "Hugetlb: %8lu kB\n", total / SZ_1K); 4693 } 4694 4695 int hugetlb_report_node_meminfo(char *buf, int len, int nid) 4696 { 4697 struct hstate *h = &default_hstate; 4698 4699 if (!hugepages_supported()) 4700 return 0; 4701 4702 return sysfs_emit_at(buf, len, 4703 "Node %d HugePages_Total: %5u\n" 4704 "Node %d HugePages_Free: %5u\n" 4705 "Node %d HugePages_Surp: %5u\n", 4706 nid, h->nr_huge_pages_node[nid], 4707 nid, h->free_huge_pages_node[nid], 4708 nid, h->surplus_huge_pages_node[nid]); 4709 } 4710 4711 void hugetlb_show_meminfo_node(int nid) 4712 { 4713 struct hstate *h; 4714 4715 if (!hugepages_supported()) 4716 return; 4717 4718 for_each_hstate(h) 4719 printk("Node %d hugepages_total=%u hugepages_free=%u hugepages_surp=%u hugepages_size=%lukB\n", 4720 nid, 4721 h->nr_huge_pages_node[nid], 4722 h->free_huge_pages_node[nid], 4723 h->surplus_huge_pages_node[nid], 4724 huge_page_size(h) / SZ_1K); 4725 } 4726 4727 void hugetlb_report_usage(struct seq_file *m, struct mm_struct *mm) 4728 { 4729 seq_printf(m, "HugetlbPages:\t%8lu kB\n", 4730 K(atomic_long_read(&mm->hugetlb_usage))); 4731 } 4732 4733 /* Return the number pages of memory we physically have, in PAGE_SIZE units. */ 4734 unsigned long hugetlb_total_pages(void) 4735 { 4736 struct hstate *h; 4737 unsigned long nr_total_pages = 0; 4738 4739 for_each_hstate(h) 4740 nr_total_pages += h->nr_huge_pages * pages_per_huge_page(h); 4741 return nr_total_pages; 4742 } 4743 4744 static int hugetlb_acct_memory(struct hstate *h, long delta) 4745 { 4746 int ret = -ENOMEM; 4747 4748 if (!delta) 4749 return 0; 4750 4751 spin_lock_irq(&hugetlb_lock); 4752 /* 4753 * When cpuset is configured, it breaks the strict hugetlb page 4754 * reservation as the accounting is done on a global variable. Such 4755 * reservation is completely rubbish in the presence of cpuset because 4756 * the reservation is not checked against page availability for the 4757 * current cpuset. Application can still potentially OOM'ed by kernel 4758 * with lack of free htlb page in cpuset that the task is in. 4759 * Attempt to enforce strict accounting with cpuset is almost 4760 * impossible (or too ugly) because cpuset is too fluid that 4761 * task or memory node can be dynamically moved between cpusets. 4762 * 4763 * The change of semantics for shared hugetlb mapping with cpuset is 4764 * undesirable. However, in order to preserve some of the semantics, 4765 * we fall back to check against current free page availability as 4766 * a best attempt and hopefully to minimize the impact of changing 4767 * semantics that cpuset has. 4768 * 4769 * Apart from cpuset, we also have memory policy mechanism that 4770 * also determines from which node the kernel will allocate memory 4771 * in a NUMA system. So similar to cpuset, we also should consider 4772 * the memory policy of the current task. Similar to the description 4773 * above. 4774 */ 4775 if (delta > 0) { 4776 if (gather_surplus_pages(h, delta) < 0) 4777 goto out; 4778 4779 if (delta > allowed_mems_nr(h)) { 4780 return_unused_surplus_pages(h, delta); 4781 goto out; 4782 } 4783 } 4784 4785 ret = 0; 4786 if (delta < 0) 4787 return_unused_surplus_pages(h, (unsigned long) -delta); 4788 4789 out: 4790 spin_unlock_irq(&hugetlb_lock); 4791 return ret; 4792 } 4793 4794 static void hugetlb_vm_op_open(struct vm_area_struct *vma) 4795 { 4796 struct resv_map *resv = vma_resv_map(vma); 4797 4798 /* 4799 * HPAGE_RESV_OWNER indicates a private mapping. 4800 * This new VMA should share its siblings reservation map if present. 4801 * The VMA will only ever have a valid reservation map pointer where 4802 * it is being copied for another still existing VMA. As that VMA 4803 * has a reference to the reservation map it cannot disappear until 4804 * after this open call completes. It is therefore safe to take a 4805 * new reference here without additional locking. 4806 */ 4807 if (resv && is_vma_resv_set(vma, HPAGE_RESV_OWNER)) { 4808 resv_map_dup_hugetlb_cgroup_uncharge_info(resv); 4809 kref_get(&resv->refs); 4810 } 4811 4812 /* 4813 * vma_lock structure for sharable mappings is vma specific. 4814 * Clear old pointer (if copied via vm_area_dup) and allocate 4815 * new structure. Before clearing, make sure vma_lock is not 4816 * for this vma. 4817 */ 4818 if (vma->vm_flags & VM_MAYSHARE) { 4819 struct hugetlb_vma_lock *vma_lock = vma->vm_private_data; 4820 4821 if (vma_lock) { 4822 if (vma_lock->vma != vma) { 4823 vma->vm_private_data = NULL; 4824 hugetlb_vma_lock_alloc(vma); 4825 } else 4826 pr_warn("HugeTLB: vma_lock already exists in %s.\n", __func__); 4827 } else 4828 hugetlb_vma_lock_alloc(vma); 4829 } 4830 } 4831 4832 static void hugetlb_vm_op_close(struct vm_area_struct *vma) 4833 { 4834 struct hstate *h = hstate_vma(vma); 4835 struct resv_map *resv; 4836 struct hugepage_subpool *spool = subpool_vma(vma); 4837 unsigned long reserve, start, end; 4838 long gbl_reserve; 4839 4840 hugetlb_vma_lock_free(vma); 4841 4842 resv = vma_resv_map(vma); 4843 if (!resv || !is_vma_resv_set(vma, HPAGE_RESV_OWNER)) 4844 return; 4845 4846 start = vma_hugecache_offset(h, vma, vma->vm_start); 4847 end = vma_hugecache_offset(h, vma, vma->vm_end); 4848 4849 reserve = (end - start) - region_count(resv, start, end); 4850 hugetlb_cgroup_uncharge_counter(resv, start, end); 4851 if (reserve) { 4852 /* 4853 * Decrement reserve counts. The global reserve count may be 4854 * adjusted if the subpool has a minimum size. 4855 */ 4856 gbl_reserve = hugepage_subpool_put_pages(spool, reserve); 4857 hugetlb_acct_memory(h, -gbl_reserve); 4858 } 4859 4860 kref_put(&resv->refs, resv_map_release); 4861 } 4862 4863 static int hugetlb_vm_op_split(struct vm_area_struct *vma, unsigned long addr) 4864 { 4865 if (addr & ~(huge_page_mask(hstate_vma(vma)))) 4866 return -EINVAL; 4867 4868 /* 4869 * PMD sharing is only possible for PUD_SIZE-aligned address ranges 4870 * in HugeTLB VMAs. If we will lose PUD_SIZE alignment due to this 4871 * split, unshare PMDs in the PUD_SIZE interval surrounding addr now. 4872 */ 4873 if (addr & ~PUD_MASK) { 4874 /* 4875 * hugetlb_vm_op_split is called right before we attempt to 4876 * split the VMA. We will need to unshare PMDs in the old and 4877 * new VMAs, so let's unshare before we split. 4878 */ 4879 unsigned long floor = addr & PUD_MASK; 4880 unsigned long ceil = floor + PUD_SIZE; 4881 4882 if (floor >= vma->vm_start && ceil <= vma->vm_end) 4883 hugetlb_unshare_pmds(vma, floor, ceil); 4884 } 4885 4886 return 0; 4887 } 4888 4889 static unsigned long hugetlb_vm_op_pagesize(struct vm_area_struct *vma) 4890 { 4891 return huge_page_size(hstate_vma(vma)); 4892 } 4893 4894 /* 4895 * We cannot handle pagefaults against hugetlb pages at all. They cause 4896 * handle_mm_fault() to try to instantiate regular-sized pages in the 4897 * hugepage VMA. do_page_fault() is supposed to trap this, so BUG is we get 4898 * this far. 4899 */ 4900 static vm_fault_t hugetlb_vm_op_fault(struct vm_fault *vmf) 4901 { 4902 BUG(); 4903 return 0; 4904 } 4905 4906 /* 4907 * When a new function is introduced to vm_operations_struct and added 4908 * to hugetlb_vm_ops, please consider adding the function to shm_vm_ops. 4909 * This is because under System V memory model, mappings created via 4910 * shmget/shmat with "huge page" specified are backed by hugetlbfs files, 4911 * their original vm_ops are overwritten with shm_vm_ops. 4912 */ 4913 const struct vm_operations_struct hugetlb_vm_ops = { 4914 .fault = hugetlb_vm_op_fault, 4915 .open = hugetlb_vm_op_open, 4916 .close = hugetlb_vm_op_close, 4917 .may_split = hugetlb_vm_op_split, 4918 .pagesize = hugetlb_vm_op_pagesize, 4919 }; 4920 4921 static pte_t make_huge_pte(struct vm_area_struct *vma, struct page *page, 4922 int writable) 4923 { 4924 pte_t entry; 4925 unsigned int shift = huge_page_shift(hstate_vma(vma)); 4926 4927 if (writable) { 4928 entry = huge_pte_mkwrite(huge_pte_mkdirty(mk_huge_pte(page, 4929 vma->vm_page_prot))); 4930 } else { 4931 entry = huge_pte_wrprotect(mk_huge_pte(page, 4932 vma->vm_page_prot)); 4933 } 4934 entry = pte_mkyoung(entry); 4935 entry = arch_make_huge_pte(entry, shift, vma->vm_flags); 4936 4937 return entry; 4938 } 4939 4940 static void set_huge_ptep_writable(struct vm_area_struct *vma, 4941 unsigned long address, pte_t *ptep) 4942 { 4943 pte_t entry; 4944 4945 entry = huge_pte_mkwrite(huge_pte_mkdirty(huge_ptep_get(ptep))); 4946 if (huge_ptep_set_access_flags(vma, address, ptep, entry, 1)) 4947 update_mmu_cache(vma, address, ptep); 4948 } 4949 4950 bool is_hugetlb_entry_migration(pte_t pte) 4951 { 4952 swp_entry_t swp; 4953 4954 if (huge_pte_none(pte) || pte_present(pte)) 4955 return false; 4956 swp = pte_to_swp_entry(pte); 4957 if (is_migration_entry(swp)) 4958 return true; 4959 else 4960 return false; 4961 } 4962 4963 static bool is_hugetlb_entry_hwpoisoned(pte_t pte) 4964 { 4965 swp_entry_t swp; 4966 4967 if (huge_pte_none(pte) || pte_present(pte)) 4968 return false; 4969 swp = pte_to_swp_entry(pte); 4970 if (is_hwpoison_entry(swp)) 4971 return true; 4972 else 4973 return false; 4974 } 4975 4976 static void 4977 hugetlb_install_folio(struct vm_area_struct *vma, pte_t *ptep, unsigned long addr, 4978 struct folio *new_folio, pte_t old) 4979 { 4980 pte_t newpte = make_huge_pte(vma, &new_folio->page, 1); 4981 4982 __folio_mark_uptodate(new_folio); 4983 hugepage_add_new_anon_rmap(new_folio, vma, addr); 4984 if (userfaultfd_wp(vma) && huge_pte_uffd_wp(old)) 4985 newpte = huge_pte_mkuffd_wp(newpte); 4986 set_huge_pte_at(vma->vm_mm, addr, ptep, newpte); 4987 hugetlb_count_add(pages_per_huge_page(hstate_vma(vma)), vma->vm_mm); 4988 folio_set_hugetlb_migratable(new_folio); 4989 } 4990 4991 int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src, 4992 struct vm_area_struct *dst_vma, 4993 struct vm_area_struct *src_vma) 4994 { 4995 pte_t *src_pte, *dst_pte, entry; 4996 struct folio *pte_folio; 4997 unsigned long addr; 4998 bool cow = is_cow_mapping(src_vma->vm_flags); 4999 struct hstate *h = hstate_vma(src_vma); 5000 unsigned long sz = huge_page_size(h); 5001 unsigned long npages = pages_per_huge_page(h); 5002 struct mmu_notifier_range range; 5003 unsigned long last_addr_mask; 5004 int ret = 0; 5005 5006 if (cow) { 5007 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, src, 5008 src_vma->vm_start, 5009 src_vma->vm_end); 5010 mmu_notifier_invalidate_range_start(&range); 5011 vma_assert_write_locked(src_vma); 5012 raw_write_seqcount_begin(&src->write_protect_seq); 5013 } else { 5014 /* 5015 * For shared mappings the vma lock must be held before 5016 * calling hugetlb_walk() in the src vma. Otherwise, the 5017 * returned ptep could go away if part of a shared pmd and 5018 * another thread calls huge_pmd_unshare. 5019 */ 5020 hugetlb_vma_lock_read(src_vma); 5021 } 5022 5023 last_addr_mask = hugetlb_mask_last_page(h); 5024 for (addr = src_vma->vm_start; addr < src_vma->vm_end; addr += sz) { 5025 spinlock_t *src_ptl, *dst_ptl; 5026 src_pte = hugetlb_walk(src_vma, addr, sz); 5027 if (!src_pte) { 5028 addr |= last_addr_mask; 5029 continue; 5030 } 5031 dst_pte = huge_pte_alloc(dst, dst_vma, addr, sz); 5032 if (!dst_pte) { 5033 ret = -ENOMEM; 5034 break; 5035 } 5036 5037 /* 5038 * If the pagetables are shared don't copy or take references. 5039 * 5040 * dst_pte == src_pte is the common case of src/dest sharing. 5041 * However, src could have 'unshared' and dst shares with 5042 * another vma. So page_count of ptep page is checked instead 5043 * to reliably determine whether pte is shared. 5044 */ 5045 if (page_count(virt_to_page(dst_pte)) > 1) { 5046 addr |= last_addr_mask; 5047 continue; 5048 } 5049 5050 dst_ptl = huge_pte_lock(h, dst, dst_pte); 5051 src_ptl = huge_pte_lockptr(h, src, src_pte); 5052 spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING); 5053 entry = huge_ptep_get(src_pte); 5054 again: 5055 if (huge_pte_none(entry)) { 5056 /* 5057 * Skip if src entry none. 5058 */ 5059 ; 5060 } else if (unlikely(is_hugetlb_entry_hwpoisoned(entry))) { 5061 if (!userfaultfd_wp(dst_vma)) 5062 entry = huge_pte_clear_uffd_wp(entry); 5063 set_huge_pte_at(dst, addr, dst_pte, entry); 5064 } else if (unlikely(is_hugetlb_entry_migration(entry))) { 5065 swp_entry_t swp_entry = pte_to_swp_entry(entry); 5066 bool uffd_wp = pte_swp_uffd_wp(entry); 5067 5068 if (!is_readable_migration_entry(swp_entry) && cow) { 5069 /* 5070 * COW mappings require pages in both 5071 * parent and child to be set to read. 5072 */ 5073 swp_entry = make_readable_migration_entry( 5074 swp_offset(swp_entry)); 5075 entry = swp_entry_to_pte(swp_entry); 5076 if (userfaultfd_wp(src_vma) && uffd_wp) 5077 entry = pte_swp_mkuffd_wp(entry); 5078 set_huge_pte_at(src, addr, src_pte, entry); 5079 } 5080 if (!userfaultfd_wp(dst_vma)) 5081 entry = huge_pte_clear_uffd_wp(entry); 5082 set_huge_pte_at(dst, addr, dst_pte, entry); 5083 } else if (unlikely(is_pte_marker(entry))) { 5084 pte_marker marker = copy_pte_marker( 5085 pte_to_swp_entry(entry), dst_vma); 5086 5087 if (marker) 5088 set_huge_pte_at(dst, addr, dst_pte, 5089 make_pte_marker(marker)); 5090 } else { 5091 entry = huge_ptep_get(src_pte); 5092 pte_folio = page_folio(pte_page(entry)); 5093 folio_get(pte_folio); 5094 5095 /* 5096 * Failing to duplicate the anon rmap is a rare case 5097 * where we see pinned hugetlb pages while they're 5098 * prone to COW. We need to do the COW earlier during 5099 * fork. 5100 * 5101 * When pre-allocating the page or copying data, we 5102 * need to be without the pgtable locks since we could 5103 * sleep during the process. 5104 */ 5105 if (!folio_test_anon(pte_folio)) { 5106 page_dup_file_rmap(&pte_folio->page, true); 5107 } else if (page_try_dup_anon_rmap(&pte_folio->page, 5108 true, src_vma)) { 5109 pte_t src_pte_old = entry; 5110 struct folio *new_folio; 5111 5112 spin_unlock(src_ptl); 5113 spin_unlock(dst_ptl); 5114 /* Do not use reserve as it's private owned */ 5115 new_folio = alloc_hugetlb_folio(dst_vma, addr, 1); 5116 if (IS_ERR(new_folio)) { 5117 folio_put(pte_folio); 5118 ret = PTR_ERR(new_folio); 5119 break; 5120 } 5121 ret = copy_user_large_folio(new_folio, 5122 pte_folio, 5123 addr, dst_vma); 5124 folio_put(pte_folio); 5125 if (ret) { 5126 folio_put(new_folio); 5127 break; 5128 } 5129 5130 /* Install the new hugetlb folio if src pte stable */ 5131 dst_ptl = huge_pte_lock(h, dst, dst_pte); 5132 src_ptl = huge_pte_lockptr(h, src, src_pte); 5133 spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING); 5134 entry = huge_ptep_get(src_pte); 5135 if (!pte_same(src_pte_old, entry)) { 5136 restore_reserve_on_error(h, dst_vma, addr, 5137 new_folio); 5138 folio_put(new_folio); 5139 /* huge_ptep of dst_pte won't change as in child */ 5140 goto again; 5141 } 5142 hugetlb_install_folio(dst_vma, dst_pte, addr, 5143 new_folio, src_pte_old); 5144 spin_unlock(src_ptl); 5145 spin_unlock(dst_ptl); 5146 continue; 5147 } 5148 5149 if (cow) { 5150 /* 5151 * No need to notify as we are downgrading page 5152 * table protection not changing it to point 5153 * to a new page. 5154 * 5155 * See Documentation/mm/mmu_notifier.rst 5156 */ 5157 huge_ptep_set_wrprotect(src, addr, src_pte); 5158 entry = huge_pte_wrprotect(entry); 5159 } 5160 5161 if (!userfaultfd_wp(dst_vma)) 5162 entry = huge_pte_clear_uffd_wp(entry); 5163 5164 set_huge_pte_at(dst, addr, dst_pte, entry); 5165 hugetlb_count_add(npages, dst); 5166 } 5167 spin_unlock(src_ptl); 5168 spin_unlock(dst_ptl); 5169 } 5170 5171 if (cow) { 5172 raw_write_seqcount_end(&src->write_protect_seq); 5173 mmu_notifier_invalidate_range_end(&range); 5174 } else { 5175 hugetlb_vma_unlock_read(src_vma); 5176 } 5177 5178 return ret; 5179 } 5180 5181 static void move_huge_pte(struct vm_area_struct *vma, unsigned long old_addr, 5182 unsigned long new_addr, pte_t *src_pte, pte_t *dst_pte) 5183 { 5184 struct hstate *h = hstate_vma(vma); 5185 struct mm_struct *mm = vma->vm_mm; 5186 spinlock_t *src_ptl, *dst_ptl; 5187 pte_t pte; 5188 5189 dst_ptl = huge_pte_lock(h, mm, dst_pte); 5190 src_ptl = huge_pte_lockptr(h, mm, src_pte); 5191 5192 /* 5193 * We don't have to worry about the ordering of src and dst ptlocks 5194 * because exclusive mmap_lock (or the i_mmap_lock) prevents deadlock. 5195 */ 5196 if (src_ptl != dst_ptl) 5197 spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING); 5198 5199 pte = huge_ptep_get_and_clear(mm, old_addr, src_pte); 5200 set_huge_pte_at(mm, new_addr, dst_pte, pte); 5201 5202 if (src_ptl != dst_ptl) 5203 spin_unlock(src_ptl); 5204 spin_unlock(dst_ptl); 5205 } 5206 5207 int move_hugetlb_page_tables(struct vm_area_struct *vma, 5208 struct vm_area_struct *new_vma, 5209 unsigned long old_addr, unsigned long new_addr, 5210 unsigned long len) 5211 { 5212 struct hstate *h = hstate_vma(vma); 5213 struct address_space *mapping = vma->vm_file->f_mapping; 5214 unsigned long sz = huge_page_size(h); 5215 struct mm_struct *mm = vma->vm_mm; 5216 unsigned long old_end = old_addr + len; 5217 unsigned long last_addr_mask; 5218 pte_t *src_pte, *dst_pte; 5219 struct mmu_notifier_range range; 5220 bool shared_pmd = false; 5221 5222 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, mm, old_addr, 5223 old_end); 5224 adjust_range_if_pmd_sharing_possible(vma, &range.start, &range.end); 5225 /* 5226 * In case of shared PMDs, we should cover the maximum possible 5227 * range. 5228 */ 5229 flush_cache_range(vma, range.start, range.end); 5230 5231 mmu_notifier_invalidate_range_start(&range); 5232 last_addr_mask = hugetlb_mask_last_page(h); 5233 /* Prevent race with file truncation */ 5234 hugetlb_vma_lock_write(vma); 5235 i_mmap_lock_write(mapping); 5236 for (; old_addr < old_end; old_addr += sz, new_addr += sz) { 5237 src_pte = hugetlb_walk(vma, old_addr, sz); 5238 if (!src_pte) { 5239 old_addr |= last_addr_mask; 5240 new_addr |= last_addr_mask; 5241 continue; 5242 } 5243 if (huge_pte_none(huge_ptep_get(src_pte))) 5244 continue; 5245 5246 if (huge_pmd_unshare(mm, vma, old_addr, src_pte)) { 5247 shared_pmd = true; 5248 old_addr |= last_addr_mask; 5249 new_addr |= last_addr_mask; 5250 continue; 5251 } 5252 5253 dst_pte = huge_pte_alloc(mm, new_vma, new_addr, sz); 5254 if (!dst_pte) 5255 break; 5256 5257 move_huge_pte(vma, old_addr, new_addr, src_pte, dst_pte); 5258 } 5259 5260 if (shared_pmd) 5261 flush_hugetlb_tlb_range(vma, range.start, range.end); 5262 else 5263 flush_hugetlb_tlb_range(vma, old_end - len, old_end); 5264 mmu_notifier_invalidate_range_end(&range); 5265 i_mmap_unlock_write(mapping); 5266 hugetlb_vma_unlock_write(vma); 5267 5268 return len + old_addr - old_end; 5269 } 5270 5271 static void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct *vma, 5272 unsigned long start, unsigned long end, 5273 struct page *ref_page, zap_flags_t zap_flags) 5274 { 5275 struct mm_struct *mm = vma->vm_mm; 5276 unsigned long address; 5277 pte_t *ptep; 5278 pte_t pte; 5279 spinlock_t *ptl; 5280 struct page *page; 5281 struct hstate *h = hstate_vma(vma); 5282 unsigned long sz = huge_page_size(h); 5283 unsigned long last_addr_mask; 5284 bool force_flush = false; 5285 5286 WARN_ON(!is_vm_hugetlb_page(vma)); 5287 BUG_ON(start & ~huge_page_mask(h)); 5288 BUG_ON(end & ~huge_page_mask(h)); 5289 5290 /* 5291 * This is a hugetlb vma, all the pte entries should point 5292 * to huge page. 5293 */ 5294 tlb_change_page_size(tlb, sz); 5295 tlb_start_vma(tlb, vma); 5296 5297 last_addr_mask = hugetlb_mask_last_page(h); 5298 address = start; 5299 for (; address < end; address += sz) { 5300 ptep = hugetlb_walk(vma, address, sz); 5301 if (!ptep) { 5302 address |= last_addr_mask; 5303 continue; 5304 } 5305 5306 ptl = huge_pte_lock(h, mm, ptep); 5307 if (huge_pmd_unshare(mm, vma, address, ptep)) { 5308 spin_unlock(ptl); 5309 tlb_flush_pmd_range(tlb, address & PUD_MASK, PUD_SIZE); 5310 force_flush = true; 5311 address |= last_addr_mask; 5312 continue; 5313 } 5314 5315 pte = huge_ptep_get(ptep); 5316 if (huge_pte_none(pte)) { 5317 spin_unlock(ptl); 5318 continue; 5319 } 5320 5321 /* 5322 * Migrating hugepage or HWPoisoned hugepage is already 5323 * unmapped and its refcount is dropped, so just clear pte here. 5324 */ 5325 if (unlikely(!pte_present(pte))) { 5326 /* 5327 * If the pte was wr-protected by uffd-wp in any of the 5328 * swap forms, meanwhile the caller does not want to 5329 * drop the uffd-wp bit in this zap, then replace the 5330 * pte with a marker. 5331 */ 5332 if (pte_swp_uffd_wp_any(pte) && 5333 !(zap_flags & ZAP_FLAG_DROP_MARKER)) 5334 set_huge_pte_at(mm, address, ptep, 5335 make_pte_marker(PTE_MARKER_UFFD_WP)); 5336 else 5337 huge_pte_clear(mm, address, ptep, sz); 5338 spin_unlock(ptl); 5339 continue; 5340 } 5341 5342 page = pte_page(pte); 5343 /* 5344 * If a reference page is supplied, it is because a specific 5345 * page is being unmapped, not a range. Ensure the page we 5346 * are about to unmap is the actual page of interest. 5347 */ 5348 if (ref_page) { 5349 if (page != ref_page) { 5350 spin_unlock(ptl); 5351 continue; 5352 } 5353 /* 5354 * Mark the VMA as having unmapped its page so that 5355 * future faults in this VMA will fail rather than 5356 * looking like data was lost 5357 */ 5358 set_vma_resv_flags(vma, HPAGE_RESV_UNMAPPED); 5359 } 5360 5361 pte = huge_ptep_get_and_clear(mm, address, ptep); 5362 tlb_remove_huge_tlb_entry(h, tlb, ptep, address); 5363 if (huge_pte_dirty(pte)) 5364 set_page_dirty(page); 5365 /* Leave a uffd-wp pte marker if needed */ 5366 if (huge_pte_uffd_wp(pte) && 5367 !(zap_flags & ZAP_FLAG_DROP_MARKER)) 5368 set_huge_pte_at(mm, address, ptep, 5369 make_pte_marker(PTE_MARKER_UFFD_WP)); 5370 hugetlb_count_sub(pages_per_huge_page(h), mm); 5371 page_remove_rmap(page, vma, true); 5372 5373 spin_unlock(ptl); 5374 tlb_remove_page_size(tlb, page, huge_page_size(h)); 5375 /* 5376 * Bail out after unmapping reference page if supplied 5377 */ 5378 if (ref_page) 5379 break; 5380 } 5381 tlb_end_vma(tlb, vma); 5382 5383 /* 5384 * If we unshared PMDs, the TLB flush was not recorded in mmu_gather. We 5385 * could defer the flush until now, since by holding i_mmap_rwsem we 5386 * guaranteed that the last refernece would not be dropped. But we must 5387 * do the flushing before we return, as otherwise i_mmap_rwsem will be 5388 * dropped and the last reference to the shared PMDs page might be 5389 * dropped as well. 5390 * 5391 * In theory we could defer the freeing of the PMD pages as well, but 5392 * huge_pmd_unshare() relies on the exact page_count for the PMD page to 5393 * detect sharing, so we cannot defer the release of the page either. 5394 * Instead, do flush now. 5395 */ 5396 if (force_flush) 5397 tlb_flush_mmu_tlbonly(tlb); 5398 } 5399 5400 void __unmap_hugepage_range_final(struct mmu_gather *tlb, 5401 struct vm_area_struct *vma, unsigned long start, 5402 unsigned long end, struct page *ref_page, 5403 zap_flags_t zap_flags) 5404 { 5405 hugetlb_vma_lock_write(vma); 5406 i_mmap_lock_write(vma->vm_file->f_mapping); 5407 5408 /* mmu notification performed in caller */ 5409 __unmap_hugepage_range(tlb, vma, start, end, ref_page, zap_flags); 5410 5411 if (zap_flags & ZAP_FLAG_UNMAP) { /* final unmap */ 5412 /* 5413 * Unlock and free the vma lock before releasing i_mmap_rwsem. 5414 * When the vma_lock is freed, this makes the vma ineligible 5415 * for pmd sharing. And, i_mmap_rwsem is required to set up 5416 * pmd sharing. This is important as page tables for this 5417 * unmapped range will be asynchrously deleted. If the page 5418 * tables are shared, there will be issues when accessed by 5419 * someone else. 5420 */ 5421 __hugetlb_vma_unlock_write_free(vma); 5422 i_mmap_unlock_write(vma->vm_file->f_mapping); 5423 } else { 5424 i_mmap_unlock_write(vma->vm_file->f_mapping); 5425 hugetlb_vma_unlock_write(vma); 5426 } 5427 } 5428 5429 void unmap_hugepage_range(struct vm_area_struct *vma, unsigned long start, 5430 unsigned long end, struct page *ref_page, 5431 zap_flags_t zap_flags) 5432 { 5433 struct mmu_notifier_range range; 5434 struct mmu_gather tlb; 5435 5436 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma->vm_mm, 5437 start, end); 5438 adjust_range_if_pmd_sharing_possible(vma, &range.start, &range.end); 5439 mmu_notifier_invalidate_range_start(&range); 5440 tlb_gather_mmu(&tlb, vma->vm_mm); 5441 5442 __unmap_hugepage_range(&tlb, vma, start, end, ref_page, zap_flags); 5443 5444 mmu_notifier_invalidate_range_end(&range); 5445 tlb_finish_mmu(&tlb); 5446 } 5447 5448 /* 5449 * This is called when the original mapper is failing to COW a MAP_PRIVATE 5450 * mapping it owns the reserve page for. The intention is to unmap the page 5451 * from other VMAs and let the children be SIGKILLed if they are faulting the 5452 * same region. 5453 */ 5454 static void unmap_ref_private(struct mm_struct *mm, struct vm_area_struct *vma, 5455 struct page *page, unsigned long address) 5456 { 5457 struct hstate *h = hstate_vma(vma); 5458 struct vm_area_struct *iter_vma; 5459 struct address_space *mapping; 5460 pgoff_t pgoff; 5461 5462 /* 5463 * vm_pgoff is in PAGE_SIZE units, hence the different calculation 5464 * from page cache lookup which is in HPAGE_SIZE units. 5465 */ 5466 address = address & huge_page_mask(h); 5467 pgoff = ((address - vma->vm_start) >> PAGE_SHIFT) + 5468 vma->vm_pgoff; 5469 mapping = vma->vm_file->f_mapping; 5470 5471 /* 5472 * Take the mapping lock for the duration of the table walk. As 5473 * this mapping should be shared between all the VMAs, 5474 * __unmap_hugepage_range() is called as the lock is already held 5475 */ 5476 i_mmap_lock_write(mapping); 5477 vma_interval_tree_foreach(iter_vma, &mapping->i_mmap, pgoff, pgoff) { 5478 /* Do not unmap the current VMA */ 5479 if (iter_vma == vma) 5480 continue; 5481 5482 /* 5483 * Shared VMAs have their own reserves and do not affect 5484 * MAP_PRIVATE accounting but it is possible that a shared 5485 * VMA is using the same page so check and skip such VMAs. 5486 */ 5487 if (iter_vma->vm_flags & VM_MAYSHARE) 5488 continue; 5489 5490 /* 5491 * Unmap the page from other VMAs without their own reserves. 5492 * They get marked to be SIGKILLed if they fault in these 5493 * areas. This is because a future no-page fault on this VMA 5494 * could insert a zeroed page instead of the data existing 5495 * from the time of fork. This would look like data corruption 5496 */ 5497 if (!is_vma_resv_set(iter_vma, HPAGE_RESV_OWNER)) 5498 unmap_hugepage_range(iter_vma, address, 5499 address + huge_page_size(h), page, 0); 5500 } 5501 i_mmap_unlock_write(mapping); 5502 } 5503 5504 /* 5505 * hugetlb_wp() should be called with page lock of the original hugepage held. 5506 * Called with hugetlb_fault_mutex_table held and pte_page locked so we 5507 * cannot race with other handlers or page migration. 5508 * Keep the pte_same checks anyway to make transition from the mutex easier. 5509 */ 5510 static vm_fault_t hugetlb_wp(struct mm_struct *mm, struct vm_area_struct *vma, 5511 unsigned long address, pte_t *ptep, unsigned int flags, 5512 struct folio *pagecache_folio, spinlock_t *ptl) 5513 { 5514 const bool unshare = flags & FAULT_FLAG_UNSHARE; 5515 pte_t pte = huge_ptep_get(ptep); 5516 struct hstate *h = hstate_vma(vma); 5517 struct folio *old_folio; 5518 struct folio *new_folio; 5519 int outside_reserve = 0; 5520 vm_fault_t ret = 0; 5521 unsigned long haddr = address & huge_page_mask(h); 5522 struct mmu_notifier_range range; 5523 5524 /* 5525 * Never handle CoW for uffd-wp protected pages. It should be only 5526 * handled when the uffd-wp protection is removed. 5527 * 5528 * Note that only the CoW optimization path (in hugetlb_no_page()) 5529 * can trigger this, because hugetlb_fault() will always resolve 5530 * uffd-wp bit first. 5531 */ 5532 if (!unshare && huge_pte_uffd_wp(pte)) 5533 return 0; 5534 5535 /* 5536 * hugetlb does not support FOLL_FORCE-style write faults that keep the 5537 * PTE mapped R/O such as maybe_mkwrite() would do. 5538 */ 5539 if (WARN_ON_ONCE(!unshare && !(vma->vm_flags & VM_WRITE))) 5540 return VM_FAULT_SIGSEGV; 5541 5542 /* Let's take out MAP_SHARED mappings first. */ 5543 if (vma->vm_flags & VM_MAYSHARE) { 5544 set_huge_ptep_writable(vma, haddr, ptep); 5545 return 0; 5546 } 5547 5548 old_folio = page_folio(pte_page(pte)); 5549 5550 delayacct_wpcopy_start(); 5551 5552 retry_avoidcopy: 5553 /* 5554 * If no-one else is actually using this page, we're the exclusive 5555 * owner and can reuse this page. 5556 */ 5557 if (folio_mapcount(old_folio) == 1 && folio_test_anon(old_folio)) { 5558 if (!PageAnonExclusive(&old_folio->page)) 5559 page_move_anon_rmap(&old_folio->page, vma); 5560 if (likely(!unshare)) 5561 set_huge_ptep_writable(vma, haddr, ptep); 5562 5563 delayacct_wpcopy_end(); 5564 return 0; 5565 } 5566 VM_BUG_ON_PAGE(folio_test_anon(old_folio) && 5567 PageAnonExclusive(&old_folio->page), &old_folio->page); 5568 5569 /* 5570 * If the process that created a MAP_PRIVATE mapping is about to 5571 * perform a COW due to a shared page count, attempt to satisfy 5572 * the allocation without using the existing reserves. The pagecache 5573 * page is used to determine if the reserve at this address was 5574 * consumed or not. If reserves were used, a partial faulted mapping 5575 * at the time of fork() could consume its reserves on COW instead 5576 * of the full address range. 5577 */ 5578 if (is_vma_resv_set(vma, HPAGE_RESV_OWNER) && 5579 old_folio != pagecache_folio) 5580 outside_reserve = 1; 5581 5582 folio_get(old_folio); 5583 5584 /* 5585 * Drop page table lock as buddy allocator may be called. It will 5586 * be acquired again before returning to the caller, as expected. 5587 */ 5588 spin_unlock(ptl); 5589 new_folio = alloc_hugetlb_folio(vma, haddr, outside_reserve); 5590 5591 if (IS_ERR(new_folio)) { 5592 /* 5593 * If a process owning a MAP_PRIVATE mapping fails to COW, 5594 * it is due to references held by a child and an insufficient 5595 * huge page pool. To guarantee the original mappers 5596 * reliability, unmap the page from child processes. The child 5597 * may get SIGKILLed if it later faults. 5598 */ 5599 if (outside_reserve) { 5600 struct address_space *mapping = vma->vm_file->f_mapping; 5601 pgoff_t idx; 5602 u32 hash; 5603 5604 folio_put(old_folio); 5605 /* 5606 * Drop hugetlb_fault_mutex and vma_lock before 5607 * unmapping. unmapping needs to hold vma_lock 5608 * in write mode. Dropping vma_lock in read mode 5609 * here is OK as COW mappings do not interact with 5610 * PMD sharing. 5611 * 5612 * Reacquire both after unmap operation. 5613 */ 5614 idx = vma_hugecache_offset(h, vma, haddr); 5615 hash = hugetlb_fault_mutex_hash(mapping, idx); 5616 hugetlb_vma_unlock_read(vma); 5617 mutex_unlock(&hugetlb_fault_mutex_table[hash]); 5618 5619 unmap_ref_private(mm, vma, &old_folio->page, haddr); 5620 5621 mutex_lock(&hugetlb_fault_mutex_table[hash]); 5622 hugetlb_vma_lock_read(vma); 5623 spin_lock(ptl); 5624 ptep = hugetlb_walk(vma, haddr, huge_page_size(h)); 5625 if (likely(ptep && 5626 pte_same(huge_ptep_get(ptep), pte))) 5627 goto retry_avoidcopy; 5628 /* 5629 * race occurs while re-acquiring page table 5630 * lock, and our job is done. 5631 */ 5632 delayacct_wpcopy_end(); 5633 return 0; 5634 } 5635 5636 ret = vmf_error(PTR_ERR(new_folio)); 5637 goto out_release_old; 5638 } 5639 5640 /* 5641 * When the original hugepage is shared one, it does not have 5642 * anon_vma prepared. 5643 */ 5644 if (unlikely(anon_vma_prepare(vma))) { 5645 ret = VM_FAULT_OOM; 5646 goto out_release_all; 5647 } 5648 5649 if (copy_user_large_folio(new_folio, old_folio, address, vma)) { 5650 ret = VM_FAULT_HWPOISON_LARGE; 5651 goto out_release_all; 5652 } 5653 __folio_mark_uptodate(new_folio); 5654 5655 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, mm, haddr, 5656 haddr + huge_page_size(h)); 5657 mmu_notifier_invalidate_range_start(&range); 5658 5659 /* 5660 * Retake the page table lock to check for racing updates 5661 * before the page tables are altered 5662 */ 5663 spin_lock(ptl); 5664 ptep = hugetlb_walk(vma, haddr, huge_page_size(h)); 5665 if (likely(ptep && pte_same(huge_ptep_get(ptep), pte))) { 5666 pte_t newpte = make_huge_pte(vma, &new_folio->page, !unshare); 5667 5668 /* Break COW or unshare */ 5669 huge_ptep_clear_flush(vma, haddr, ptep); 5670 page_remove_rmap(&old_folio->page, vma, true); 5671 hugepage_add_new_anon_rmap(new_folio, vma, haddr); 5672 if (huge_pte_uffd_wp(pte)) 5673 newpte = huge_pte_mkuffd_wp(newpte); 5674 set_huge_pte_at(mm, haddr, ptep, newpte); 5675 folio_set_hugetlb_migratable(new_folio); 5676 /* Make the old page be freed below */ 5677 new_folio = old_folio; 5678 } 5679 spin_unlock(ptl); 5680 mmu_notifier_invalidate_range_end(&range); 5681 out_release_all: 5682 /* 5683 * No restore in case of successful pagetable update (Break COW or 5684 * unshare) 5685 */ 5686 if (new_folio != old_folio) 5687 restore_reserve_on_error(h, vma, haddr, new_folio); 5688 folio_put(new_folio); 5689 out_release_old: 5690 folio_put(old_folio); 5691 5692 spin_lock(ptl); /* Caller expects lock to be held */ 5693 5694 delayacct_wpcopy_end(); 5695 return ret; 5696 } 5697 5698 /* 5699 * Return whether there is a pagecache page to back given address within VMA. 5700 */ 5701 static bool hugetlbfs_pagecache_present(struct hstate *h, 5702 struct vm_area_struct *vma, unsigned long address) 5703 { 5704 struct address_space *mapping = vma->vm_file->f_mapping; 5705 pgoff_t idx = vma_hugecache_offset(h, vma, address); 5706 struct folio *folio; 5707 5708 folio = filemap_get_folio(mapping, idx); 5709 if (IS_ERR(folio)) 5710 return false; 5711 folio_put(folio); 5712 return true; 5713 } 5714 5715 int hugetlb_add_to_page_cache(struct folio *folio, struct address_space *mapping, 5716 pgoff_t idx) 5717 { 5718 struct inode *inode = mapping->host; 5719 struct hstate *h = hstate_inode(inode); 5720 int err; 5721 5722 __folio_set_locked(folio); 5723 err = __filemap_add_folio(mapping, folio, idx, GFP_KERNEL, NULL); 5724 5725 if (unlikely(err)) { 5726 __folio_clear_locked(folio); 5727 return err; 5728 } 5729 folio_clear_hugetlb_restore_reserve(folio); 5730 5731 /* 5732 * mark folio dirty so that it will not be removed from cache/file 5733 * by non-hugetlbfs specific code paths. 5734 */ 5735 folio_mark_dirty(folio); 5736 5737 spin_lock(&inode->i_lock); 5738 inode->i_blocks += blocks_per_huge_page(h); 5739 spin_unlock(&inode->i_lock); 5740 return 0; 5741 } 5742 5743 static inline vm_fault_t hugetlb_handle_userfault(struct vm_area_struct *vma, 5744 struct address_space *mapping, 5745 pgoff_t idx, 5746 unsigned int flags, 5747 unsigned long haddr, 5748 unsigned long addr, 5749 unsigned long reason) 5750 { 5751 u32 hash; 5752 struct vm_fault vmf = { 5753 .vma = vma, 5754 .address = haddr, 5755 .real_address = addr, 5756 .flags = flags, 5757 5758 /* 5759 * Hard to debug if it ends up being 5760 * used by a callee that assumes 5761 * something about the other 5762 * uninitialized fields... same as in 5763 * memory.c 5764 */ 5765 }; 5766 5767 /* 5768 * vma_lock and hugetlb_fault_mutex must be dropped before handling 5769 * userfault. Also mmap_lock could be dropped due to handling 5770 * userfault, any vma operation should be careful from here. 5771 */ 5772 hugetlb_vma_unlock_read(vma); 5773 hash = hugetlb_fault_mutex_hash(mapping, idx); 5774 mutex_unlock(&hugetlb_fault_mutex_table[hash]); 5775 return handle_userfault(&vmf, reason); 5776 } 5777 5778 /* 5779 * Recheck pte with pgtable lock. Returns true if pte didn't change, or 5780 * false if pte changed or is changing. 5781 */ 5782 static bool hugetlb_pte_stable(struct hstate *h, struct mm_struct *mm, 5783 pte_t *ptep, pte_t old_pte) 5784 { 5785 spinlock_t *ptl; 5786 bool same; 5787 5788 ptl = huge_pte_lock(h, mm, ptep); 5789 same = pte_same(huge_ptep_get(ptep), old_pte); 5790 spin_unlock(ptl); 5791 5792 return same; 5793 } 5794 5795 static vm_fault_t hugetlb_no_page(struct mm_struct *mm, 5796 struct vm_area_struct *vma, 5797 struct address_space *mapping, pgoff_t idx, 5798 unsigned long address, pte_t *ptep, 5799 pte_t old_pte, unsigned int flags) 5800 { 5801 struct hstate *h = hstate_vma(vma); 5802 vm_fault_t ret = VM_FAULT_SIGBUS; 5803 int anon_rmap = 0; 5804 unsigned long size; 5805 struct folio *folio; 5806 pte_t new_pte; 5807 spinlock_t *ptl; 5808 unsigned long haddr = address & huge_page_mask(h); 5809 bool new_folio, new_pagecache_folio = false; 5810 u32 hash = hugetlb_fault_mutex_hash(mapping, idx); 5811 5812 /* 5813 * Currently, we are forced to kill the process in the event the 5814 * original mapper has unmapped pages from the child due to a failed 5815 * COW/unsharing. Warn that such a situation has occurred as it may not 5816 * be obvious. 5817 */ 5818 if (is_vma_resv_set(vma, HPAGE_RESV_UNMAPPED)) { 5819 pr_warn_ratelimited("PID %d killed due to inadequate hugepage pool\n", 5820 current->pid); 5821 goto out; 5822 } 5823 5824 /* 5825 * Use page lock to guard against racing truncation 5826 * before we get page_table_lock. 5827 */ 5828 new_folio = false; 5829 folio = filemap_lock_folio(mapping, idx); 5830 if (IS_ERR(folio)) { 5831 size = i_size_read(mapping->host) >> huge_page_shift(h); 5832 if (idx >= size) 5833 goto out; 5834 /* Check for page in userfault range */ 5835 if (userfaultfd_missing(vma)) { 5836 /* 5837 * Since hugetlb_no_page() was examining pte 5838 * without pgtable lock, we need to re-test under 5839 * lock because the pte may not be stable and could 5840 * have changed from under us. Try to detect 5841 * either changed or during-changing ptes and retry 5842 * properly when needed. 5843 * 5844 * Note that userfaultfd is actually fine with 5845 * false positives (e.g. caused by pte changed), 5846 * but not wrong logical events (e.g. caused by 5847 * reading a pte during changing). The latter can 5848 * confuse the userspace, so the strictness is very 5849 * much preferred. E.g., MISSING event should 5850 * never happen on the page after UFFDIO_COPY has 5851 * correctly installed the page and returned. 5852 */ 5853 if (!hugetlb_pte_stable(h, mm, ptep, old_pte)) { 5854 ret = 0; 5855 goto out; 5856 } 5857 5858 return hugetlb_handle_userfault(vma, mapping, idx, flags, 5859 haddr, address, 5860 VM_UFFD_MISSING); 5861 } 5862 5863 folio = alloc_hugetlb_folio(vma, haddr, 0); 5864 if (IS_ERR(folio)) { 5865 /* 5866 * Returning error will result in faulting task being 5867 * sent SIGBUS. The hugetlb fault mutex prevents two 5868 * tasks from racing to fault in the same page which 5869 * could result in false unable to allocate errors. 5870 * Page migration does not take the fault mutex, but 5871 * does a clear then write of pte's under page table 5872 * lock. Page fault code could race with migration, 5873 * notice the clear pte and try to allocate a page 5874 * here. Before returning error, get ptl and make 5875 * sure there really is no pte entry. 5876 */ 5877 if (hugetlb_pte_stable(h, mm, ptep, old_pte)) 5878 ret = vmf_error(PTR_ERR(folio)); 5879 else 5880 ret = 0; 5881 goto out; 5882 } 5883 clear_huge_page(&folio->page, address, pages_per_huge_page(h)); 5884 __folio_mark_uptodate(folio); 5885 new_folio = true; 5886 5887 if (vma->vm_flags & VM_MAYSHARE) { 5888 int err = hugetlb_add_to_page_cache(folio, mapping, idx); 5889 if (err) { 5890 /* 5891 * err can't be -EEXIST which implies someone 5892 * else consumed the reservation since hugetlb 5893 * fault mutex is held when add a hugetlb page 5894 * to the page cache. So it's safe to call 5895 * restore_reserve_on_error() here. 5896 */ 5897 restore_reserve_on_error(h, vma, haddr, folio); 5898 folio_put(folio); 5899 goto out; 5900 } 5901 new_pagecache_folio = true; 5902 } else { 5903 folio_lock(folio); 5904 if (unlikely(anon_vma_prepare(vma))) { 5905 ret = VM_FAULT_OOM; 5906 goto backout_unlocked; 5907 } 5908 anon_rmap = 1; 5909 } 5910 } else { 5911 /* 5912 * If memory error occurs between mmap() and fault, some process 5913 * don't have hwpoisoned swap entry for errored virtual address. 5914 * So we need to block hugepage fault by PG_hwpoison bit check. 5915 */ 5916 if (unlikely(folio_test_hwpoison(folio))) { 5917 ret = VM_FAULT_HWPOISON_LARGE | 5918 VM_FAULT_SET_HINDEX(hstate_index(h)); 5919 goto backout_unlocked; 5920 } 5921 5922 /* Check for page in userfault range. */ 5923 if (userfaultfd_minor(vma)) { 5924 folio_unlock(folio); 5925 folio_put(folio); 5926 /* See comment in userfaultfd_missing() block above */ 5927 if (!hugetlb_pte_stable(h, mm, ptep, old_pte)) { 5928 ret = 0; 5929 goto out; 5930 } 5931 return hugetlb_handle_userfault(vma, mapping, idx, flags, 5932 haddr, address, 5933 VM_UFFD_MINOR); 5934 } 5935 } 5936 5937 /* 5938 * If we are going to COW a private mapping later, we examine the 5939 * pending reservations for this page now. This will ensure that 5940 * any allocations necessary to record that reservation occur outside 5941 * the spinlock. 5942 */ 5943 if ((flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED)) { 5944 if (vma_needs_reservation(h, vma, haddr) < 0) { 5945 ret = VM_FAULT_OOM; 5946 goto backout_unlocked; 5947 } 5948 /* Just decrements count, does not deallocate */ 5949 vma_end_reservation(h, vma, haddr); 5950 } 5951 5952 ptl = huge_pte_lock(h, mm, ptep); 5953 ret = 0; 5954 /* If pte changed from under us, retry */ 5955 if (!pte_same(huge_ptep_get(ptep), old_pte)) 5956 goto backout; 5957 5958 if (anon_rmap) 5959 hugepage_add_new_anon_rmap(folio, vma, haddr); 5960 else 5961 page_dup_file_rmap(&folio->page, true); 5962 new_pte = make_huge_pte(vma, &folio->page, ((vma->vm_flags & VM_WRITE) 5963 && (vma->vm_flags & VM_SHARED))); 5964 /* 5965 * If this pte was previously wr-protected, keep it wr-protected even 5966 * if populated. 5967 */ 5968 if (unlikely(pte_marker_uffd_wp(old_pte))) 5969 new_pte = huge_pte_mkuffd_wp(new_pte); 5970 set_huge_pte_at(mm, haddr, ptep, new_pte); 5971 5972 hugetlb_count_add(pages_per_huge_page(h), mm); 5973 if ((flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED)) { 5974 /* Optimization, do the COW without a second fault */ 5975 ret = hugetlb_wp(mm, vma, address, ptep, flags, folio, ptl); 5976 } 5977 5978 spin_unlock(ptl); 5979 5980 /* 5981 * Only set hugetlb_migratable in newly allocated pages. Existing pages 5982 * found in the pagecache may not have hugetlb_migratable if they have 5983 * been isolated for migration. 5984 */ 5985 if (new_folio) 5986 folio_set_hugetlb_migratable(folio); 5987 5988 folio_unlock(folio); 5989 out: 5990 hugetlb_vma_unlock_read(vma); 5991 mutex_unlock(&hugetlb_fault_mutex_table[hash]); 5992 return ret; 5993 5994 backout: 5995 spin_unlock(ptl); 5996 backout_unlocked: 5997 if (new_folio && !new_pagecache_folio) 5998 restore_reserve_on_error(h, vma, haddr, folio); 5999 6000 folio_unlock(folio); 6001 folio_put(folio); 6002 goto out; 6003 } 6004 6005 #ifdef CONFIG_SMP 6006 u32 hugetlb_fault_mutex_hash(struct address_space *mapping, pgoff_t idx) 6007 { 6008 unsigned long key[2]; 6009 u32 hash; 6010 6011 key[0] = (unsigned long) mapping; 6012 key[1] = idx; 6013 6014 hash = jhash2((u32 *)&key, sizeof(key)/(sizeof(u32)), 0); 6015 6016 return hash & (num_fault_mutexes - 1); 6017 } 6018 #else 6019 /* 6020 * For uniprocessor systems we always use a single mutex, so just 6021 * return 0 and avoid the hashing overhead. 6022 */ 6023 u32 hugetlb_fault_mutex_hash(struct address_space *mapping, pgoff_t idx) 6024 { 6025 return 0; 6026 } 6027 #endif 6028 6029 vm_fault_t hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma, 6030 unsigned long address, unsigned int flags) 6031 { 6032 pte_t *ptep, entry; 6033 spinlock_t *ptl; 6034 vm_fault_t ret; 6035 u32 hash; 6036 pgoff_t idx; 6037 struct folio *folio = NULL; 6038 struct folio *pagecache_folio = NULL; 6039 struct hstate *h = hstate_vma(vma); 6040 struct address_space *mapping; 6041 int need_wait_lock = 0; 6042 unsigned long haddr = address & huge_page_mask(h); 6043 6044 /* TODO: Handle faults under the VMA lock */ 6045 if (flags & FAULT_FLAG_VMA_LOCK) { 6046 vma_end_read(vma); 6047 return VM_FAULT_RETRY; 6048 } 6049 6050 /* 6051 * Serialize hugepage allocation and instantiation, so that we don't 6052 * get spurious allocation failures if two CPUs race to instantiate 6053 * the same page in the page cache. 6054 */ 6055 mapping = vma->vm_file->f_mapping; 6056 idx = vma_hugecache_offset(h, vma, haddr); 6057 hash = hugetlb_fault_mutex_hash(mapping, idx); 6058 mutex_lock(&hugetlb_fault_mutex_table[hash]); 6059 6060 /* 6061 * Acquire vma lock before calling huge_pte_alloc and hold 6062 * until finished with ptep. This prevents huge_pmd_unshare from 6063 * being called elsewhere and making the ptep no longer valid. 6064 */ 6065 hugetlb_vma_lock_read(vma); 6066 ptep = huge_pte_alloc(mm, vma, haddr, huge_page_size(h)); 6067 if (!ptep) { 6068 hugetlb_vma_unlock_read(vma); 6069 mutex_unlock(&hugetlb_fault_mutex_table[hash]); 6070 return VM_FAULT_OOM; 6071 } 6072 6073 entry = huge_ptep_get(ptep); 6074 if (huge_pte_none_mostly(entry)) { 6075 if (is_pte_marker(entry)) { 6076 pte_marker marker = 6077 pte_marker_get(pte_to_swp_entry(entry)); 6078 6079 if (marker & PTE_MARKER_POISONED) { 6080 ret = VM_FAULT_HWPOISON_LARGE; 6081 goto out_mutex; 6082 } 6083 } 6084 6085 /* 6086 * Other PTE markers should be handled the same way as none PTE. 6087 * 6088 * hugetlb_no_page will drop vma lock and hugetlb fault 6089 * mutex internally, which make us return immediately. 6090 */ 6091 return hugetlb_no_page(mm, vma, mapping, idx, address, ptep, 6092 entry, flags); 6093 } 6094 6095 ret = 0; 6096 6097 /* 6098 * entry could be a migration/hwpoison entry at this point, so this 6099 * check prevents the kernel from going below assuming that we have 6100 * an active hugepage in pagecache. This goto expects the 2nd page 6101 * fault, and is_hugetlb_entry_(migration|hwpoisoned) check will 6102 * properly handle it. 6103 */ 6104 if (!pte_present(entry)) { 6105 if (unlikely(is_hugetlb_entry_migration(entry))) { 6106 /* 6107 * Release the hugetlb fault lock now, but retain 6108 * the vma lock, because it is needed to guard the 6109 * huge_pte_lockptr() later in 6110 * migration_entry_wait_huge(). The vma lock will 6111 * be released there. 6112 */ 6113 mutex_unlock(&hugetlb_fault_mutex_table[hash]); 6114 migration_entry_wait_huge(vma, ptep); 6115 return 0; 6116 } else if (unlikely(is_hugetlb_entry_hwpoisoned(entry))) 6117 ret = VM_FAULT_HWPOISON_LARGE | 6118 VM_FAULT_SET_HINDEX(hstate_index(h)); 6119 goto out_mutex; 6120 } 6121 6122 /* 6123 * If we are going to COW/unshare the mapping later, we examine the 6124 * pending reservations for this page now. This will ensure that any 6125 * allocations necessary to record that reservation occur outside the 6126 * spinlock. Also lookup the pagecache page now as it is used to 6127 * determine if a reservation has been consumed. 6128 */ 6129 if ((flags & (FAULT_FLAG_WRITE|FAULT_FLAG_UNSHARE)) && 6130 !(vma->vm_flags & VM_MAYSHARE) && !huge_pte_write(entry)) { 6131 if (vma_needs_reservation(h, vma, haddr) < 0) { 6132 ret = VM_FAULT_OOM; 6133 goto out_mutex; 6134 } 6135 /* Just decrements count, does not deallocate */ 6136 vma_end_reservation(h, vma, haddr); 6137 6138 pagecache_folio = filemap_lock_folio(mapping, idx); 6139 if (IS_ERR(pagecache_folio)) 6140 pagecache_folio = NULL; 6141 } 6142 6143 ptl = huge_pte_lock(h, mm, ptep); 6144 6145 /* Check for a racing update before calling hugetlb_wp() */ 6146 if (unlikely(!pte_same(entry, huge_ptep_get(ptep)))) 6147 goto out_ptl; 6148 6149 /* Handle userfault-wp first, before trying to lock more pages */ 6150 if (userfaultfd_wp(vma) && huge_pte_uffd_wp(huge_ptep_get(ptep)) && 6151 (flags & FAULT_FLAG_WRITE) && !huge_pte_write(entry)) { 6152 struct vm_fault vmf = { 6153 .vma = vma, 6154 .address = haddr, 6155 .real_address = address, 6156 .flags = flags, 6157 }; 6158 6159 spin_unlock(ptl); 6160 if (pagecache_folio) { 6161 folio_unlock(pagecache_folio); 6162 folio_put(pagecache_folio); 6163 } 6164 hugetlb_vma_unlock_read(vma); 6165 mutex_unlock(&hugetlb_fault_mutex_table[hash]); 6166 return handle_userfault(&vmf, VM_UFFD_WP); 6167 } 6168 6169 /* 6170 * hugetlb_wp() requires page locks of pte_page(entry) and 6171 * pagecache_folio, so here we need take the former one 6172 * when folio != pagecache_folio or !pagecache_folio. 6173 */ 6174 folio = page_folio(pte_page(entry)); 6175 if (folio != pagecache_folio) 6176 if (!folio_trylock(folio)) { 6177 need_wait_lock = 1; 6178 goto out_ptl; 6179 } 6180 6181 folio_get(folio); 6182 6183 if (flags & (FAULT_FLAG_WRITE|FAULT_FLAG_UNSHARE)) { 6184 if (!huge_pte_write(entry)) { 6185 ret = hugetlb_wp(mm, vma, address, ptep, flags, 6186 pagecache_folio, ptl); 6187 goto out_put_page; 6188 } else if (likely(flags & FAULT_FLAG_WRITE)) { 6189 entry = huge_pte_mkdirty(entry); 6190 } 6191 } 6192 entry = pte_mkyoung(entry); 6193 if (huge_ptep_set_access_flags(vma, haddr, ptep, entry, 6194 flags & FAULT_FLAG_WRITE)) 6195 update_mmu_cache(vma, haddr, ptep); 6196 out_put_page: 6197 if (folio != pagecache_folio) 6198 folio_unlock(folio); 6199 folio_put(folio); 6200 out_ptl: 6201 spin_unlock(ptl); 6202 6203 if (pagecache_folio) { 6204 folio_unlock(pagecache_folio); 6205 folio_put(pagecache_folio); 6206 } 6207 out_mutex: 6208 hugetlb_vma_unlock_read(vma); 6209 mutex_unlock(&hugetlb_fault_mutex_table[hash]); 6210 /* 6211 * Generally it's safe to hold refcount during waiting page lock. But 6212 * here we just wait to defer the next page fault to avoid busy loop and 6213 * the page is not used after unlocked before returning from the current 6214 * page fault. So we are safe from accessing freed page, even if we wait 6215 * here without taking refcount. 6216 */ 6217 if (need_wait_lock) 6218 folio_wait_locked(folio); 6219 return ret; 6220 } 6221 6222 #ifdef CONFIG_USERFAULTFD 6223 /* 6224 * Used by userfaultfd UFFDIO_* ioctls. Based on userfaultfd's mfill_atomic_pte 6225 * with modifications for hugetlb pages. 6226 */ 6227 int hugetlb_mfill_atomic_pte(pte_t *dst_pte, 6228 struct vm_area_struct *dst_vma, 6229 unsigned long dst_addr, 6230 unsigned long src_addr, 6231 uffd_flags_t flags, 6232 struct folio **foliop) 6233 { 6234 struct mm_struct *dst_mm = dst_vma->vm_mm; 6235 bool is_continue = uffd_flags_mode_is(flags, MFILL_ATOMIC_CONTINUE); 6236 bool wp_enabled = (flags & MFILL_ATOMIC_WP); 6237 struct hstate *h = hstate_vma(dst_vma); 6238 struct address_space *mapping = dst_vma->vm_file->f_mapping; 6239 pgoff_t idx = vma_hugecache_offset(h, dst_vma, dst_addr); 6240 unsigned long size; 6241 int vm_shared = dst_vma->vm_flags & VM_SHARED; 6242 pte_t _dst_pte; 6243 spinlock_t *ptl; 6244 int ret = -ENOMEM; 6245 struct folio *folio; 6246 int writable; 6247 bool folio_in_pagecache = false; 6248 6249 if (uffd_flags_mode_is(flags, MFILL_ATOMIC_POISON)) { 6250 ptl = huge_pte_lock(h, dst_mm, dst_pte); 6251 6252 /* Don't overwrite any existing PTEs (even markers) */ 6253 if (!huge_pte_none(huge_ptep_get(dst_pte))) { 6254 spin_unlock(ptl); 6255 return -EEXIST; 6256 } 6257 6258 _dst_pte = make_pte_marker(PTE_MARKER_POISONED); 6259 set_huge_pte_at(dst_mm, dst_addr, dst_pte, _dst_pte); 6260 6261 /* No need to invalidate - it was non-present before */ 6262 update_mmu_cache(dst_vma, dst_addr, dst_pte); 6263 6264 spin_unlock(ptl); 6265 return 0; 6266 } 6267 6268 if (is_continue) { 6269 ret = -EFAULT; 6270 folio = filemap_lock_folio(mapping, idx); 6271 if (IS_ERR(folio)) 6272 goto out; 6273 folio_in_pagecache = true; 6274 } else if (!*foliop) { 6275 /* If a folio already exists, then it's UFFDIO_COPY for 6276 * a non-missing case. Return -EEXIST. 6277 */ 6278 if (vm_shared && 6279 hugetlbfs_pagecache_present(h, dst_vma, dst_addr)) { 6280 ret = -EEXIST; 6281 goto out; 6282 } 6283 6284 folio = alloc_hugetlb_folio(dst_vma, dst_addr, 0); 6285 if (IS_ERR(folio)) { 6286 ret = -ENOMEM; 6287 goto out; 6288 } 6289 6290 ret = copy_folio_from_user(folio, (const void __user *) src_addr, 6291 false); 6292 6293 /* fallback to copy_from_user outside mmap_lock */ 6294 if (unlikely(ret)) { 6295 ret = -ENOENT; 6296 /* Free the allocated folio which may have 6297 * consumed a reservation. 6298 */ 6299 restore_reserve_on_error(h, dst_vma, dst_addr, folio); 6300 folio_put(folio); 6301 6302 /* Allocate a temporary folio to hold the copied 6303 * contents. 6304 */ 6305 folio = alloc_hugetlb_folio_vma(h, dst_vma, dst_addr); 6306 if (!folio) { 6307 ret = -ENOMEM; 6308 goto out; 6309 } 6310 *foliop = folio; 6311 /* Set the outparam foliop and return to the caller to 6312 * copy the contents outside the lock. Don't free the 6313 * folio. 6314 */ 6315 goto out; 6316 } 6317 } else { 6318 if (vm_shared && 6319 hugetlbfs_pagecache_present(h, dst_vma, dst_addr)) { 6320 folio_put(*foliop); 6321 ret = -EEXIST; 6322 *foliop = NULL; 6323 goto out; 6324 } 6325 6326 folio = alloc_hugetlb_folio(dst_vma, dst_addr, 0); 6327 if (IS_ERR(folio)) { 6328 folio_put(*foliop); 6329 ret = -ENOMEM; 6330 *foliop = NULL; 6331 goto out; 6332 } 6333 ret = copy_user_large_folio(folio, *foliop, dst_addr, dst_vma); 6334 folio_put(*foliop); 6335 *foliop = NULL; 6336 if (ret) { 6337 folio_put(folio); 6338 goto out; 6339 } 6340 } 6341 6342 /* 6343 * The memory barrier inside __folio_mark_uptodate makes sure that 6344 * preceding stores to the page contents become visible before 6345 * the set_pte_at() write. 6346 */ 6347 __folio_mark_uptodate(folio); 6348 6349 /* Add shared, newly allocated pages to the page cache. */ 6350 if (vm_shared && !is_continue) { 6351 size = i_size_read(mapping->host) >> huge_page_shift(h); 6352 ret = -EFAULT; 6353 if (idx >= size) 6354 goto out_release_nounlock; 6355 6356 /* 6357 * Serialization between remove_inode_hugepages() and 6358 * hugetlb_add_to_page_cache() below happens through the 6359 * hugetlb_fault_mutex_table that here must be hold by 6360 * the caller. 6361 */ 6362 ret = hugetlb_add_to_page_cache(folio, mapping, idx); 6363 if (ret) 6364 goto out_release_nounlock; 6365 folio_in_pagecache = true; 6366 } 6367 6368 ptl = huge_pte_lock(h, dst_mm, dst_pte); 6369 6370 ret = -EIO; 6371 if (folio_test_hwpoison(folio)) 6372 goto out_release_unlock; 6373 6374 /* 6375 * We allow to overwrite a pte marker: consider when both MISSING|WP 6376 * registered, we firstly wr-protect a none pte which has no page cache 6377 * page backing it, then access the page. 6378 */ 6379 ret = -EEXIST; 6380 if (!huge_pte_none_mostly(huge_ptep_get(dst_pte))) 6381 goto out_release_unlock; 6382 6383 if (folio_in_pagecache) 6384 page_dup_file_rmap(&folio->page, true); 6385 else 6386 hugepage_add_new_anon_rmap(folio, dst_vma, dst_addr); 6387 6388 /* 6389 * For either: (1) CONTINUE on a non-shared VMA, or (2) UFFDIO_COPY 6390 * with wp flag set, don't set pte write bit. 6391 */ 6392 if (wp_enabled || (is_continue && !vm_shared)) 6393 writable = 0; 6394 else 6395 writable = dst_vma->vm_flags & VM_WRITE; 6396 6397 _dst_pte = make_huge_pte(dst_vma, &folio->page, writable); 6398 /* 6399 * Always mark UFFDIO_COPY page dirty; note that this may not be 6400 * extremely important for hugetlbfs for now since swapping is not 6401 * supported, but we should still be clear in that this page cannot be 6402 * thrown away at will, even if write bit not set. 6403 */ 6404 _dst_pte = huge_pte_mkdirty(_dst_pte); 6405 _dst_pte = pte_mkyoung(_dst_pte); 6406 6407 if (wp_enabled) 6408 _dst_pte = huge_pte_mkuffd_wp(_dst_pte); 6409 6410 set_huge_pte_at(dst_mm, dst_addr, dst_pte, _dst_pte); 6411 6412 hugetlb_count_add(pages_per_huge_page(h), dst_mm); 6413 6414 /* No need to invalidate - it was non-present before */ 6415 update_mmu_cache(dst_vma, dst_addr, dst_pte); 6416 6417 spin_unlock(ptl); 6418 if (!is_continue) 6419 folio_set_hugetlb_migratable(folio); 6420 if (vm_shared || is_continue) 6421 folio_unlock(folio); 6422 ret = 0; 6423 out: 6424 return ret; 6425 out_release_unlock: 6426 spin_unlock(ptl); 6427 if (vm_shared || is_continue) 6428 folio_unlock(folio); 6429 out_release_nounlock: 6430 if (!folio_in_pagecache) 6431 restore_reserve_on_error(h, dst_vma, dst_addr, folio); 6432 folio_put(folio); 6433 goto out; 6434 } 6435 #endif /* CONFIG_USERFAULTFD */ 6436 6437 struct page *hugetlb_follow_page_mask(struct vm_area_struct *vma, 6438 unsigned long address, unsigned int flags, 6439 unsigned int *page_mask) 6440 { 6441 struct hstate *h = hstate_vma(vma); 6442 struct mm_struct *mm = vma->vm_mm; 6443 unsigned long haddr = address & huge_page_mask(h); 6444 struct page *page = NULL; 6445 spinlock_t *ptl; 6446 pte_t *pte, entry; 6447 int ret; 6448 6449 hugetlb_vma_lock_read(vma); 6450 pte = hugetlb_walk(vma, haddr, huge_page_size(h)); 6451 if (!pte) 6452 goto out_unlock; 6453 6454 ptl = huge_pte_lock(h, mm, pte); 6455 entry = huge_ptep_get(pte); 6456 if (pte_present(entry)) { 6457 page = pte_page(entry); 6458 6459 if (!huge_pte_write(entry)) { 6460 if (flags & FOLL_WRITE) { 6461 page = NULL; 6462 goto out; 6463 } 6464 6465 if (gup_must_unshare(vma, flags, page)) { 6466 /* Tell the caller to do unsharing */ 6467 page = ERR_PTR(-EMLINK); 6468 goto out; 6469 } 6470 } 6471 6472 page += ((address & ~huge_page_mask(h)) >> PAGE_SHIFT); 6473 6474 /* 6475 * Note that page may be a sub-page, and with vmemmap 6476 * optimizations the page struct may be read only. 6477 * try_grab_page() will increase the ref count on the 6478 * head page, so this will be OK. 6479 * 6480 * try_grab_page() should always be able to get the page here, 6481 * because we hold the ptl lock and have verified pte_present(). 6482 */ 6483 ret = try_grab_page(page, flags); 6484 6485 if (WARN_ON_ONCE(ret)) { 6486 page = ERR_PTR(ret); 6487 goto out; 6488 } 6489 6490 *page_mask = (1U << huge_page_order(h)) - 1; 6491 } 6492 out: 6493 spin_unlock(ptl); 6494 out_unlock: 6495 hugetlb_vma_unlock_read(vma); 6496 6497 /* 6498 * Fixup retval for dump requests: if pagecache doesn't exist, 6499 * don't try to allocate a new page but just skip it. 6500 */ 6501 if (!page && (flags & FOLL_DUMP) && 6502 !hugetlbfs_pagecache_present(h, vma, address)) 6503 page = ERR_PTR(-EFAULT); 6504 6505 return page; 6506 } 6507 6508 long hugetlb_change_protection(struct vm_area_struct *vma, 6509 unsigned long address, unsigned long end, 6510 pgprot_t newprot, unsigned long cp_flags) 6511 { 6512 struct mm_struct *mm = vma->vm_mm; 6513 unsigned long start = address; 6514 pte_t *ptep; 6515 pte_t pte; 6516 struct hstate *h = hstate_vma(vma); 6517 long pages = 0, psize = huge_page_size(h); 6518 bool shared_pmd = false; 6519 struct mmu_notifier_range range; 6520 unsigned long last_addr_mask; 6521 bool uffd_wp = cp_flags & MM_CP_UFFD_WP; 6522 bool uffd_wp_resolve = cp_flags & MM_CP_UFFD_WP_RESOLVE; 6523 6524 /* 6525 * In the case of shared PMDs, the area to flush could be beyond 6526 * start/end. Set range.start/range.end to cover the maximum possible 6527 * range if PMD sharing is possible. 6528 */ 6529 mmu_notifier_range_init(&range, MMU_NOTIFY_PROTECTION_VMA, 6530 0, mm, start, end); 6531 adjust_range_if_pmd_sharing_possible(vma, &range.start, &range.end); 6532 6533 BUG_ON(address >= end); 6534 flush_cache_range(vma, range.start, range.end); 6535 6536 mmu_notifier_invalidate_range_start(&range); 6537 hugetlb_vma_lock_write(vma); 6538 i_mmap_lock_write(vma->vm_file->f_mapping); 6539 last_addr_mask = hugetlb_mask_last_page(h); 6540 for (; address < end; address += psize) { 6541 spinlock_t *ptl; 6542 ptep = hugetlb_walk(vma, address, psize); 6543 if (!ptep) { 6544 if (!uffd_wp) { 6545 address |= last_addr_mask; 6546 continue; 6547 } 6548 /* 6549 * Userfaultfd wr-protect requires pgtable 6550 * pre-allocations to install pte markers. 6551 */ 6552 ptep = huge_pte_alloc(mm, vma, address, psize); 6553 if (!ptep) { 6554 pages = -ENOMEM; 6555 break; 6556 } 6557 } 6558 ptl = huge_pte_lock(h, mm, ptep); 6559 if (huge_pmd_unshare(mm, vma, address, ptep)) { 6560 /* 6561 * When uffd-wp is enabled on the vma, unshare 6562 * shouldn't happen at all. Warn about it if it 6563 * happened due to some reason. 6564 */ 6565 WARN_ON_ONCE(uffd_wp || uffd_wp_resolve); 6566 pages++; 6567 spin_unlock(ptl); 6568 shared_pmd = true; 6569 address |= last_addr_mask; 6570 continue; 6571 } 6572 pte = huge_ptep_get(ptep); 6573 if (unlikely(is_hugetlb_entry_hwpoisoned(pte))) { 6574 /* Nothing to do. */ 6575 } else if (unlikely(is_hugetlb_entry_migration(pte))) { 6576 swp_entry_t entry = pte_to_swp_entry(pte); 6577 struct page *page = pfn_swap_entry_to_page(entry); 6578 pte_t newpte = pte; 6579 6580 if (is_writable_migration_entry(entry)) { 6581 if (PageAnon(page)) 6582 entry = make_readable_exclusive_migration_entry( 6583 swp_offset(entry)); 6584 else 6585 entry = make_readable_migration_entry( 6586 swp_offset(entry)); 6587 newpte = swp_entry_to_pte(entry); 6588 pages++; 6589 } 6590 6591 if (uffd_wp) 6592 newpte = pte_swp_mkuffd_wp(newpte); 6593 else if (uffd_wp_resolve) 6594 newpte = pte_swp_clear_uffd_wp(newpte); 6595 if (!pte_same(pte, newpte)) 6596 set_huge_pte_at(mm, address, ptep, newpte); 6597 } else if (unlikely(is_pte_marker(pte))) { 6598 /* No other markers apply for now. */ 6599 WARN_ON_ONCE(!pte_marker_uffd_wp(pte)); 6600 if (uffd_wp_resolve) 6601 /* Safe to modify directly (non-present->none). */ 6602 huge_pte_clear(mm, address, ptep, psize); 6603 } else if (!huge_pte_none(pte)) { 6604 pte_t old_pte; 6605 unsigned int shift = huge_page_shift(hstate_vma(vma)); 6606 6607 old_pte = huge_ptep_modify_prot_start(vma, address, ptep); 6608 pte = huge_pte_modify(old_pte, newprot); 6609 pte = arch_make_huge_pte(pte, shift, vma->vm_flags); 6610 if (uffd_wp) 6611 pte = huge_pte_mkuffd_wp(pte); 6612 else if (uffd_wp_resolve) 6613 pte = huge_pte_clear_uffd_wp(pte); 6614 huge_ptep_modify_prot_commit(vma, address, ptep, old_pte, pte); 6615 pages++; 6616 } else { 6617 /* None pte */ 6618 if (unlikely(uffd_wp)) 6619 /* Safe to modify directly (none->non-present). */ 6620 set_huge_pte_at(mm, address, ptep, 6621 make_pte_marker(PTE_MARKER_UFFD_WP)); 6622 } 6623 spin_unlock(ptl); 6624 } 6625 /* 6626 * Must flush TLB before releasing i_mmap_rwsem: x86's huge_pmd_unshare 6627 * may have cleared our pud entry and done put_page on the page table: 6628 * once we release i_mmap_rwsem, another task can do the final put_page 6629 * and that page table be reused and filled with junk. If we actually 6630 * did unshare a page of pmds, flush the range corresponding to the pud. 6631 */ 6632 if (shared_pmd) 6633 flush_hugetlb_tlb_range(vma, range.start, range.end); 6634 else 6635 flush_hugetlb_tlb_range(vma, start, end); 6636 /* 6637 * No need to call mmu_notifier_arch_invalidate_secondary_tlbs() we are 6638 * downgrading page table protection not changing it to point to a new 6639 * page. 6640 * 6641 * See Documentation/mm/mmu_notifier.rst 6642 */ 6643 i_mmap_unlock_write(vma->vm_file->f_mapping); 6644 hugetlb_vma_unlock_write(vma); 6645 mmu_notifier_invalidate_range_end(&range); 6646 6647 return pages > 0 ? (pages << h->order) : pages; 6648 } 6649 6650 /* Return true if reservation was successful, false otherwise. */ 6651 bool hugetlb_reserve_pages(struct inode *inode, 6652 long from, long to, 6653 struct vm_area_struct *vma, 6654 vm_flags_t vm_flags) 6655 { 6656 long chg = -1, add = -1; 6657 struct hstate *h = hstate_inode(inode); 6658 struct hugepage_subpool *spool = subpool_inode(inode); 6659 struct resv_map *resv_map; 6660 struct hugetlb_cgroup *h_cg = NULL; 6661 long gbl_reserve, regions_needed = 0; 6662 6663 /* This should never happen */ 6664 if (from > to) { 6665 VM_WARN(1, "%s called with a negative range\n", __func__); 6666 return false; 6667 } 6668 6669 /* 6670 * vma specific semaphore used for pmd sharing and fault/truncation 6671 * synchronization 6672 */ 6673 hugetlb_vma_lock_alloc(vma); 6674 6675 /* 6676 * Only apply hugepage reservation if asked. At fault time, an 6677 * attempt will be made for VM_NORESERVE to allocate a page 6678 * without using reserves 6679 */ 6680 if (vm_flags & VM_NORESERVE) 6681 return true; 6682 6683 /* 6684 * Shared mappings base their reservation on the number of pages that 6685 * are already allocated on behalf of the file. Private mappings need 6686 * to reserve the full area even if read-only as mprotect() may be 6687 * called to make the mapping read-write. Assume !vma is a shm mapping 6688 */ 6689 if (!vma || vma->vm_flags & VM_MAYSHARE) { 6690 /* 6691 * resv_map can not be NULL as hugetlb_reserve_pages is only 6692 * called for inodes for which resv_maps were created (see 6693 * hugetlbfs_get_inode). 6694 */ 6695 resv_map = inode_resv_map(inode); 6696 6697 chg = region_chg(resv_map, from, to, ®ions_needed); 6698 } else { 6699 /* Private mapping. */ 6700 resv_map = resv_map_alloc(); 6701 if (!resv_map) 6702 goto out_err; 6703 6704 chg = to - from; 6705 6706 set_vma_resv_map(vma, resv_map); 6707 set_vma_resv_flags(vma, HPAGE_RESV_OWNER); 6708 } 6709 6710 if (chg < 0) 6711 goto out_err; 6712 6713 if (hugetlb_cgroup_charge_cgroup_rsvd(hstate_index(h), 6714 chg * pages_per_huge_page(h), &h_cg) < 0) 6715 goto out_err; 6716 6717 if (vma && !(vma->vm_flags & VM_MAYSHARE) && h_cg) { 6718 /* For private mappings, the hugetlb_cgroup uncharge info hangs 6719 * of the resv_map. 6720 */ 6721 resv_map_set_hugetlb_cgroup_uncharge_info(resv_map, h_cg, h); 6722 } 6723 6724 /* 6725 * There must be enough pages in the subpool for the mapping. If 6726 * the subpool has a minimum size, there may be some global 6727 * reservations already in place (gbl_reserve). 6728 */ 6729 gbl_reserve = hugepage_subpool_get_pages(spool, chg); 6730 if (gbl_reserve < 0) 6731 goto out_uncharge_cgroup; 6732 6733 /* 6734 * Check enough hugepages are available for the reservation. 6735 * Hand the pages back to the subpool if there are not 6736 */ 6737 if (hugetlb_acct_memory(h, gbl_reserve) < 0) 6738 goto out_put_pages; 6739 6740 /* 6741 * Account for the reservations made. Shared mappings record regions 6742 * that have reservations as they are shared by multiple VMAs. 6743 * When the last VMA disappears, the region map says how much 6744 * the reservation was and the page cache tells how much of 6745 * the reservation was consumed. Private mappings are per-VMA and 6746 * only the consumed reservations are tracked. When the VMA 6747 * disappears, the original reservation is the VMA size and the 6748 * consumed reservations are stored in the map. Hence, nothing 6749 * else has to be done for private mappings here 6750 */ 6751 if (!vma || vma->vm_flags & VM_MAYSHARE) { 6752 add = region_add(resv_map, from, to, regions_needed, h, h_cg); 6753 6754 if (unlikely(add < 0)) { 6755 hugetlb_acct_memory(h, -gbl_reserve); 6756 goto out_put_pages; 6757 } else if (unlikely(chg > add)) { 6758 /* 6759 * pages in this range were added to the reserve 6760 * map between region_chg and region_add. This 6761 * indicates a race with alloc_hugetlb_folio. Adjust 6762 * the subpool and reserve counts modified above 6763 * based on the difference. 6764 */ 6765 long rsv_adjust; 6766 6767 /* 6768 * hugetlb_cgroup_uncharge_cgroup_rsvd() will put the 6769 * reference to h_cg->css. See comment below for detail. 6770 */ 6771 hugetlb_cgroup_uncharge_cgroup_rsvd( 6772 hstate_index(h), 6773 (chg - add) * pages_per_huge_page(h), h_cg); 6774 6775 rsv_adjust = hugepage_subpool_put_pages(spool, 6776 chg - add); 6777 hugetlb_acct_memory(h, -rsv_adjust); 6778 } else if (h_cg) { 6779 /* 6780 * The file_regions will hold their own reference to 6781 * h_cg->css. So we should release the reference held 6782 * via hugetlb_cgroup_charge_cgroup_rsvd() when we are 6783 * done. 6784 */ 6785 hugetlb_cgroup_put_rsvd_cgroup(h_cg); 6786 } 6787 } 6788 return true; 6789 6790 out_put_pages: 6791 /* put back original number of pages, chg */ 6792 (void)hugepage_subpool_put_pages(spool, chg); 6793 out_uncharge_cgroup: 6794 hugetlb_cgroup_uncharge_cgroup_rsvd(hstate_index(h), 6795 chg * pages_per_huge_page(h), h_cg); 6796 out_err: 6797 hugetlb_vma_lock_free(vma); 6798 if (!vma || vma->vm_flags & VM_MAYSHARE) 6799 /* Only call region_abort if the region_chg succeeded but the 6800 * region_add failed or didn't run. 6801 */ 6802 if (chg >= 0 && add < 0) 6803 region_abort(resv_map, from, to, regions_needed); 6804 if (vma && is_vma_resv_set(vma, HPAGE_RESV_OWNER)) 6805 kref_put(&resv_map->refs, resv_map_release); 6806 return false; 6807 } 6808 6809 long hugetlb_unreserve_pages(struct inode *inode, long start, long end, 6810 long freed) 6811 { 6812 struct hstate *h = hstate_inode(inode); 6813 struct resv_map *resv_map = inode_resv_map(inode); 6814 long chg = 0; 6815 struct hugepage_subpool *spool = subpool_inode(inode); 6816 long gbl_reserve; 6817 6818 /* 6819 * Since this routine can be called in the evict inode path for all 6820 * hugetlbfs inodes, resv_map could be NULL. 6821 */ 6822 if (resv_map) { 6823 chg = region_del(resv_map, start, end); 6824 /* 6825 * region_del() can fail in the rare case where a region 6826 * must be split and another region descriptor can not be 6827 * allocated. If end == LONG_MAX, it will not fail. 6828 */ 6829 if (chg < 0) 6830 return chg; 6831 } 6832 6833 spin_lock(&inode->i_lock); 6834 inode->i_blocks -= (blocks_per_huge_page(h) * freed); 6835 spin_unlock(&inode->i_lock); 6836 6837 /* 6838 * If the subpool has a minimum size, the number of global 6839 * reservations to be released may be adjusted. 6840 * 6841 * Note that !resv_map implies freed == 0. So (chg - freed) 6842 * won't go negative. 6843 */ 6844 gbl_reserve = hugepage_subpool_put_pages(spool, (chg - freed)); 6845 hugetlb_acct_memory(h, -gbl_reserve); 6846 6847 return 0; 6848 } 6849 6850 #ifdef CONFIG_ARCH_WANT_HUGE_PMD_SHARE 6851 static unsigned long page_table_shareable(struct vm_area_struct *svma, 6852 struct vm_area_struct *vma, 6853 unsigned long addr, pgoff_t idx) 6854 { 6855 unsigned long saddr = ((idx - svma->vm_pgoff) << PAGE_SHIFT) + 6856 svma->vm_start; 6857 unsigned long sbase = saddr & PUD_MASK; 6858 unsigned long s_end = sbase + PUD_SIZE; 6859 6860 /* Allow segments to share if only one is marked locked */ 6861 unsigned long vm_flags = vma->vm_flags & ~VM_LOCKED_MASK; 6862 unsigned long svm_flags = svma->vm_flags & ~VM_LOCKED_MASK; 6863 6864 /* 6865 * match the virtual addresses, permission and the alignment of the 6866 * page table page. 6867 * 6868 * Also, vma_lock (vm_private_data) is required for sharing. 6869 */ 6870 if (pmd_index(addr) != pmd_index(saddr) || 6871 vm_flags != svm_flags || 6872 !range_in_vma(svma, sbase, s_end) || 6873 !svma->vm_private_data) 6874 return 0; 6875 6876 return saddr; 6877 } 6878 6879 bool want_pmd_share(struct vm_area_struct *vma, unsigned long addr) 6880 { 6881 unsigned long start = addr & PUD_MASK; 6882 unsigned long end = start + PUD_SIZE; 6883 6884 #ifdef CONFIG_USERFAULTFD 6885 if (uffd_disable_huge_pmd_share(vma)) 6886 return false; 6887 #endif 6888 /* 6889 * check on proper vm_flags and page table alignment 6890 */ 6891 if (!(vma->vm_flags & VM_MAYSHARE)) 6892 return false; 6893 if (!vma->vm_private_data) /* vma lock required for sharing */ 6894 return false; 6895 if (!range_in_vma(vma, start, end)) 6896 return false; 6897 return true; 6898 } 6899 6900 /* 6901 * Determine if start,end range within vma could be mapped by shared pmd. 6902 * If yes, adjust start and end to cover range associated with possible 6903 * shared pmd mappings. 6904 */ 6905 void adjust_range_if_pmd_sharing_possible(struct vm_area_struct *vma, 6906 unsigned long *start, unsigned long *end) 6907 { 6908 unsigned long v_start = ALIGN(vma->vm_start, PUD_SIZE), 6909 v_end = ALIGN_DOWN(vma->vm_end, PUD_SIZE); 6910 6911 /* 6912 * vma needs to span at least one aligned PUD size, and the range 6913 * must be at least partially within in. 6914 */ 6915 if (!(vma->vm_flags & VM_MAYSHARE) || !(v_end > v_start) || 6916 (*end <= v_start) || (*start >= v_end)) 6917 return; 6918 6919 /* Extend the range to be PUD aligned for a worst case scenario */ 6920 if (*start > v_start) 6921 *start = ALIGN_DOWN(*start, PUD_SIZE); 6922 6923 if (*end < v_end) 6924 *end = ALIGN(*end, PUD_SIZE); 6925 } 6926 6927 /* 6928 * Search for a shareable pmd page for hugetlb. In any case calls pmd_alloc() 6929 * and returns the corresponding pte. While this is not necessary for the 6930 * !shared pmd case because we can allocate the pmd later as well, it makes the 6931 * code much cleaner. pmd allocation is essential for the shared case because 6932 * pud has to be populated inside the same i_mmap_rwsem section - otherwise 6933 * racing tasks could either miss the sharing (see huge_pte_offset) or select a 6934 * bad pmd for sharing. 6935 */ 6936 pte_t *huge_pmd_share(struct mm_struct *mm, struct vm_area_struct *vma, 6937 unsigned long addr, pud_t *pud) 6938 { 6939 struct address_space *mapping = vma->vm_file->f_mapping; 6940 pgoff_t idx = ((addr - vma->vm_start) >> PAGE_SHIFT) + 6941 vma->vm_pgoff; 6942 struct vm_area_struct *svma; 6943 unsigned long saddr; 6944 pte_t *spte = NULL; 6945 pte_t *pte; 6946 6947 i_mmap_lock_read(mapping); 6948 vma_interval_tree_foreach(svma, &mapping->i_mmap, idx, idx) { 6949 if (svma == vma) 6950 continue; 6951 6952 saddr = page_table_shareable(svma, vma, addr, idx); 6953 if (saddr) { 6954 spte = hugetlb_walk(svma, saddr, 6955 vma_mmu_pagesize(svma)); 6956 if (spte) { 6957 get_page(virt_to_page(spte)); 6958 break; 6959 } 6960 } 6961 } 6962 6963 if (!spte) 6964 goto out; 6965 6966 spin_lock(&mm->page_table_lock); 6967 if (pud_none(*pud)) { 6968 pud_populate(mm, pud, 6969 (pmd_t *)((unsigned long)spte & PAGE_MASK)); 6970 mm_inc_nr_pmds(mm); 6971 } else { 6972 put_page(virt_to_page(spte)); 6973 } 6974 spin_unlock(&mm->page_table_lock); 6975 out: 6976 pte = (pte_t *)pmd_alloc(mm, pud, addr); 6977 i_mmap_unlock_read(mapping); 6978 return pte; 6979 } 6980 6981 /* 6982 * unmap huge page backed by shared pte. 6983 * 6984 * Hugetlb pte page is ref counted at the time of mapping. If pte is shared 6985 * indicated by page_count > 1, unmap is achieved by clearing pud and 6986 * decrementing the ref count. If count == 1, the pte page is not shared. 6987 * 6988 * Called with page table lock held. 6989 * 6990 * returns: 1 successfully unmapped a shared pte page 6991 * 0 the underlying pte page is not shared, or it is the last user 6992 */ 6993 int huge_pmd_unshare(struct mm_struct *mm, struct vm_area_struct *vma, 6994 unsigned long addr, pte_t *ptep) 6995 { 6996 pgd_t *pgd = pgd_offset(mm, addr); 6997 p4d_t *p4d = p4d_offset(pgd, addr); 6998 pud_t *pud = pud_offset(p4d, addr); 6999 7000 i_mmap_assert_write_locked(vma->vm_file->f_mapping); 7001 hugetlb_vma_assert_locked(vma); 7002 BUG_ON(page_count(virt_to_page(ptep)) == 0); 7003 if (page_count(virt_to_page(ptep)) == 1) 7004 return 0; 7005 7006 pud_clear(pud); 7007 put_page(virt_to_page(ptep)); 7008 mm_dec_nr_pmds(mm); 7009 return 1; 7010 } 7011 7012 #else /* !CONFIG_ARCH_WANT_HUGE_PMD_SHARE */ 7013 7014 pte_t *huge_pmd_share(struct mm_struct *mm, struct vm_area_struct *vma, 7015 unsigned long addr, pud_t *pud) 7016 { 7017 return NULL; 7018 } 7019 7020 int huge_pmd_unshare(struct mm_struct *mm, struct vm_area_struct *vma, 7021 unsigned long addr, pte_t *ptep) 7022 { 7023 return 0; 7024 } 7025 7026 void adjust_range_if_pmd_sharing_possible(struct vm_area_struct *vma, 7027 unsigned long *start, unsigned long *end) 7028 { 7029 } 7030 7031 bool want_pmd_share(struct vm_area_struct *vma, unsigned long addr) 7032 { 7033 return false; 7034 } 7035 #endif /* CONFIG_ARCH_WANT_HUGE_PMD_SHARE */ 7036 7037 #ifdef CONFIG_ARCH_WANT_GENERAL_HUGETLB 7038 pte_t *huge_pte_alloc(struct mm_struct *mm, struct vm_area_struct *vma, 7039 unsigned long addr, unsigned long sz) 7040 { 7041 pgd_t *pgd; 7042 p4d_t *p4d; 7043 pud_t *pud; 7044 pte_t *pte = NULL; 7045 7046 pgd = pgd_offset(mm, addr); 7047 p4d = p4d_alloc(mm, pgd, addr); 7048 if (!p4d) 7049 return NULL; 7050 pud = pud_alloc(mm, p4d, addr); 7051 if (pud) { 7052 if (sz == PUD_SIZE) { 7053 pte = (pte_t *)pud; 7054 } else { 7055 BUG_ON(sz != PMD_SIZE); 7056 if (want_pmd_share(vma, addr) && pud_none(*pud)) 7057 pte = huge_pmd_share(mm, vma, addr, pud); 7058 else 7059 pte = (pte_t *)pmd_alloc(mm, pud, addr); 7060 } 7061 } 7062 7063 if (pte) { 7064 pte_t pteval = ptep_get_lockless(pte); 7065 7066 BUG_ON(pte_present(pteval) && !pte_huge(pteval)); 7067 } 7068 7069 return pte; 7070 } 7071 7072 /* 7073 * huge_pte_offset() - Walk the page table to resolve the hugepage 7074 * entry at address @addr 7075 * 7076 * Return: Pointer to page table entry (PUD or PMD) for 7077 * address @addr, or NULL if a !p*d_present() entry is encountered and the 7078 * size @sz doesn't match the hugepage size at this level of the page 7079 * table. 7080 */ 7081 pte_t *huge_pte_offset(struct mm_struct *mm, 7082 unsigned long addr, unsigned long sz) 7083 { 7084 pgd_t *pgd; 7085 p4d_t *p4d; 7086 pud_t *pud; 7087 pmd_t *pmd; 7088 7089 pgd = pgd_offset(mm, addr); 7090 if (!pgd_present(*pgd)) 7091 return NULL; 7092 p4d = p4d_offset(pgd, addr); 7093 if (!p4d_present(*p4d)) 7094 return NULL; 7095 7096 pud = pud_offset(p4d, addr); 7097 if (sz == PUD_SIZE) 7098 /* must be pud huge, non-present or none */ 7099 return (pte_t *)pud; 7100 if (!pud_present(*pud)) 7101 return NULL; 7102 /* must have a valid entry and size to go further */ 7103 7104 pmd = pmd_offset(pud, addr); 7105 /* must be pmd huge, non-present or none */ 7106 return (pte_t *)pmd; 7107 } 7108 7109 /* 7110 * Return a mask that can be used to update an address to the last huge 7111 * page in a page table page mapping size. Used to skip non-present 7112 * page table entries when linearly scanning address ranges. Architectures 7113 * with unique huge page to page table relationships can define their own 7114 * version of this routine. 7115 */ 7116 unsigned long hugetlb_mask_last_page(struct hstate *h) 7117 { 7118 unsigned long hp_size = huge_page_size(h); 7119 7120 if (hp_size == PUD_SIZE) 7121 return P4D_SIZE - PUD_SIZE; 7122 else if (hp_size == PMD_SIZE) 7123 return PUD_SIZE - PMD_SIZE; 7124 else 7125 return 0UL; 7126 } 7127 7128 #else 7129 7130 /* See description above. Architectures can provide their own version. */ 7131 __weak unsigned long hugetlb_mask_last_page(struct hstate *h) 7132 { 7133 #ifdef CONFIG_ARCH_WANT_HUGE_PMD_SHARE 7134 if (huge_page_size(h) == PMD_SIZE) 7135 return PUD_SIZE - PMD_SIZE; 7136 #endif 7137 return 0UL; 7138 } 7139 7140 #endif /* CONFIG_ARCH_WANT_GENERAL_HUGETLB */ 7141 7142 /* 7143 * These functions are overwritable if your architecture needs its own 7144 * behavior. 7145 */ 7146 bool isolate_hugetlb(struct folio *folio, struct list_head *list) 7147 { 7148 bool ret = true; 7149 7150 spin_lock_irq(&hugetlb_lock); 7151 if (!folio_test_hugetlb(folio) || 7152 !folio_test_hugetlb_migratable(folio) || 7153 !folio_try_get(folio)) { 7154 ret = false; 7155 goto unlock; 7156 } 7157 folio_clear_hugetlb_migratable(folio); 7158 list_move_tail(&folio->lru, list); 7159 unlock: 7160 spin_unlock_irq(&hugetlb_lock); 7161 return ret; 7162 } 7163 7164 int get_hwpoison_hugetlb_folio(struct folio *folio, bool *hugetlb, bool unpoison) 7165 { 7166 int ret = 0; 7167 7168 *hugetlb = false; 7169 spin_lock_irq(&hugetlb_lock); 7170 if (folio_test_hugetlb(folio)) { 7171 *hugetlb = true; 7172 if (folio_test_hugetlb_freed(folio)) 7173 ret = 0; 7174 else if (folio_test_hugetlb_migratable(folio) || unpoison) 7175 ret = folio_try_get(folio); 7176 else 7177 ret = -EBUSY; 7178 } 7179 spin_unlock_irq(&hugetlb_lock); 7180 return ret; 7181 } 7182 7183 int get_huge_page_for_hwpoison(unsigned long pfn, int flags, 7184 bool *migratable_cleared) 7185 { 7186 int ret; 7187 7188 spin_lock_irq(&hugetlb_lock); 7189 ret = __get_huge_page_for_hwpoison(pfn, flags, migratable_cleared); 7190 spin_unlock_irq(&hugetlb_lock); 7191 return ret; 7192 } 7193 7194 void folio_putback_active_hugetlb(struct folio *folio) 7195 { 7196 spin_lock_irq(&hugetlb_lock); 7197 folio_set_hugetlb_migratable(folio); 7198 list_move_tail(&folio->lru, &(folio_hstate(folio))->hugepage_activelist); 7199 spin_unlock_irq(&hugetlb_lock); 7200 folio_put(folio); 7201 } 7202 7203 void move_hugetlb_state(struct folio *old_folio, struct folio *new_folio, int reason) 7204 { 7205 struct hstate *h = folio_hstate(old_folio); 7206 7207 hugetlb_cgroup_migrate(old_folio, new_folio); 7208 set_page_owner_migrate_reason(&new_folio->page, reason); 7209 7210 /* 7211 * transfer temporary state of the new hugetlb folio. This is 7212 * reverse to other transitions because the newpage is going to 7213 * be final while the old one will be freed so it takes over 7214 * the temporary status. 7215 * 7216 * Also note that we have to transfer the per-node surplus state 7217 * here as well otherwise the global surplus count will not match 7218 * the per-node's. 7219 */ 7220 if (folio_test_hugetlb_temporary(new_folio)) { 7221 int old_nid = folio_nid(old_folio); 7222 int new_nid = folio_nid(new_folio); 7223 7224 folio_set_hugetlb_temporary(old_folio); 7225 folio_clear_hugetlb_temporary(new_folio); 7226 7227 7228 /* 7229 * There is no need to transfer the per-node surplus state 7230 * when we do not cross the node. 7231 */ 7232 if (new_nid == old_nid) 7233 return; 7234 spin_lock_irq(&hugetlb_lock); 7235 if (h->surplus_huge_pages_node[old_nid]) { 7236 h->surplus_huge_pages_node[old_nid]--; 7237 h->surplus_huge_pages_node[new_nid]++; 7238 } 7239 spin_unlock_irq(&hugetlb_lock); 7240 } 7241 } 7242 7243 static void hugetlb_unshare_pmds(struct vm_area_struct *vma, 7244 unsigned long start, 7245 unsigned long end) 7246 { 7247 struct hstate *h = hstate_vma(vma); 7248 unsigned long sz = huge_page_size(h); 7249 struct mm_struct *mm = vma->vm_mm; 7250 struct mmu_notifier_range range; 7251 unsigned long address; 7252 spinlock_t *ptl; 7253 pte_t *ptep; 7254 7255 if (!(vma->vm_flags & VM_MAYSHARE)) 7256 return; 7257 7258 if (start >= end) 7259 return; 7260 7261 flush_cache_range(vma, start, end); 7262 /* 7263 * No need to call adjust_range_if_pmd_sharing_possible(), because 7264 * we have already done the PUD_SIZE alignment. 7265 */ 7266 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, mm, 7267 start, end); 7268 mmu_notifier_invalidate_range_start(&range); 7269 hugetlb_vma_lock_write(vma); 7270 i_mmap_lock_write(vma->vm_file->f_mapping); 7271 for (address = start; address < end; address += PUD_SIZE) { 7272 ptep = hugetlb_walk(vma, address, sz); 7273 if (!ptep) 7274 continue; 7275 ptl = huge_pte_lock(h, mm, ptep); 7276 huge_pmd_unshare(mm, vma, address, ptep); 7277 spin_unlock(ptl); 7278 } 7279 flush_hugetlb_tlb_range(vma, start, end); 7280 i_mmap_unlock_write(vma->vm_file->f_mapping); 7281 hugetlb_vma_unlock_write(vma); 7282 /* 7283 * No need to call mmu_notifier_arch_invalidate_secondary_tlbs(), see 7284 * Documentation/mm/mmu_notifier.rst. 7285 */ 7286 mmu_notifier_invalidate_range_end(&range); 7287 } 7288 7289 /* 7290 * This function will unconditionally remove all the shared pmd pgtable entries 7291 * within the specific vma for a hugetlbfs memory range. 7292 */ 7293 void hugetlb_unshare_all_pmds(struct vm_area_struct *vma) 7294 { 7295 hugetlb_unshare_pmds(vma, ALIGN(vma->vm_start, PUD_SIZE), 7296 ALIGN_DOWN(vma->vm_end, PUD_SIZE)); 7297 } 7298 7299 #ifdef CONFIG_CMA 7300 static bool cma_reserve_called __initdata; 7301 7302 static int __init cmdline_parse_hugetlb_cma(char *p) 7303 { 7304 int nid, count = 0; 7305 unsigned long tmp; 7306 char *s = p; 7307 7308 while (*s) { 7309 if (sscanf(s, "%lu%n", &tmp, &count) != 1) 7310 break; 7311 7312 if (s[count] == ':') { 7313 if (tmp >= MAX_NUMNODES) 7314 break; 7315 nid = array_index_nospec(tmp, MAX_NUMNODES); 7316 7317 s += count + 1; 7318 tmp = memparse(s, &s); 7319 hugetlb_cma_size_in_node[nid] = tmp; 7320 hugetlb_cma_size += tmp; 7321 7322 /* 7323 * Skip the separator if have one, otherwise 7324 * break the parsing. 7325 */ 7326 if (*s == ',') 7327 s++; 7328 else 7329 break; 7330 } else { 7331 hugetlb_cma_size = memparse(p, &p); 7332 break; 7333 } 7334 } 7335 7336 return 0; 7337 } 7338 7339 early_param("hugetlb_cma", cmdline_parse_hugetlb_cma); 7340 7341 void __init hugetlb_cma_reserve(int order) 7342 { 7343 unsigned long size, reserved, per_node; 7344 bool node_specific_cma_alloc = false; 7345 int nid; 7346 7347 cma_reserve_called = true; 7348 7349 if (!hugetlb_cma_size) 7350 return; 7351 7352 for (nid = 0; nid < MAX_NUMNODES; nid++) { 7353 if (hugetlb_cma_size_in_node[nid] == 0) 7354 continue; 7355 7356 if (!node_online(nid)) { 7357 pr_warn("hugetlb_cma: invalid node %d specified\n", nid); 7358 hugetlb_cma_size -= hugetlb_cma_size_in_node[nid]; 7359 hugetlb_cma_size_in_node[nid] = 0; 7360 continue; 7361 } 7362 7363 if (hugetlb_cma_size_in_node[nid] < (PAGE_SIZE << order)) { 7364 pr_warn("hugetlb_cma: cma area of node %d should be at least %lu MiB\n", 7365 nid, (PAGE_SIZE << order) / SZ_1M); 7366 hugetlb_cma_size -= hugetlb_cma_size_in_node[nid]; 7367 hugetlb_cma_size_in_node[nid] = 0; 7368 } else { 7369 node_specific_cma_alloc = true; 7370 } 7371 } 7372 7373 /* Validate the CMA size again in case some invalid nodes specified. */ 7374 if (!hugetlb_cma_size) 7375 return; 7376 7377 if (hugetlb_cma_size < (PAGE_SIZE << order)) { 7378 pr_warn("hugetlb_cma: cma area should be at least %lu MiB\n", 7379 (PAGE_SIZE << order) / SZ_1M); 7380 hugetlb_cma_size = 0; 7381 return; 7382 } 7383 7384 if (!node_specific_cma_alloc) { 7385 /* 7386 * If 3 GB area is requested on a machine with 4 numa nodes, 7387 * let's allocate 1 GB on first three nodes and ignore the last one. 7388 */ 7389 per_node = DIV_ROUND_UP(hugetlb_cma_size, nr_online_nodes); 7390 pr_info("hugetlb_cma: reserve %lu MiB, up to %lu MiB per node\n", 7391 hugetlb_cma_size / SZ_1M, per_node / SZ_1M); 7392 } 7393 7394 reserved = 0; 7395 for_each_online_node(nid) { 7396 int res; 7397 char name[CMA_MAX_NAME]; 7398 7399 if (node_specific_cma_alloc) { 7400 if (hugetlb_cma_size_in_node[nid] == 0) 7401 continue; 7402 7403 size = hugetlb_cma_size_in_node[nid]; 7404 } else { 7405 size = min(per_node, hugetlb_cma_size - reserved); 7406 } 7407 7408 size = round_up(size, PAGE_SIZE << order); 7409 7410 snprintf(name, sizeof(name), "hugetlb%d", nid); 7411 /* 7412 * Note that 'order per bit' is based on smallest size that 7413 * may be returned to CMA allocator in the case of 7414 * huge page demotion. 7415 */ 7416 res = cma_declare_contiguous_nid(0, size, 0, 7417 PAGE_SIZE << HUGETLB_PAGE_ORDER, 7418 0, false, name, 7419 &hugetlb_cma[nid], nid); 7420 if (res) { 7421 pr_warn("hugetlb_cma: reservation failed: err %d, node %d", 7422 res, nid); 7423 continue; 7424 } 7425 7426 reserved += size; 7427 pr_info("hugetlb_cma: reserved %lu MiB on node %d\n", 7428 size / SZ_1M, nid); 7429 7430 if (reserved >= hugetlb_cma_size) 7431 break; 7432 } 7433 7434 if (!reserved) 7435 /* 7436 * hugetlb_cma_size is used to determine if allocations from 7437 * cma are possible. Set to zero if no cma regions are set up. 7438 */ 7439 hugetlb_cma_size = 0; 7440 } 7441 7442 static void __init hugetlb_cma_check(void) 7443 { 7444 if (!hugetlb_cma_size || cma_reserve_called) 7445 return; 7446 7447 pr_warn("hugetlb_cma: the option isn't supported by current arch\n"); 7448 } 7449 7450 #endif /* CONFIG_CMA */ 7451