1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Generic hugetlb support. 4 * (C) Nadia Yvette Chambers, April 2004 5 */ 6 #include <linux/list.h> 7 #include <linux/init.h> 8 #include <linux/mm.h> 9 #include <linux/seq_file.h> 10 #include <linux/sysctl.h> 11 #include <linux/highmem.h> 12 #include <linux/mmu_notifier.h> 13 #include <linux/nodemask.h> 14 #include <linux/pagemap.h> 15 #include <linux/mempolicy.h> 16 #include <linux/compiler.h> 17 #include <linux/cpuset.h> 18 #include <linux/mutex.h> 19 #include <linux/memblock.h> 20 #include <linux/sysfs.h> 21 #include <linux/slab.h> 22 #include <linux/sched/mm.h> 23 #include <linux/mmdebug.h> 24 #include <linux/sched/signal.h> 25 #include <linux/rmap.h> 26 #include <linux/string_helpers.h> 27 #include <linux/swap.h> 28 #include <linux/swapops.h> 29 #include <linux/jhash.h> 30 #include <linux/numa.h> 31 #include <linux/llist.h> 32 #include <linux/cma.h> 33 #include <linux/migrate.h> 34 35 #include <asm/page.h> 36 #include <asm/pgalloc.h> 37 #include <asm/tlb.h> 38 39 #include <linux/io.h> 40 #include <linux/hugetlb.h> 41 #include <linux/hugetlb_cgroup.h> 42 #include <linux/node.h> 43 #include <linux/page_owner.h> 44 #include "internal.h" 45 #include "hugetlb_vmemmap.h" 46 47 int hugetlb_max_hstate __read_mostly; 48 unsigned int default_hstate_idx; 49 struct hstate hstates[HUGE_MAX_HSTATE]; 50 51 #ifdef CONFIG_CMA 52 static struct cma *hugetlb_cma[MAX_NUMNODES]; 53 #endif 54 static unsigned long hugetlb_cma_size __initdata; 55 56 /* 57 * Minimum page order among possible hugepage sizes, set to a proper value 58 * at boot time. 59 */ 60 static unsigned int minimum_order __read_mostly = UINT_MAX; 61 62 __initdata LIST_HEAD(huge_boot_pages); 63 64 /* for command line parsing */ 65 static struct hstate * __initdata parsed_hstate; 66 static unsigned long __initdata default_hstate_max_huge_pages; 67 static bool __initdata parsed_valid_hugepagesz = true; 68 static bool __initdata parsed_default_hugepagesz; 69 70 /* 71 * Protects updates to hugepage_freelists, hugepage_activelist, nr_huge_pages, 72 * free_huge_pages, and surplus_huge_pages. 73 */ 74 DEFINE_SPINLOCK(hugetlb_lock); 75 76 /* 77 * Serializes faults on the same logical page. This is used to 78 * prevent spurious OOMs when the hugepage pool is fully utilized. 79 */ 80 static int num_fault_mutexes; 81 struct mutex *hugetlb_fault_mutex_table ____cacheline_aligned_in_smp; 82 83 /* Forward declaration */ 84 static int hugetlb_acct_memory(struct hstate *h, long delta); 85 86 static inline bool subpool_is_free(struct hugepage_subpool *spool) 87 { 88 if (spool->count) 89 return false; 90 if (spool->max_hpages != -1) 91 return spool->used_hpages == 0; 92 if (spool->min_hpages != -1) 93 return spool->rsv_hpages == spool->min_hpages; 94 95 return true; 96 } 97 98 static inline void unlock_or_release_subpool(struct hugepage_subpool *spool, 99 unsigned long irq_flags) 100 { 101 spin_unlock_irqrestore(&spool->lock, irq_flags); 102 103 /* If no pages are used, and no other handles to the subpool 104 * remain, give up any reservations based on minimum size and 105 * free the subpool */ 106 if (subpool_is_free(spool)) { 107 if (spool->min_hpages != -1) 108 hugetlb_acct_memory(spool->hstate, 109 -spool->min_hpages); 110 kfree(spool); 111 } 112 } 113 114 struct hugepage_subpool *hugepage_new_subpool(struct hstate *h, long max_hpages, 115 long min_hpages) 116 { 117 struct hugepage_subpool *spool; 118 119 spool = kzalloc(sizeof(*spool), GFP_KERNEL); 120 if (!spool) 121 return NULL; 122 123 spin_lock_init(&spool->lock); 124 spool->count = 1; 125 spool->max_hpages = max_hpages; 126 spool->hstate = h; 127 spool->min_hpages = min_hpages; 128 129 if (min_hpages != -1 && hugetlb_acct_memory(h, min_hpages)) { 130 kfree(spool); 131 return NULL; 132 } 133 spool->rsv_hpages = min_hpages; 134 135 return spool; 136 } 137 138 void hugepage_put_subpool(struct hugepage_subpool *spool) 139 { 140 unsigned long flags; 141 142 spin_lock_irqsave(&spool->lock, flags); 143 BUG_ON(!spool->count); 144 spool->count--; 145 unlock_or_release_subpool(spool, flags); 146 } 147 148 /* 149 * Subpool accounting for allocating and reserving pages. 150 * Return -ENOMEM if there are not enough resources to satisfy the 151 * request. Otherwise, return the number of pages by which the 152 * global pools must be adjusted (upward). The returned value may 153 * only be different than the passed value (delta) in the case where 154 * a subpool minimum size must be maintained. 155 */ 156 static long hugepage_subpool_get_pages(struct hugepage_subpool *spool, 157 long delta) 158 { 159 long ret = delta; 160 161 if (!spool) 162 return ret; 163 164 spin_lock_irq(&spool->lock); 165 166 if (spool->max_hpages != -1) { /* maximum size accounting */ 167 if ((spool->used_hpages + delta) <= spool->max_hpages) 168 spool->used_hpages += delta; 169 else { 170 ret = -ENOMEM; 171 goto unlock_ret; 172 } 173 } 174 175 /* minimum size accounting */ 176 if (spool->min_hpages != -1 && spool->rsv_hpages) { 177 if (delta > spool->rsv_hpages) { 178 /* 179 * Asking for more reserves than those already taken on 180 * behalf of subpool. Return difference. 181 */ 182 ret = delta - spool->rsv_hpages; 183 spool->rsv_hpages = 0; 184 } else { 185 ret = 0; /* reserves already accounted for */ 186 spool->rsv_hpages -= delta; 187 } 188 } 189 190 unlock_ret: 191 spin_unlock_irq(&spool->lock); 192 return ret; 193 } 194 195 /* 196 * Subpool accounting for freeing and unreserving pages. 197 * Return the number of global page reservations that must be dropped. 198 * The return value may only be different than the passed value (delta) 199 * in the case where a subpool minimum size must be maintained. 200 */ 201 static long hugepage_subpool_put_pages(struct hugepage_subpool *spool, 202 long delta) 203 { 204 long ret = delta; 205 unsigned long flags; 206 207 if (!spool) 208 return delta; 209 210 spin_lock_irqsave(&spool->lock, flags); 211 212 if (spool->max_hpages != -1) /* maximum size accounting */ 213 spool->used_hpages -= delta; 214 215 /* minimum size accounting */ 216 if (spool->min_hpages != -1 && spool->used_hpages < spool->min_hpages) { 217 if (spool->rsv_hpages + delta <= spool->min_hpages) 218 ret = 0; 219 else 220 ret = spool->rsv_hpages + delta - spool->min_hpages; 221 222 spool->rsv_hpages += delta; 223 if (spool->rsv_hpages > spool->min_hpages) 224 spool->rsv_hpages = spool->min_hpages; 225 } 226 227 /* 228 * If hugetlbfs_put_super couldn't free spool due to an outstanding 229 * quota reference, free it now. 230 */ 231 unlock_or_release_subpool(spool, flags); 232 233 return ret; 234 } 235 236 static inline struct hugepage_subpool *subpool_inode(struct inode *inode) 237 { 238 return HUGETLBFS_SB(inode->i_sb)->spool; 239 } 240 241 static inline struct hugepage_subpool *subpool_vma(struct vm_area_struct *vma) 242 { 243 return subpool_inode(file_inode(vma->vm_file)); 244 } 245 246 /* Helper that removes a struct file_region from the resv_map cache and returns 247 * it for use. 248 */ 249 static struct file_region * 250 get_file_region_entry_from_cache(struct resv_map *resv, long from, long to) 251 { 252 struct file_region *nrg = NULL; 253 254 VM_BUG_ON(resv->region_cache_count <= 0); 255 256 resv->region_cache_count--; 257 nrg = list_first_entry(&resv->region_cache, struct file_region, link); 258 list_del(&nrg->link); 259 260 nrg->from = from; 261 nrg->to = to; 262 263 return nrg; 264 } 265 266 static void copy_hugetlb_cgroup_uncharge_info(struct file_region *nrg, 267 struct file_region *rg) 268 { 269 #ifdef CONFIG_CGROUP_HUGETLB 270 nrg->reservation_counter = rg->reservation_counter; 271 nrg->css = rg->css; 272 if (rg->css) 273 css_get(rg->css); 274 #endif 275 } 276 277 /* Helper that records hugetlb_cgroup uncharge info. */ 278 static void record_hugetlb_cgroup_uncharge_info(struct hugetlb_cgroup *h_cg, 279 struct hstate *h, 280 struct resv_map *resv, 281 struct file_region *nrg) 282 { 283 #ifdef CONFIG_CGROUP_HUGETLB 284 if (h_cg) { 285 nrg->reservation_counter = 286 &h_cg->rsvd_hugepage[hstate_index(h)]; 287 nrg->css = &h_cg->css; 288 /* 289 * The caller will hold exactly one h_cg->css reference for the 290 * whole contiguous reservation region. But this area might be 291 * scattered when there are already some file_regions reside in 292 * it. As a result, many file_regions may share only one css 293 * reference. In order to ensure that one file_region must hold 294 * exactly one h_cg->css reference, we should do css_get for 295 * each file_region and leave the reference held by caller 296 * untouched. 297 */ 298 css_get(&h_cg->css); 299 if (!resv->pages_per_hpage) 300 resv->pages_per_hpage = pages_per_huge_page(h); 301 /* pages_per_hpage should be the same for all entries in 302 * a resv_map. 303 */ 304 VM_BUG_ON(resv->pages_per_hpage != pages_per_huge_page(h)); 305 } else { 306 nrg->reservation_counter = NULL; 307 nrg->css = NULL; 308 } 309 #endif 310 } 311 312 static void put_uncharge_info(struct file_region *rg) 313 { 314 #ifdef CONFIG_CGROUP_HUGETLB 315 if (rg->css) 316 css_put(rg->css); 317 #endif 318 } 319 320 static bool has_same_uncharge_info(struct file_region *rg, 321 struct file_region *org) 322 { 323 #ifdef CONFIG_CGROUP_HUGETLB 324 return rg && org && 325 rg->reservation_counter == org->reservation_counter && 326 rg->css == org->css; 327 328 #else 329 return true; 330 #endif 331 } 332 333 static void coalesce_file_region(struct resv_map *resv, struct file_region *rg) 334 { 335 struct file_region *nrg = NULL, *prg = NULL; 336 337 prg = list_prev_entry(rg, link); 338 if (&prg->link != &resv->regions && prg->to == rg->from && 339 has_same_uncharge_info(prg, rg)) { 340 prg->to = rg->to; 341 342 list_del(&rg->link); 343 put_uncharge_info(rg); 344 kfree(rg); 345 346 rg = prg; 347 } 348 349 nrg = list_next_entry(rg, link); 350 if (&nrg->link != &resv->regions && nrg->from == rg->to && 351 has_same_uncharge_info(nrg, rg)) { 352 nrg->from = rg->from; 353 354 list_del(&rg->link); 355 put_uncharge_info(rg); 356 kfree(rg); 357 } 358 } 359 360 static inline long 361 hugetlb_resv_map_add(struct resv_map *map, struct file_region *rg, long from, 362 long to, struct hstate *h, struct hugetlb_cgroup *cg, 363 long *regions_needed) 364 { 365 struct file_region *nrg; 366 367 if (!regions_needed) { 368 nrg = get_file_region_entry_from_cache(map, from, to); 369 record_hugetlb_cgroup_uncharge_info(cg, h, map, nrg); 370 list_add(&nrg->link, rg->link.prev); 371 coalesce_file_region(map, nrg); 372 } else 373 *regions_needed += 1; 374 375 return to - from; 376 } 377 378 /* 379 * Must be called with resv->lock held. 380 * 381 * Calling this with regions_needed != NULL will count the number of pages 382 * to be added but will not modify the linked list. And regions_needed will 383 * indicate the number of file_regions needed in the cache to carry out to add 384 * the regions for this range. 385 */ 386 static long add_reservation_in_range(struct resv_map *resv, long f, long t, 387 struct hugetlb_cgroup *h_cg, 388 struct hstate *h, long *regions_needed) 389 { 390 long add = 0; 391 struct list_head *head = &resv->regions; 392 long last_accounted_offset = f; 393 struct file_region *rg = NULL, *trg = NULL; 394 395 if (regions_needed) 396 *regions_needed = 0; 397 398 /* In this loop, we essentially handle an entry for the range 399 * [last_accounted_offset, rg->from), at every iteration, with some 400 * bounds checking. 401 */ 402 list_for_each_entry_safe(rg, trg, head, link) { 403 /* Skip irrelevant regions that start before our range. */ 404 if (rg->from < f) { 405 /* If this region ends after the last accounted offset, 406 * then we need to update last_accounted_offset. 407 */ 408 if (rg->to > last_accounted_offset) 409 last_accounted_offset = rg->to; 410 continue; 411 } 412 413 /* When we find a region that starts beyond our range, we've 414 * finished. 415 */ 416 if (rg->from >= t) 417 break; 418 419 /* Add an entry for last_accounted_offset -> rg->from, and 420 * update last_accounted_offset. 421 */ 422 if (rg->from > last_accounted_offset) 423 add += hugetlb_resv_map_add(resv, rg, 424 last_accounted_offset, 425 rg->from, h, h_cg, 426 regions_needed); 427 428 last_accounted_offset = rg->to; 429 } 430 431 /* Handle the case where our range extends beyond 432 * last_accounted_offset. 433 */ 434 if (last_accounted_offset < t) 435 add += hugetlb_resv_map_add(resv, rg, last_accounted_offset, 436 t, h, h_cg, regions_needed); 437 438 VM_BUG_ON(add < 0); 439 return add; 440 } 441 442 /* Must be called with resv->lock acquired. Will drop lock to allocate entries. 443 */ 444 static int allocate_file_region_entries(struct resv_map *resv, 445 int regions_needed) 446 __must_hold(&resv->lock) 447 { 448 struct list_head allocated_regions; 449 int to_allocate = 0, i = 0; 450 struct file_region *trg = NULL, *rg = NULL; 451 452 VM_BUG_ON(regions_needed < 0); 453 454 INIT_LIST_HEAD(&allocated_regions); 455 456 /* 457 * Check for sufficient descriptors in the cache to accommodate 458 * the number of in progress add operations plus regions_needed. 459 * 460 * This is a while loop because when we drop the lock, some other call 461 * to region_add or region_del may have consumed some region_entries, 462 * so we keep looping here until we finally have enough entries for 463 * (adds_in_progress + regions_needed). 464 */ 465 while (resv->region_cache_count < 466 (resv->adds_in_progress + regions_needed)) { 467 to_allocate = resv->adds_in_progress + regions_needed - 468 resv->region_cache_count; 469 470 /* At this point, we should have enough entries in the cache 471 * for all the existing adds_in_progress. We should only be 472 * needing to allocate for regions_needed. 473 */ 474 VM_BUG_ON(resv->region_cache_count < resv->adds_in_progress); 475 476 spin_unlock(&resv->lock); 477 for (i = 0; i < to_allocate; i++) { 478 trg = kmalloc(sizeof(*trg), GFP_KERNEL); 479 if (!trg) 480 goto out_of_memory; 481 list_add(&trg->link, &allocated_regions); 482 } 483 484 spin_lock(&resv->lock); 485 486 list_splice(&allocated_regions, &resv->region_cache); 487 resv->region_cache_count += to_allocate; 488 } 489 490 return 0; 491 492 out_of_memory: 493 list_for_each_entry_safe(rg, trg, &allocated_regions, link) { 494 list_del(&rg->link); 495 kfree(rg); 496 } 497 return -ENOMEM; 498 } 499 500 /* 501 * Add the huge page range represented by [f, t) to the reserve 502 * map. Regions will be taken from the cache to fill in this range. 503 * Sufficient regions should exist in the cache due to the previous 504 * call to region_chg with the same range, but in some cases the cache will not 505 * have sufficient entries due to races with other code doing region_add or 506 * region_del. The extra needed entries will be allocated. 507 * 508 * regions_needed is the out value provided by a previous call to region_chg. 509 * 510 * Return the number of new huge pages added to the map. This number is greater 511 * than or equal to zero. If file_region entries needed to be allocated for 512 * this operation and we were not able to allocate, it returns -ENOMEM. 513 * region_add of regions of length 1 never allocate file_regions and cannot 514 * fail; region_chg will always allocate at least 1 entry and a region_add for 515 * 1 page will only require at most 1 entry. 516 */ 517 static long region_add(struct resv_map *resv, long f, long t, 518 long in_regions_needed, struct hstate *h, 519 struct hugetlb_cgroup *h_cg) 520 { 521 long add = 0, actual_regions_needed = 0; 522 523 spin_lock(&resv->lock); 524 retry: 525 526 /* Count how many regions are actually needed to execute this add. */ 527 add_reservation_in_range(resv, f, t, NULL, NULL, 528 &actual_regions_needed); 529 530 /* 531 * Check for sufficient descriptors in the cache to accommodate 532 * this add operation. Note that actual_regions_needed may be greater 533 * than in_regions_needed, as the resv_map may have been modified since 534 * the region_chg call. In this case, we need to make sure that we 535 * allocate extra entries, such that we have enough for all the 536 * existing adds_in_progress, plus the excess needed for this 537 * operation. 538 */ 539 if (actual_regions_needed > in_regions_needed && 540 resv->region_cache_count < 541 resv->adds_in_progress + 542 (actual_regions_needed - in_regions_needed)) { 543 /* region_add operation of range 1 should never need to 544 * allocate file_region entries. 545 */ 546 VM_BUG_ON(t - f <= 1); 547 548 if (allocate_file_region_entries( 549 resv, actual_regions_needed - in_regions_needed)) { 550 return -ENOMEM; 551 } 552 553 goto retry; 554 } 555 556 add = add_reservation_in_range(resv, f, t, h_cg, h, NULL); 557 558 resv->adds_in_progress -= in_regions_needed; 559 560 spin_unlock(&resv->lock); 561 return add; 562 } 563 564 /* 565 * Examine the existing reserve map and determine how many 566 * huge pages in the specified range [f, t) are NOT currently 567 * represented. This routine is called before a subsequent 568 * call to region_add that will actually modify the reserve 569 * map to add the specified range [f, t). region_chg does 570 * not change the number of huge pages represented by the 571 * map. A number of new file_region structures is added to the cache as a 572 * placeholder, for the subsequent region_add call to use. At least 1 573 * file_region structure is added. 574 * 575 * out_regions_needed is the number of regions added to the 576 * resv->adds_in_progress. This value needs to be provided to a follow up call 577 * to region_add or region_abort for proper accounting. 578 * 579 * Returns the number of huge pages that need to be added to the existing 580 * reservation map for the range [f, t). This number is greater or equal to 581 * zero. -ENOMEM is returned if a new file_region structure or cache entry 582 * is needed and can not be allocated. 583 */ 584 static long region_chg(struct resv_map *resv, long f, long t, 585 long *out_regions_needed) 586 { 587 long chg = 0; 588 589 spin_lock(&resv->lock); 590 591 /* Count how many hugepages in this range are NOT represented. */ 592 chg = add_reservation_in_range(resv, f, t, NULL, NULL, 593 out_regions_needed); 594 595 if (*out_regions_needed == 0) 596 *out_regions_needed = 1; 597 598 if (allocate_file_region_entries(resv, *out_regions_needed)) 599 return -ENOMEM; 600 601 resv->adds_in_progress += *out_regions_needed; 602 603 spin_unlock(&resv->lock); 604 return chg; 605 } 606 607 /* 608 * Abort the in progress add operation. The adds_in_progress field 609 * of the resv_map keeps track of the operations in progress between 610 * calls to region_chg and region_add. Operations are sometimes 611 * aborted after the call to region_chg. In such cases, region_abort 612 * is called to decrement the adds_in_progress counter. regions_needed 613 * is the value returned by the region_chg call, it is used to decrement 614 * the adds_in_progress counter. 615 * 616 * NOTE: The range arguments [f, t) are not needed or used in this 617 * routine. They are kept to make reading the calling code easier as 618 * arguments will match the associated region_chg call. 619 */ 620 static void region_abort(struct resv_map *resv, long f, long t, 621 long regions_needed) 622 { 623 spin_lock(&resv->lock); 624 VM_BUG_ON(!resv->region_cache_count); 625 resv->adds_in_progress -= regions_needed; 626 spin_unlock(&resv->lock); 627 } 628 629 /* 630 * Delete the specified range [f, t) from the reserve map. If the 631 * t parameter is LONG_MAX, this indicates that ALL regions after f 632 * should be deleted. Locate the regions which intersect [f, t) 633 * and either trim, delete or split the existing regions. 634 * 635 * Returns the number of huge pages deleted from the reserve map. 636 * In the normal case, the return value is zero or more. In the 637 * case where a region must be split, a new region descriptor must 638 * be allocated. If the allocation fails, -ENOMEM will be returned. 639 * NOTE: If the parameter t == LONG_MAX, then we will never split 640 * a region and possibly return -ENOMEM. Callers specifying 641 * t == LONG_MAX do not need to check for -ENOMEM error. 642 */ 643 static long region_del(struct resv_map *resv, long f, long t) 644 { 645 struct list_head *head = &resv->regions; 646 struct file_region *rg, *trg; 647 struct file_region *nrg = NULL; 648 long del = 0; 649 650 retry: 651 spin_lock(&resv->lock); 652 list_for_each_entry_safe(rg, trg, head, link) { 653 /* 654 * Skip regions before the range to be deleted. file_region 655 * ranges are normally of the form [from, to). However, there 656 * may be a "placeholder" entry in the map which is of the form 657 * (from, to) with from == to. Check for placeholder entries 658 * at the beginning of the range to be deleted. 659 */ 660 if (rg->to <= f && (rg->to != rg->from || rg->to != f)) 661 continue; 662 663 if (rg->from >= t) 664 break; 665 666 if (f > rg->from && t < rg->to) { /* Must split region */ 667 /* 668 * Check for an entry in the cache before dropping 669 * lock and attempting allocation. 670 */ 671 if (!nrg && 672 resv->region_cache_count > resv->adds_in_progress) { 673 nrg = list_first_entry(&resv->region_cache, 674 struct file_region, 675 link); 676 list_del(&nrg->link); 677 resv->region_cache_count--; 678 } 679 680 if (!nrg) { 681 spin_unlock(&resv->lock); 682 nrg = kmalloc(sizeof(*nrg), GFP_KERNEL); 683 if (!nrg) 684 return -ENOMEM; 685 goto retry; 686 } 687 688 del += t - f; 689 hugetlb_cgroup_uncharge_file_region( 690 resv, rg, t - f, false); 691 692 /* New entry for end of split region */ 693 nrg->from = t; 694 nrg->to = rg->to; 695 696 copy_hugetlb_cgroup_uncharge_info(nrg, rg); 697 698 INIT_LIST_HEAD(&nrg->link); 699 700 /* Original entry is trimmed */ 701 rg->to = f; 702 703 list_add(&nrg->link, &rg->link); 704 nrg = NULL; 705 break; 706 } 707 708 if (f <= rg->from && t >= rg->to) { /* Remove entire region */ 709 del += rg->to - rg->from; 710 hugetlb_cgroup_uncharge_file_region(resv, rg, 711 rg->to - rg->from, true); 712 list_del(&rg->link); 713 kfree(rg); 714 continue; 715 } 716 717 if (f <= rg->from) { /* Trim beginning of region */ 718 hugetlb_cgroup_uncharge_file_region(resv, rg, 719 t - rg->from, false); 720 721 del += t - rg->from; 722 rg->from = t; 723 } else { /* Trim end of region */ 724 hugetlb_cgroup_uncharge_file_region(resv, rg, 725 rg->to - f, false); 726 727 del += rg->to - f; 728 rg->to = f; 729 } 730 } 731 732 spin_unlock(&resv->lock); 733 kfree(nrg); 734 return del; 735 } 736 737 /* 738 * A rare out of memory error was encountered which prevented removal of 739 * the reserve map region for a page. The huge page itself was free'ed 740 * and removed from the page cache. This routine will adjust the subpool 741 * usage count, and the global reserve count if needed. By incrementing 742 * these counts, the reserve map entry which could not be deleted will 743 * appear as a "reserved" entry instead of simply dangling with incorrect 744 * counts. 745 */ 746 void hugetlb_fix_reserve_counts(struct inode *inode) 747 { 748 struct hugepage_subpool *spool = subpool_inode(inode); 749 long rsv_adjust; 750 bool reserved = false; 751 752 rsv_adjust = hugepage_subpool_get_pages(spool, 1); 753 if (rsv_adjust > 0) { 754 struct hstate *h = hstate_inode(inode); 755 756 if (!hugetlb_acct_memory(h, 1)) 757 reserved = true; 758 } else if (!rsv_adjust) { 759 reserved = true; 760 } 761 762 if (!reserved) 763 pr_warn("hugetlb: Huge Page Reserved count may go negative.\n"); 764 } 765 766 /* 767 * Count and return the number of huge pages in the reserve map 768 * that intersect with the range [f, t). 769 */ 770 static long region_count(struct resv_map *resv, long f, long t) 771 { 772 struct list_head *head = &resv->regions; 773 struct file_region *rg; 774 long chg = 0; 775 776 spin_lock(&resv->lock); 777 /* Locate each segment we overlap with, and count that overlap. */ 778 list_for_each_entry(rg, head, link) { 779 long seg_from; 780 long seg_to; 781 782 if (rg->to <= f) 783 continue; 784 if (rg->from >= t) 785 break; 786 787 seg_from = max(rg->from, f); 788 seg_to = min(rg->to, t); 789 790 chg += seg_to - seg_from; 791 } 792 spin_unlock(&resv->lock); 793 794 return chg; 795 } 796 797 /* 798 * Convert the address within this vma to the page offset within 799 * the mapping, in pagecache page units; huge pages here. 800 */ 801 static pgoff_t vma_hugecache_offset(struct hstate *h, 802 struct vm_area_struct *vma, unsigned long address) 803 { 804 return ((address - vma->vm_start) >> huge_page_shift(h)) + 805 (vma->vm_pgoff >> huge_page_order(h)); 806 } 807 808 pgoff_t linear_hugepage_index(struct vm_area_struct *vma, 809 unsigned long address) 810 { 811 return vma_hugecache_offset(hstate_vma(vma), vma, address); 812 } 813 EXPORT_SYMBOL_GPL(linear_hugepage_index); 814 815 /* 816 * Return the size of the pages allocated when backing a VMA. In the majority 817 * cases this will be same size as used by the page table entries. 818 */ 819 unsigned long vma_kernel_pagesize(struct vm_area_struct *vma) 820 { 821 if (vma->vm_ops && vma->vm_ops->pagesize) 822 return vma->vm_ops->pagesize(vma); 823 return PAGE_SIZE; 824 } 825 EXPORT_SYMBOL_GPL(vma_kernel_pagesize); 826 827 /* 828 * Return the page size being used by the MMU to back a VMA. In the majority 829 * of cases, the page size used by the kernel matches the MMU size. On 830 * architectures where it differs, an architecture-specific 'strong' 831 * version of this symbol is required. 832 */ 833 __weak unsigned long vma_mmu_pagesize(struct vm_area_struct *vma) 834 { 835 return vma_kernel_pagesize(vma); 836 } 837 838 /* 839 * Flags for MAP_PRIVATE reservations. These are stored in the bottom 840 * bits of the reservation map pointer, which are always clear due to 841 * alignment. 842 */ 843 #define HPAGE_RESV_OWNER (1UL << 0) 844 #define HPAGE_RESV_UNMAPPED (1UL << 1) 845 #define HPAGE_RESV_MASK (HPAGE_RESV_OWNER | HPAGE_RESV_UNMAPPED) 846 847 /* 848 * These helpers are used to track how many pages are reserved for 849 * faults in a MAP_PRIVATE mapping. Only the process that called mmap() 850 * is guaranteed to have their future faults succeed. 851 * 852 * With the exception of reset_vma_resv_huge_pages() which is called at fork(), 853 * the reserve counters are updated with the hugetlb_lock held. It is safe 854 * to reset the VMA at fork() time as it is not in use yet and there is no 855 * chance of the global counters getting corrupted as a result of the values. 856 * 857 * The private mapping reservation is represented in a subtly different 858 * manner to a shared mapping. A shared mapping has a region map associated 859 * with the underlying file, this region map represents the backing file 860 * pages which have ever had a reservation assigned which this persists even 861 * after the page is instantiated. A private mapping has a region map 862 * associated with the original mmap which is attached to all VMAs which 863 * reference it, this region map represents those offsets which have consumed 864 * reservation ie. where pages have been instantiated. 865 */ 866 static unsigned long get_vma_private_data(struct vm_area_struct *vma) 867 { 868 return (unsigned long)vma->vm_private_data; 869 } 870 871 static void set_vma_private_data(struct vm_area_struct *vma, 872 unsigned long value) 873 { 874 vma->vm_private_data = (void *)value; 875 } 876 877 static void 878 resv_map_set_hugetlb_cgroup_uncharge_info(struct resv_map *resv_map, 879 struct hugetlb_cgroup *h_cg, 880 struct hstate *h) 881 { 882 #ifdef CONFIG_CGROUP_HUGETLB 883 if (!h_cg || !h) { 884 resv_map->reservation_counter = NULL; 885 resv_map->pages_per_hpage = 0; 886 resv_map->css = NULL; 887 } else { 888 resv_map->reservation_counter = 889 &h_cg->rsvd_hugepage[hstate_index(h)]; 890 resv_map->pages_per_hpage = pages_per_huge_page(h); 891 resv_map->css = &h_cg->css; 892 } 893 #endif 894 } 895 896 struct resv_map *resv_map_alloc(void) 897 { 898 struct resv_map *resv_map = kmalloc(sizeof(*resv_map), GFP_KERNEL); 899 struct file_region *rg = kmalloc(sizeof(*rg), GFP_KERNEL); 900 901 if (!resv_map || !rg) { 902 kfree(resv_map); 903 kfree(rg); 904 return NULL; 905 } 906 907 kref_init(&resv_map->refs); 908 spin_lock_init(&resv_map->lock); 909 INIT_LIST_HEAD(&resv_map->regions); 910 911 resv_map->adds_in_progress = 0; 912 /* 913 * Initialize these to 0. On shared mappings, 0's here indicate these 914 * fields don't do cgroup accounting. On private mappings, these will be 915 * re-initialized to the proper values, to indicate that hugetlb cgroup 916 * reservations are to be un-charged from here. 917 */ 918 resv_map_set_hugetlb_cgroup_uncharge_info(resv_map, NULL, NULL); 919 920 INIT_LIST_HEAD(&resv_map->region_cache); 921 list_add(&rg->link, &resv_map->region_cache); 922 resv_map->region_cache_count = 1; 923 924 return resv_map; 925 } 926 927 void resv_map_release(struct kref *ref) 928 { 929 struct resv_map *resv_map = container_of(ref, struct resv_map, refs); 930 struct list_head *head = &resv_map->region_cache; 931 struct file_region *rg, *trg; 932 933 /* Clear out any active regions before we release the map. */ 934 region_del(resv_map, 0, LONG_MAX); 935 936 /* ... and any entries left in the cache */ 937 list_for_each_entry_safe(rg, trg, head, link) { 938 list_del(&rg->link); 939 kfree(rg); 940 } 941 942 VM_BUG_ON(resv_map->adds_in_progress); 943 944 kfree(resv_map); 945 } 946 947 static inline struct resv_map *inode_resv_map(struct inode *inode) 948 { 949 /* 950 * At inode evict time, i_mapping may not point to the original 951 * address space within the inode. This original address space 952 * contains the pointer to the resv_map. So, always use the 953 * address space embedded within the inode. 954 * The VERY common case is inode->mapping == &inode->i_data but, 955 * this may not be true for device special inodes. 956 */ 957 return (struct resv_map *)(&inode->i_data)->private_data; 958 } 959 960 static struct resv_map *vma_resv_map(struct vm_area_struct *vma) 961 { 962 VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma); 963 if (vma->vm_flags & VM_MAYSHARE) { 964 struct address_space *mapping = vma->vm_file->f_mapping; 965 struct inode *inode = mapping->host; 966 967 return inode_resv_map(inode); 968 969 } else { 970 return (struct resv_map *)(get_vma_private_data(vma) & 971 ~HPAGE_RESV_MASK); 972 } 973 } 974 975 static void set_vma_resv_map(struct vm_area_struct *vma, struct resv_map *map) 976 { 977 VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma); 978 VM_BUG_ON_VMA(vma->vm_flags & VM_MAYSHARE, vma); 979 980 set_vma_private_data(vma, (get_vma_private_data(vma) & 981 HPAGE_RESV_MASK) | (unsigned long)map); 982 } 983 984 static void set_vma_resv_flags(struct vm_area_struct *vma, unsigned long flags) 985 { 986 VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma); 987 VM_BUG_ON_VMA(vma->vm_flags & VM_MAYSHARE, vma); 988 989 set_vma_private_data(vma, get_vma_private_data(vma) | flags); 990 } 991 992 static int is_vma_resv_set(struct vm_area_struct *vma, unsigned long flag) 993 { 994 VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma); 995 996 return (get_vma_private_data(vma) & flag) != 0; 997 } 998 999 /* Reset counters to 0 and clear all HPAGE_RESV_* flags */ 1000 void reset_vma_resv_huge_pages(struct vm_area_struct *vma) 1001 { 1002 VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma); 1003 if (!(vma->vm_flags & VM_MAYSHARE)) 1004 vma->vm_private_data = (void *)0; 1005 } 1006 1007 /* Returns true if the VMA has associated reserve pages */ 1008 static bool vma_has_reserves(struct vm_area_struct *vma, long chg) 1009 { 1010 if (vma->vm_flags & VM_NORESERVE) { 1011 /* 1012 * This address is already reserved by other process(chg == 0), 1013 * so, we should decrement reserved count. Without decrementing, 1014 * reserve count remains after releasing inode, because this 1015 * allocated page will go into page cache and is regarded as 1016 * coming from reserved pool in releasing step. Currently, we 1017 * don't have any other solution to deal with this situation 1018 * properly, so add work-around here. 1019 */ 1020 if (vma->vm_flags & VM_MAYSHARE && chg == 0) 1021 return true; 1022 else 1023 return false; 1024 } 1025 1026 /* Shared mappings always use reserves */ 1027 if (vma->vm_flags & VM_MAYSHARE) { 1028 /* 1029 * We know VM_NORESERVE is not set. Therefore, there SHOULD 1030 * be a region map for all pages. The only situation where 1031 * there is no region map is if a hole was punched via 1032 * fallocate. In this case, there really are no reserves to 1033 * use. This situation is indicated if chg != 0. 1034 */ 1035 if (chg) 1036 return false; 1037 else 1038 return true; 1039 } 1040 1041 /* 1042 * Only the process that called mmap() has reserves for 1043 * private mappings. 1044 */ 1045 if (is_vma_resv_set(vma, HPAGE_RESV_OWNER)) { 1046 /* 1047 * Like the shared case above, a hole punch or truncate 1048 * could have been performed on the private mapping. 1049 * Examine the value of chg to determine if reserves 1050 * actually exist or were previously consumed. 1051 * Very Subtle - The value of chg comes from a previous 1052 * call to vma_needs_reserves(). The reserve map for 1053 * private mappings has different (opposite) semantics 1054 * than that of shared mappings. vma_needs_reserves() 1055 * has already taken this difference in semantics into 1056 * account. Therefore, the meaning of chg is the same 1057 * as in the shared case above. Code could easily be 1058 * combined, but keeping it separate draws attention to 1059 * subtle differences. 1060 */ 1061 if (chg) 1062 return false; 1063 else 1064 return true; 1065 } 1066 1067 return false; 1068 } 1069 1070 static void enqueue_huge_page(struct hstate *h, struct page *page) 1071 { 1072 int nid = page_to_nid(page); 1073 1074 lockdep_assert_held(&hugetlb_lock); 1075 list_move(&page->lru, &h->hugepage_freelists[nid]); 1076 h->free_huge_pages++; 1077 h->free_huge_pages_node[nid]++; 1078 SetHPageFreed(page); 1079 } 1080 1081 static struct page *dequeue_huge_page_node_exact(struct hstate *h, int nid) 1082 { 1083 struct page *page; 1084 bool pin = !!(current->flags & PF_MEMALLOC_PIN); 1085 1086 lockdep_assert_held(&hugetlb_lock); 1087 list_for_each_entry(page, &h->hugepage_freelists[nid], lru) { 1088 if (pin && !is_pinnable_page(page)) 1089 continue; 1090 1091 if (PageHWPoison(page)) 1092 continue; 1093 1094 list_move(&page->lru, &h->hugepage_activelist); 1095 set_page_refcounted(page); 1096 ClearHPageFreed(page); 1097 h->free_huge_pages--; 1098 h->free_huge_pages_node[nid]--; 1099 return page; 1100 } 1101 1102 return NULL; 1103 } 1104 1105 static struct page *dequeue_huge_page_nodemask(struct hstate *h, gfp_t gfp_mask, int nid, 1106 nodemask_t *nmask) 1107 { 1108 unsigned int cpuset_mems_cookie; 1109 struct zonelist *zonelist; 1110 struct zone *zone; 1111 struct zoneref *z; 1112 int node = NUMA_NO_NODE; 1113 1114 zonelist = node_zonelist(nid, gfp_mask); 1115 1116 retry_cpuset: 1117 cpuset_mems_cookie = read_mems_allowed_begin(); 1118 for_each_zone_zonelist_nodemask(zone, z, zonelist, gfp_zone(gfp_mask), nmask) { 1119 struct page *page; 1120 1121 if (!cpuset_zone_allowed(zone, gfp_mask)) 1122 continue; 1123 /* 1124 * no need to ask again on the same node. Pool is node rather than 1125 * zone aware 1126 */ 1127 if (zone_to_nid(zone) == node) 1128 continue; 1129 node = zone_to_nid(zone); 1130 1131 page = dequeue_huge_page_node_exact(h, node); 1132 if (page) 1133 return page; 1134 } 1135 if (unlikely(read_mems_allowed_retry(cpuset_mems_cookie))) 1136 goto retry_cpuset; 1137 1138 return NULL; 1139 } 1140 1141 static struct page *dequeue_huge_page_vma(struct hstate *h, 1142 struct vm_area_struct *vma, 1143 unsigned long address, int avoid_reserve, 1144 long chg) 1145 { 1146 struct page *page; 1147 struct mempolicy *mpol; 1148 gfp_t gfp_mask; 1149 nodemask_t *nodemask; 1150 int nid; 1151 1152 /* 1153 * A child process with MAP_PRIVATE mappings created by their parent 1154 * have no page reserves. This check ensures that reservations are 1155 * not "stolen". The child may still get SIGKILLed 1156 */ 1157 if (!vma_has_reserves(vma, chg) && 1158 h->free_huge_pages - h->resv_huge_pages == 0) 1159 goto err; 1160 1161 /* If reserves cannot be used, ensure enough pages are in the pool */ 1162 if (avoid_reserve && h->free_huge_pages - h->resv_huge_pages == 0) 1163 goto err; 1164 1165 gfp_mask = htlb_alloc_mask(h); 1166 nid = huge_node(vma, address, gfp_mask, &mpol, &nodemask); 1167 page = dequeue_huge_page_nodemask(h, gfp_mask, nid, nodemask); 1168 if (page && !avoid_reserve && vma_has_reserves(vma, chg)) { 1169 SetHPageRestoreReserve(page); 1170 h->resv_huge_pages--; 1171 } 1172 1173 mpol_cond_put(mpol); 1174 return page; 1175 1176 err: 1177 return NULL; 1178 } 1179 1180 /* 1181 * common helper functions for hstate_next_node_to_{alloc|free}. 1182 * We may have allocated or freed a huge page based on a different 1183 * nodes_allowed previously, so h->next_node_to_{alloc|free} might 1184 * be outside of *nodes_allowed. Ensure that we use an allowed 1185 * node for alloc or free. 1186 */ 1187 static int next_node_allowed(int nid, nodemask_t *nodes_allowed) 1188 { 1189 nid = next_node_in(nid, *nodes_allowed); 1190 VM_BUG_ON(nid >= MAX_NUMNODES); 1191 1192 return nid; 1193 } 1194 1195 static int get_valid_node_allowed(int nid, nodemask_t *nodes_allowed) 1196 { 1197 if (!node_isset(nid, *nodes_allowed)) 1198 nid = next_node_allowed(nid, nodes_allowed); 1199 return nid; 1200 } 1201 1202 /* 1203 * returns the previously saved node ["this node"] from which to 1204 * allocate a persistent huge page for the pool and advance the 1205 * next node from which to allocate, handling wrap at end of node 1206 * mask. 1207 */ 1208 static int hstate_next_node_to_alloc(struct hstate *h, 1209 nodemask_t *nodes_allowed) 1210 { 1211 int nid; 1212 1213 VM_BUG_ON(!nodes_allowed); 1214 1215 nid = get_valid_node_allowed(h->next_nid_to_alloc, nodes_allowed); 1216 h->next_nid_to_alloc = next_node_allowed(nid, nodes_allowed); 1217 1218 return nid; 1219 } 1220 1221 /* 1222 * helper for remove_pool_huge_page() - return the previously saved 1223 * node ["this node"] from which to free a huge page. Advance the 1224 * next node id whether or not we find a free huge page to free so 1225 * that the next attempt to free addresses the next node. 1226 */ 1227 static int hstate_next_node_to_free(struct hstate *h, nodemask_t *nodes_allowed) 1228 { 1229 int nid; 1230 1231 VM_BUG_ON(!nodes_allowed); 1232 1233 nid = get_valid_node_allowed(h->next_nid_to_free, nodes_allowed); 1234 h->next_nid_to_free = next_node_allowed(nid, nodes_allowed); 1235 1236 return nid; 1237 } 1238 1239 #define for_each_node_mask_to_alloc(hs, nr_nodes, node, mask) \ 1240 for (nr_nodes = nodes_weight(*mask); \ 1241 nr_nodes > 0 && \ 1242 ((node = hstate_next_node_to_alloc(hs, mask)) || 1); \ 1243 nr_nodes--) 1244 1245 #define for_each_node_mask_to_free(hs, nr_nodes, node, mask) \ 1246 for (nr_nodes = nodes_weight(*mask); \ 1247 nr_nodes > 0 && \ 1248 ((node = hstate_next_node_to_free(hs, mask)) || 1); \ 1249 nr_nodes--) 1250 1251 #ifdef CONFIG_ARCH_HAS_GIGANTIC_PAGE 1252 static void destroy_compound_gigantic_page(struct page *page, 1253 unsigned int order) 1254 { 1255 int i; 1256 int nr_pages = 1 << order; 1257 struct page *p = page + 1; 1258 1259 atomic_set(compound_mapcount_ptr(page), 0); 1260 atomic_set(compound_pincount_ptr(page), 0); 1261 1262 for (i = 1; i < nr_pages; i++, p = mem_map_next(p, page, i)) { 1263 clear_compound_head(p); 1264 set_page_refcounted(p); 1265 } 1266 1267 set_compound_order(page, 0); 1268 page[1].compound_nr = 0; 1269 __ClearPageHead(page); 1270 } 1271 1272 static void free_gigantic_page(struct page *page, unsigned int order) 1273 { 1274 /* 1275 * If the page isn't allocated using the cma allocator, 1276 * cma_release() returns false. 1277 */ 1278 #ifdef CONFIG_CMA 1279 if (cma_release(hugetlb_cma[page_to_nid(page)], page, 1 << order)) 1280 return; 1281 #endif 1282 1283 free_contig_range(page_to_pfn(page), 1 << order); 1284 } 1285 1286 #ifdef CONFIG_CONTIG_ALLOC 1287 static struct page *alloc_gigantic_page(struct hstate *h, gfp_t gfp_mask, 1288 int nid, nodemask_t *nodemask) 1289 { 1290 unsigned long nr_pages = pages_per_huge_page(h); 1291 if (nid == NUMA_NO_NODE) 1292 nid = numa_mem_id(); 1293 1294 #ifdef CONFIG_CMA 1295 { 1296 struct page *page; 1297 int node; 1298 1299 if (hugetlb_cma[nid]) { 1300 page = cma_alloc(hugetlb_cma[nid], nr_pages, 1301 huge_page_order(h), true); 1302 if (page) 1303 return page; 1304 } 1305 1306 if (!(gfp_mask & __GFP_THISNODE)) { 1307 for_each_node_mask(node, *nodemask) { 1308 if (node == nid || !hugetlb_cma[node]) 1309 continue; 1310 1311 page = cma_alloc(hugetlb_cma[node], nr_pages, 1312 huge_page_order(h), true); 1313 if (page) 1314 return page; 1315 } 1316 } 1317 } 1318 #endif 1319 1320 return alloc_contig_pages(nr_pages, gfp_mask, nid, nodemask); 1321 } 1322 1323 #else /* !CONFIG_CONTIG_ALLOC */ 1324 static struct page *alloc_gigantic_page(struct hstate *h, gfp_t gfp_mask, 1325 int nid, nodemask_t *nodemask) 1326 { 1327 return NULL; 1328 } 1329 #endif /* CONFIG_CONTIG_ALLOC */ 1330 1331 #else /* !CONFIG_ARCH_HAS_GIGANTIC_PAGE */ 1332 static struct page *alloc_gigantic_page(struct hstate *h, gfp_t gfp_mask, 1333 int nid, nodemask_t *nodemask) 1334 { 1335 return NULL; 1336 } 1337 static inline void free_gigantic_page(struct page *page, unsigned int order) { } 1338 static inline void destroy_compound_gigantic_page(struct page *page, 1339 unsigned int order) { } 1340 #endif 1341 1342 /* 1343 * Remove hugetlb page from lists, and update dtor so that page appears 1344 * as just a compound page. A reference is held on the page. 1345 * 1346 * Must be called with hugetlb lock held. 1347 */ 1348 static void remove_hugetlb_page(struct hstate *h, struct page *page, 1349 bool adjust_surplus) 1350 { 1351 int nid = page_to_nid(page); 1352 1353 VM_BUG_ON_PAGE(hugetlb_cgroup_from_page(page), page); 1354 VM_BUG_ON_PAGE(hugetlb_cgroup_from_page_rsvd(page), page); 1355 1356 lockdep_assert_held(&hugetlb_lock); 1357 if (hstate_is_gigantic(h) && !gigantic_page_runtime_supported()) 1358 return; 1359 1360 list_del(&page->lru); 1361 1362 if (HPageFreed(page)) { 1363 h->free_huge_pages--; 1364 h->free_huge_pages_node[nid]--; 1365 } 1366 if (adjust_surplus) { 1367 h->surplus_huge_pages--; 1368 h->surplus_huge_pages_node[nid]--; 1369 } 1370 1371 set_page_refcounted(page); 1372 set_compound_page_dtor(page, NULL_COMPOUND_DTOR); 1373 1374 h->nr_huge_pages--; 1375 h->nr_huge_pages_node[nid]--; 1376 } 1377 1378 static void add_hugetlb_page(struct hstate *h, struct page *page, 1379 bool adjust_surplus) 1380 { 1381 int zeroed; 1382 int nid = page_to_nid(page); 1383 1384 VM_BUG_ON_PAGE(!HPageVmemmapOptimized(page), page); 1385 1386 lockdep_assert_held(&hugetlb_lock); 1387 1388 INIT_LIST_HEAD(&page->lru); 1389 h->nr_huge_pages++; 1390 h->nr_huge_pages_node[nid]++; 1391 1392 if (adjust_surplus) { 1393 h->surplus_huge_pages++; 1394 h->surplus_huge_pages_node[nid]++; 1395 } 1396 1397 set_compound_page_dtor(page, HUGETLB_PAGE_DTOR); 1398 set_page_private(page, 0); 1399 SetHPageVmemmapOptimized(page); 1400 1401 /* 1402 * This page is now managed by the hugetlb allocator and has 1403 * no users -- drop the last reference. 1404 */ 1405 zeroed = put_page_testzero(page); 1406 VM_BUG_ON_PAGE(!zeroed, page); 1407 arch_clear_hugepage_flags(page); 1408 enqueue_huge_page(h, page); 1409 } 1410 1411 static void __update_and_free_page(struct hstate *h, struct page *page) 1412 { 1413 int i; 1414 struct page *subpage = page; 1415 1416 if (hstate_is_gigantic(h) && !gigantic_page_runtime_supported()) 1417 return; 1418 1419 if (alloc_huge_page_vmemmap(h, page)) { 1420 spin_lock_irq(&hugetlb_lock); 1421 /* 1422 * If we cannot allocate vmemmap pages, just refuse to free the 1423 * page and put the page back on the hugetlb free list and treat 1424 * as a surplus page. 1425 */ 1426 add_hugetlb_page(h, page, true); 1427 spin_unlock_irq(&hugetlb_lock); 1428 return; 1429 } 1430 1431 for (i = 0; i < pages_per_huge_page(h); 1432 i++, subpage = mem_map_next(subpage, page, i)) { 1433 subpage->flags &= ~(1 << PG_locked | 1 << PG_error | 1434 1 << PG_referenced | 1 << PG_dirty | 1435 1 << PG_active | 1 << PG_private | 1436 1 << PG_writeback); 1437 } 1438 if (hstate_is_gigantic(h)) { 1439 destroy_compound_gigantic_page(page, huge_page_order(h)); 1440 free_gigantic_page(page, huge_page_order(h)); 1441 } else { 1442 __free_pages(page, huge_page_order(h)); 1443 } 1444 } 1445 1446 /* 1447 * As update_and_free_page() can be called under any context, so we cannot 1448 * use GFP_KERNEL to allocate vmemmap pages. However, we can defer the 1449 * actual freeing in a workqueue to prevent from using GFP_ATOMIC to allocate 1450 * the vmemmap pages. 1451 * 1452 * free_hpage_workfn() locklessly retrieves the linked list of pages to be 1453 * freed and frees them one-by-one. As the page->mapping pointer is going 1454 * to be cleared in free_hpage_workfn() anyway, it is reused as the llist_node 1455 * structure of a lockless linked list of huge pages to be freed. 1456 */ 1457 static LLIST_HEAD(hpage_freelist); 1458 1459 static void free_hpage_workfn(struct work_struct *work) 1460 { 1461 struct llist_node *node; 1462 1463 node = llist_del_all(&hpage_freelist); 1464 1465 while (node) { 1466 struct page *page; 1467 struct hstate *h; 1468 1469 page = container_of((struct address_space **)node, 1470 struct page, mapping); 1471 node = node->next; 1472 page->mapping = NULL; 1473 /* 1474 * The VM_BUG_ON_PAGE(!PageHuge(page), page) in page_hstate() 1475 * is going to trigger because a previous call to 1476 * remove_hugetlb_page() will set_compound_page_dtor(page, 1477 * NULL_COMPOUND_DTOR), so do not use page_hstate() directly. 1478 */ 1479 h = size_to_hstate(page_size(page)); 1480 1481 __update_and_free_page(h, page); 1482 1483 cond_resched(); 1484 } 1485 } 1486 static DECLARE_WORK(free_hpage_work, free_hpage_workfn); 1487 1488 static inline void flush_free_hpage_work(struct hstate *h) 1489 { 1490 if (free_vmemmap_pages_per_hpage(h)) 1491 flush_work(&free_hpage_work); 1492 } 1493 1494 static void update_and_free_page(struct hstate *h, struct page *page, 1495 bool atomic) 1496 { 1497 if (!HPageVmemmapOptimized(page) || !atomic) { 1498 __update_and_free_page(h, page); 1499 return; 1500 } 1501 1502 /* 1503 * Defer freeing to avoid using GFP_ATOMIC to allocate vmemmap pages. 1504 * 1505 * Only call schedule_work() if hpage_freelist is previously 1506 * empty. Otherwise, schedule_work() had been called but the workfn 1507 * hasn't retrieved the list yet. 1508 */ 1509 if (llist_add((struct llist_node *)&page->mapping, &hpage_freelist)) 1510 schedule_work(&free_hpage_work); 1511 } 1512 1513 static void update_and_free_pages_bulk(struct hstate *h, struct list_head *list) 1514 { 1515 struct page *page, *t_page; 1516 1517 list_for_each_entry_safe(page, t_page, list, lru) { 1518 update_and_free_page(h, page, false); 1519 cond_resched(); 1520 } 1521 } 1522 1523 struct hstate *size_to_hstate(unsigned long size) 1524 { 1525 struct hstate *h; 1526 1527 for_each_hstate(h) { 1528 if (huge_page_size(h) == size) 1529 return h; 1530 } 1531 return NULL; 1532 } 1533 1534 void free_huge_page(struct page *page) 1535 { 1536 /* 1537 * Can't pass hstate in here because it is called from the 1538 * compound page destructor. 1539 */ 1540 struct hstate *h = page_hstate(page); 1541 int nid = page_to_nid(page); 1542 struct hugepage_subpool *spool = hugetlb_page_subpool(page); 1543 bool restore_reserve; 1544 unsigned long flags; 1545 1546 VM_BUG_ON_PAGE(page_count(page), page); 1547 VM_BUG_ON_PAGE(page_mapcount(page), page); 1548 1549 hugetlb_set_page_subpool(page, NULL); 1550 page->mapping = NULL; 1551 restore_reserve = HPageRestoreReserve(page); 1552 ClearHPageRestoreReserve(page); 1553 1554 /* 1555 * If HPageRestoreReserve was set on page, page allocation consumed a 1556 * reservation. If the page was associated with a subpool, there 1557 * would have been a page reserved in the subpool before allocation 1558 * via hugepage_subpool_get_pages(). Since we are 'restoring' the 1559 * reservation, do not call hugepage_subpool_put_pages() as this will 1560 * remove the reserved page from the subpool. 1561 */ 1562 if (!restore_reserve) { 1563 /* 1564 * A return code of zero implies that the subpool will be 1565 * under its minimum size if the reservation is not restored 1566 * after page is free. Therefore, force restore_reserve 1567 * operation. 1568 */ 1569 if (hugepage_subpool_put_pages(spool, 1) == 0) 1570 restore_reserve = true; 1571 } 1572 1573 spin_lock_irqsave(&hugetlb_lock, flags); 1574 ClearHPageMigratable(page); 1575 hugetlb_cgroup_uncharge_page(hstate_index(h), 1576 pages_per_huge_page(h), page); 1577 hugetlb_cgroup_uncharge_page_rsvd(hstate_index(h), 1578 pages_per_huge_page(h), page); 1579 if (restore_reserve) 1580 h->resv_huge_pages++; 1581 1582 if (HPageTemporary(page)) { 1583 remove_hugetlb_page(h, page, false); 1584 spin_unlock_irqrestore(&hugetlb_lock, flags); 1585 update_and_free_page(h, page, true); 1586 } else if (h->surplus_huge_pages_node[nid]) { 1587 /* remove the page from active list */ 1588 remove_hugetlb_page(h, page, true); 1589 spin_unlock_irqrestore(&hugetlb_lock, flags); 1590 update_and_free_page(h, page, true); 1591 } else { 1592 arch_clear_hugepage_flags(page); 1593 enqueue_huge_page(h, page); 1594 spin_unlock_irqrestore(&hugetlb_lock, flags); 1595 } 1596 } 1597 1598 /* 1599 * Must be called with the hugetlb lock held 1600 */ 1601 static void __prep_account_new_huge_page(struct hstate *h, int nid) 1602 { 1603 lockdep_assert_held(&hugetlb_lock); 1604 h->nr_huge_pages++; 1605 h->nr_huge_pages_node[nid]++; 1606 } 1607 1608 static void __prep_new_huge_page(struct hstate *h, struct page *page) 1609 { 1610 free_huge_page_vmemmap(h, page); 1611 INIT_LIST_HEAD(&page->lru); 1612 set_compound_page_dtor(page, HUGETLB_PAGE_DTOR); 1613 hugetlb_set_page_subpool(page, NULL); 1614 set_hugetlb_cgroup(page, NULL); 1615 set_hugetlb_cgroup_rsvd(page, NULL); 1616 } 1617 1618 static void prep_new_huge_page(struct hstate *h, struct page *page, int nid) 1619 { 1620 __prep_new_huge_page(h, page); 1621 spin_lock_irq(&hugetlb_lock); 1622 __prep_account_new_huge_page(h, nid); 1623 spin_unlock_irq(&hugetlb_lock); 1624 } 1625 1626 static bool prep_compound_gigantic_page(struct page *page, unsigned int order) 1627 { 1628 int i, j; 1629 int nr_pages = 1 << order; 1630 struct page *p = page + 1; 1631 1632 /* we rely on prep_new_huge_page to set the destructor */ 1633 set_compound_order(page, order); 1634 __ClearPageReserved(page); 1635 __SetPageHead(page); 1636 for (i = 1; i < nr_pages; i++, p = mem_map_next(p, page, i)) { 1637 /* 1638 * For gigantic hugepages allocated through bootmem at 1639 * boot, it's safer to be consistent with the not-gigantic 1640 * hugepages and clear the PG_reserved bit from all tail pages 1641 * too. Otherwise drivers using get_user_pages() to access tail 1642 * pages may get the reference counting wrong if they see 1643 * PG_reserved set on a tail page (despite the head page not 1644 * having PG_reserved set). Enforcing this consistency between 1645 * head and tail pages allows drivers to optimize away a check 1646 * on the head page when they need know if put_page() is needed 1647 * after get_user_pages(). 1648 */ 1649 __ClearPageReserved(p); 1650 /* 1651 * Subtle and very unlikely 1652 * 1653 * Gigantic 'page allocators' such as memblock or cma will 1654 * return a set of pages with each page ref counted. We need 1655 * to turn this set of pages into a compound page with tail 1656 * page ref counts set to zero. Code such as speculative page 1657 * cache adding could take a ref on a 'to be' tail page. 1658 * We need to respect any increased ref count, and only set 1659 * the ref count to zero if count is currently 1. If count 1660 * is not 1, we call synchronize_rcu in the hope that a rcu 1661 * grace period will cause ref count to drop and then retry. 1662 * If count is still inflated on retry we return an error and 1663 * must discard the pages. 1664 */ 1665 if (!page_ref_freeze(p, 1)) { 1666 pr_info("HugeTLB unexpected inflated ref count on freshly allocated page\n"); 1667 synchronize_rcu(); 1668 if (!page_ref_freeze(p, 1)) 1669 goto out_error; 1670 } 1671 set_page_count(p, 0); 1672 set_compound_head(p, page); 1673 } 1674 atomic_set(compound_mapcount_ptr(page), -1); 1675 atomic_set(compound_pincount_ptr(page), 0); 1676 return true; 1677 1678 out_error: 1679 /* undo tail page modifications made above */ 1680 p = page + 1; 1681 for (j = 1; j < i; j++, p = mem_map_next(p, page, j)) { 1682 clear_compound_head(p); 1683 set_page_refcounted(p); 1684 } 1685 /* need to clear PG_reserved on remaining tail pages */ 1686 for (; j < nr_pages; j++, p = mem_map_next(p, page, j)) 1687 __ClearPageReserved(p); 1688 set_compound_order(page, 0); 1689 page[1].compound_nr = 0; 1690 __ClearPageHead(page); 1691 return false; 1692 } 1693 1694 /* 1695 * PageHuge() only returns true for hugetlbfs pages, but not for normal or 1696 * transparent huge pages. See the PageTransHuge() documentation for more 1697 * details. 1698 */ 1699 int PageHuge(struct page *page) 1700 { 1701 if (!PageCompound(page)) 1702 return 0; 1703 1704 page = compound_head(page); 1705 return page[1].compound_dtor == HUGETLB_PAGE_DTOR; 1706 } 1707 EXPORT_SYMBOL_GPL(PageHuge); 1708 1709 /* 1710 * PageHeadHuge() only returns true for hugetlbfs head page, but not for 1711 * normal or transparent huge pages. 1712 */ 1713 int PageHeadHuge(struct page *page_head) 1714 { 1715 if (!PageHead(page_head)) 1716 return 0; 1717 1718 return page_head[1].compound_dtor == HUGETLB_PAGE_DTOR; 1719 } 1720 1721 /* 1722 * Find and lock address space (mapping) in write mode. 1723 * 1724 * Upon entry, the page is locked which means that page_mapping() is 1725 * stable. Due to locking order, we can only trylock_write. If we can 1726 * not get the lock, simply return NULL to caller. 1727 */ 1728 struct address_space *hugetlb_page_mapping_lock_write(struct page *hpage) 1729 { 1730 struct address_space *mapping = page_mapping(hpage); 1731 1732 if (!mapping) 1733 return mapping; 1734 1735 if (i_mmap_trylock_write(mapping)) 1736 return mapping; 1737 1738 return NULL; 1739 } 1740 1741 pgoff_t hugetlb_basepage_index(struct page *page) 1742 { 1743 struct page *page_head = compound_head(page); 1744 pgoff_t index = page_index(page_head); 1745 unsigned long compound_idx; 1746 1747 if (compound_order(page_head) >= MAX_ORDER) 1748 compound_idx = page_to_pfn(page) - page_to_pfn(page_head); 1749 else 1750 compound_idx = page - page_head; 1751 1752 return (index << compound_order(page_head)) + compound_idx; 1753 } 1754 1755 static struct page *alloc_buddy_huge_page(struct hstate *h, 1756 gfp_t gfp_mask, int nid, nodemask_t *nmask, 1757 nodemask_t *node_alloc_noretry) 1758 { 1759 int order = huge_page_order(h); 1760 struct page *page; 1761 bool alloc_try_hard = true; 1762 1763 /* 1764 * By default we always try hard to allocate the page with 1765 * __GFP_RETRY_MAYFAIL flag. However, if we are allocating pages in 1766 * a loop (to adjust global huge page counts) and previous allocation 1767 * failed, do not continue to try hard on the same node. Use the 1768 * node_alloc_noretry bitmap to manage this state information. 1769 */ 1770 if (node_alloc_noretry && node_isset(nid, *node_alloc_noretry)) 1771 alloc_try_hard = false; 1772 gfp_mask |= __GFP_COMP|__GFP_NOWARN; 1773 if (alloc_try_hard) 1774 gfp_mask |= __GFP_RETRY_MAYFAIL; 1775 if (nid == NUMA_NO_NODE) 1776 nid = numa_mem_id(); 1777 page = __alloc_pages(gfp_mask, order, nid, nmask); 1778 if (page) 1779 __count_vm_event(HTLB_BUDDY_PGALLOC); 1780 else 1781 __count_vm_event(HTLB_BUDDY_PGALLOC_FAIL); 1782 1783 /* 1784 * If we did not specify __GFP_RETRY_MAYFAIL, but still got a page this 1785 * indicates an overall state change. Clear bit so that we resume 1786 * normal 'try hard' allocations. 1787 */ 1788 if (node_alloc_noretry && page && !alloc_try_hard) 1789 node_clear(nid, *node_alloc_noretry); 1790 1791 /* 1792 * If we tried hard to get a page but failed, set bit so that 1793 * subsequent attempts will not try as hard until there is an 1794 * overall state change. 1795 */ 1796 if (node_alloc_noretry && !page && alloc_try_hard) 1797 node_set(nid, *node_alloc_noretry); 1798 1799 return page; 1800 } 1801 1802 /* 1803 * Common helper to allocate a fresh hugetlb page. All specific allocators 1804 * should use this function to get new hugetlb pages 1805 */ 1806 static struct page *alloc_fresh_huge_page(struct hstate *h, 1807 gfp_t gfp_mask, int nid, nodemask_t *nmask, 1808 nodemask_t *node_alloc_noretry) 1809 { 1810 struct page *page; 1811 bool retry = false; 1812 1813 retry: 1814 if (hstate_is_gigantic(h)) 1815 page = alloc_gigantic_page(h, gfp_mask, nid, nmask); 1816 else 1817 page = alloc_buddy_huge_page(h, gfp_mask, 1818 nid, nmask, node_alloc_noretry); 1819 if (!page) 1820 return NULL; 1821 1822 if (hstate_is_gigantic(h)) { 1823 if (!prep_compound_gigantic_page(page, huge_page_order(h))) { 1824 /* 1825 * Rare failure to convert pages to compound page. 1826 * Free pages and try again - ONCE! 1827 */ 1828 free_gigantic_page(page, huge_page_order(h)); 1829 if (!retry) { 1830 retry = true; 1831 goto retry; 1832 } 1833 pr_warn("HugeTLB page can not be used due to unexpected inflated ref count\n"); 1834 return NULL; 1835 } 1836 } 1837 prep_new_huge_page(h, page, page_to_nid(page)); 1838 1839 return page; 1840 } 1841 1842 /* 1843 * Allocates a fresh page to the hugetlb allocator pool in the node interleaved 1844 * manner. 1845 */ 1846 static int alloc_pool_huge_page(struct hstate *h, nodemask_t *nodes_allowed, 1847 nodemask_t *node_alloc_noretry) 1848 { 1849 struct page *page; 1850 int nr_nodes, node; 1851 gfp_t gfp_mask = htlb_alloc_mask(h) | __GFP_THISNODE; 1852 1853 for_each_node_mask_to_alloc(h, nr_nodes, node, nodes_allowed) { 1854 page = alloc_fresh_huge_page(h, gfp_mask, node, nodes_allowed, 1855 node_alloc_noretry); 1856 if (page) 1857 break; 1858 } 1859 1860 if (!page) 1861 return 0; 1862 1863 put_page(page); /* free it into the hugepage allocator */ 1864 1865 return 1; 1866 } 1867 1868 /* 1869 * Remove huge page from pool from next node to free. Attempt to keep 1870 * persistent huge pages more or less balanced over allowed nodes. 1871 * This routine only 'removes' the hugetlb page. The caller must make 1872 * an additional call to free the page to low level allocators. 1873 * Called with hugetlb_lock locked. 1874 */ 1875 static struct page *remove_pool_huge_page(struct hstate *h, 1876 nodemask_t *nodes_allowed, 1877 bool acct_surplus) 1878 { 1879 int nr_nodes, node; 1880 struct page *page = NULL; 1881 1882 lockdep_assert_held(&hugetlb_lock); 1883 for_each_node_mask_to_free(h, nr_nodes, node, nodes_allowed) { 1884 /* 1885 * If we're returning unused surplus pages, only examine 1886 * nodes with surplus pages. 1887 */ 1888 if ((!acct_surplus || h->surplus_huge_pages_node[node]) && 1889 !list_empty(&h->hugepage_freelists[node])) { 1890 page = list_entry(h->hugepage_freelists[node].next, 1891 struct page, lru); 1892 remove_hugetlb_page(h, page, acct_surplus); 1893 break; 1894 } 1895 } 1896 1897 return page; 1898 } 1899 1900 /* 1901 * Dissolve a given free hugepage into free buddy pages. This function does 1902 * nothing for in-use hugepages and non-hugepages. 1903 * This function returns values like below: 1904 * 1905 * -ENOMEM: failed to allocate vmemmap pages to free the freed hugepages 1906 * when the system is under memory pressure and the feature of 1907 * freeing unused vmemmap pages associated with each hugetlb page 1908 * is enabled. 1909 * -EBUSY: failed to dissolved free hugepages or the hugepage is in-use 1910 * (allocated or reserved.) 1911 * 0: successfully dissolved free hugepages or the page is not a 1912 * hugepage (considered as already dissolved) 1913 */ 1914 int dissolve_free_huge_page(struct page *page) 1915 { 1916 int rc = -EBUSY; 1917 1918 retry: 1919 /* Not to disrupt normal path by vainly holding hugetlb_lock */ 1920 if (!PageHuge(page)) 1921 return 0; 1922 1923 spin_lock_irq(&hugetlb_lock); 1924 if (!PageHuge(page)) { 1925 rc = 0; 1926 goto out; 1927 } 1928 1929 if (!page_count(page)) { 1930 struct page *head = compound_head(page); 1931 struct hstate *h = page_hstate(head); 1932 if (h->free_huge_pages - h->resv_huge_pages == 0) 1933 goto out; 1934 1935 /* 1936 * We should make sure that the page is already on the free list 1937 * when it is dissolved. 1938 */ 1939 if (unlikely(!HPageFreed(head))) { 1940 spin_unlock_irq(&hugetlb_lock); 1941 cond_resched(); 1942 1943 /* 1944 * Theoretically, we should return -EBUSY when we 1945 * encounter this race. In fact, we have a chance 1946 * to successfully dissolve the page if we do a 1947 * retry. Because the race window is quite small. 1948 * If we seize this opportunity, it is an optimization 1949 * for increasing the success rate of dissolving page. 1950 */ 1951 goto retry; 1952 } 1953 1954 remove_hugetlb_page(h, head, false); 1955 h->max_huge_pages--; 1956 spin_unlock_irq(&hugetlb_lock); 1957 1958 /* 1959 * Normally update_and_free_page will allocate required vmemmmap 1960 * before freeing the page. update_and_free_page will fail to 1961 * free the page if it can not allocate required vmemmap. We 1962 * need to adjust max_huge_pages if the page is not freed. 1963 * Attempt to allocate vmemmmap here so that we can take 1964 * appropriate action on failure. 1965 */ 1966 rc = alloc_huge_page_vmemmap(h, head); 1967 if (!rc) { 1968 /* 1969 * Move PageHWPoison flag from head page to the raw 1970 * error page, which makes any subpages rather than 1971 * the error page reusable. 1972 */ 1973 if (PageHWPoison(head) && page != head) { 1974 SetPageHWPoison(page); 1975 ClearPageHWPoison(head); 1976 } 1977 update_and_free_page(h, head, false); 1978 } else { 1979 spin_lock_irq(&hugetlb_lock); 1980 add_hugetlb_page(h, head, false); 1981 h->max_huge_pages++; 1982 spin_unlock_irq(&hugetlb_lock); 1983 } 1984 1985 return rc; 1986 } 1987 out: 1988 spin_unlock_irq(&hugetlb_lock); 1989 return rc; 1990 } 1991 1992 /* 1993 * Dissolve free hugepages in a given pfn range. Used by memory hotplug to 1994 * make specified memory blocks removable from the system. 1995 * Note that this will dissolve a free gigantic hugepage completely, if any 1996 * part of it lies within the given range. 1997 * Also note that if dissolve_free_huge_page() returns with an error, all 1998 * free hugepages that were dissolved before that error are lost. 1999 */ 2000 int dissolve_free_huge_pages(unsigned long start_pfn, unsigned long end_pfn) 2001 { 2002 unsigned long pfn; 2003 struct page *page; 2004 int rc = 0; 2005 2006 if (!hugepages_supported()) 2007 return rc; 2008 2009 for (pfn = start_pfn; pfn < end_pfn; pfn += 1 << minimum_order) { 2010 page = pfn_to_page(pfn); 2011 rc = dissolve_free_huge_page(page); 2012 if (rc) 2013 break; 2014 } 2015 2016 return rc; 2017 } 2018 2019 /* 2020 * Allocates a fresh surplus page from the page allocator. 2021 */ 2022 static struct page *alloc_surplus_huge_page(struct hstate *h, gfp_t gfp_mask, 2023 int nid, nodemask_t *nmask) 2024 { 2025 struct page *page = NULL; 2026 2027 if (hstate_is_gigantic(h)) 2028 return NULL; 2029 2030 spin_lock_irq(&hugetlb_lock); 2031 if (h->surplus_huge_pages >= h->nr_overcommit_huge_pages) 2032 goto out_unlock; 2033 spin_unlock_irq(&hugetlb_lock); 2034 2035 page = alloc_fresh_huge_page(h, gfp_mask, nid, nmask, NULL); 2036 if (!page) 2037 return NULL; 2038 2039 spin_lock_irq(&hugetlb_lock); 2040 /* 2041 * We could have raced with the pool size change. 2042 * Double check that and simply deallocate the new page 2043 * if we would end up overcommiting the surpluses. Abuse 2044 * temporary page to workaround the nasty free_huge_page 2045 * codeflow 2046 */ 2047 if (h->surplus_huge_pages >= h->nr_overcommit_huge_pages) { 2048 SetHPageTemporary(page); 2049 spin_unlock_irq(&hugetlb_lock); 2050 put_page(page); 2051 return NULL; 2052 } else { 2053 h->surplus_huge_pages++; 2054 h->surplus_huge_pages_node[page_to_nid(page)]++; 2055 } 2056 2057 out_unlock: 2058 spin_unlock_irq(&hugetlb_lock); 2059 2060 return page; 2061 } 2062 2063 static struct page *alloc_migrate_huge_page(struct hstate *h, gfp_t gfp_mask, 2064 int nid, nodemask_t *nmask) 2065 { 2066 struct page *page; 2067 2068 if (hstate_is_gigantic(h)) 2069 return NULL; 2070 2071 page = alloc_fresh_huge_page(h, gfp_mask, nid, nmask, NULL); 2072 if (!page) 2073 return NULL; 2074 2075 /* 2076 * We do not account these pages as surplus because they are only 2077 * temporary and will be released properly on the last reference 2078 */ 2079 SetHPageTemporary(page); 2080 2081 return page; 2082 } 2083 2084 /* 2085 * Use the VMA's mpolicy to allocate a huge page from the buddy. 2086 */ 2087 static 2088 struct page *alloc_buddy_huge_page_with_mpol(struct hstate *h, 2089 struct vm_area_struct *vma, unsigned long addr) 2090 { 2091 struct page *page; 2092 struct mempolicy *mpol; 2093 gfp_t gfp_mask = htlb_alloc_mask(h); 2094 int nid; 2095 nodemask_t *nodemask; 2096 2097 nid = huge_node(vma, addr, gfp_mask, &mpol, &nodemask); 2098 page = alloc_surplus_huge_page(h, gfp_mask, nid, nodemask); 2099 mpol_cond_put(mpol); 2100 2101 return page; 2102 } 2103 2104 /* page migration callback function */ 2105 struct page *alloc_huge_page_nodemask(struct hstate *h, int preferred_nid, 2106 nodemask_t *nmask, gfp_t gfp_mask) 2107 { 2108 spin_lock_irq(&hugetlb_lock); 2109 if (h->free_huge_pages - h->resv_huge_pages > 0) { 2110 struct page *page; 2111 2112 page = dequeue_huge_page_nodemask(h, gfp_mask, preferred_nid, nmask); 2113 if (page) { 2114 spin_unlock_irq(&hugetlb_lock); 2115 return page; 2116 } 2117 } 2118 spin_unlock_irq(&hugetlb_lock); 2119 2120 return alloc_migrate_huge_page(h, gfp_mask, preferred_nid, nmask); 2121 } 2122 2123 /* mempolicy aware migration callback */ 2124 struct page *alloc_huge_page_vma(struct hstate *h, struct vm_area_struct *vma, 2125 unsigned long address) 2126 { 2127 struct mempolicy *mpol; 2128 nodemask_t *nodemask; 2129 struct page *page; 2130 gfp_t gfp_mask; 2131 int node; 2132 2133 gfp_mask = htlb_alloc_mask(h); 2134 node = huge_node(vma, address, gfp_mask, &mpol, &nodemask); 2135 page = alloc_huge_page_nodemask(h, node, nodemask, gfp_mask); 2136 mpol_cond_put(mpol); 2137 2138 return page; 2139 } 2140 2141 /* 2142 * Increase the hugetlb pool such that it can accommodate a reservation 2143 * of size 'delta'. 2144 */ 2145 static int gather_surplus_pages(struct hstate *h, long delta) 2146 __must_hold(&hugetlb_lock) 2147 { 2148 struct list_head surplus_list; 2149 struct page *page, *tmp; 2150 int ret; 2151 long i; 2152 long needed, allocated; 2153 bool alloc_ok = true; 2154 2155 lockdep_assert_held(&hugetlb_lock); 2156 needed = (h->resv_huge_pages + delta) - h->free_huge_pages; 2157 if (needed <= 0) { 2158 h->resv_huge_pages += delta; 2159 return 0; 2160 } 2161 2162 allocated = 0; 2163 INIT_LIST_HEAD(&surplus_list); 2164 2165 ret = -ENOMEM; 2166 retry: 2167 spin_unlock_irq(&hugetlb_lock); 2168 for (i = 0; i < needed; i++) { 2169 page = alloc_surplus_huge_page(h, htlb_alloc_mask(h), 2170 NUMA_NO_NODE, NULL); 2171 if (!page) { 2172 alloc_ok = false; 2173 break; 2174 } 2175 list_add(&page->lru, &surplus_list); 2176 cond_resched(); 2177 } 2178 allocated += i; 2179 2180 /* 2181 * After retaking hugetlb_lock, we need to recalculate 'needed' 2182 * because either resv_huge_pages or free_huge_pages may have changed. 2183 */ 2184 spin_lock_irq(&hugetlb_lock); 2185 needed = (h->resv_huge_pages + delta) - 2186 (h->free_huge_pages + allocated); 2187 if (needed > 0) { 2188 if (alloc_ok) 2189 goto retry; 2190 /* 2191 * We were not able to allocate enough pages to 2192 * satisfy the entire reservation so we free what 2193 * we've allocated so far. 2194 */ 2195 goto free; 2196 } 2197 /* 2198 * The surplus_list now contains _at_least_ the number of extra pages 2199 * needed to accommodate the reservation. Add the appropriate number 2200 * of pages to the hugetlb pool and free the extras back to the buddy 2201 * allocator. Commit the entire reservation here to prevent another 2202 * process from stealing the pages as they are added to the pool but 2203 * before they are reserved. 2204 */ 2205 needed += allocated; 2206 h->resv_huge_pages += delta; 2207 ret = 0; 2208 2209 /* Free the needed pages to the hugetlb pool */ 2210 list_for_each_entry_safe(page, tmp, &surplus_list, lru) { 2211 int zeroed; 2212 2213 if ((--needed) < 0) 2214 break; 2215 /* 2216 * This page is now managed by the hugetlb allocator and has 2217 * no users -- drop the buddy allocator's reference. 2218 */ 2219 zeroed = put_page_testzero(page); 2220 VM_BUG_ON_PAGE(!zeroed, page); 2221 enqueue_huge_page(h, page); 2222 } 2223 free: 2224 spin_unlock_irq(&hugetlb_lock); 2225 2226 /* Free unnecessary surplus pages to the buddy allocator */ 2227 list_for_each_entry_safe(page, tmp, &surplus_list, lru) 2228 put_page(page); 2229 spin_lock_irq(&hugetlb_lock); 2230 2231 return ret; 2232 } 2233 2234 /* 2235 * This routine has two main purposes: 2236 * 1) Decrement the reservation count (resv_huge_pages) by the value passed 2237 * in unused_resv_pages. This corresponds to the prior adjustments made 2238 * to the associated reservation map. 2239 * 2) Free any unused surplus pages that may have been allocated to satisfy 2240 * the reservation. As many as unused_resv_pages may be freed. 2241 */ 2242 static void return_unused_surplus_pages(struct hstate *h, 2243 unsigned long unused_resv_pages) 2244 { 2245 unsigned long nr_pages; 2246 struct page *page; 2247 LIST_HEAD(page_list); 2248 2249 lockdep_assert_held(&hugetlb_lock); 2250 /* Uncommit the reservation */ 2251 h->resv_huge_pages -= unused_resv_pages; 2252 2253 /* Cannot return gigantic pages currently */ 2254 if (hstate_is_gigantic(h)) 2255 goto out; 2256 2257 /* 2258 * Part (or even all) of the reservation could have been backed 2259 * by pre-allocated pages. Only free surplus pages. 2260 */ 2261 nr_pages = min(unused_resv_pages, h->surplus_huge_pages); 2262 2263 /* 2264 * We want to release as many surplus pages as possible, spread 2265 * evenly across all nodes with memory. Iterate across these nodes 2266 * until we can no longer free unreserved surplus pages. This occurs 2267 * when the nodes with surplus pages have no free pages. 2268 * remove_pool_huge_page() will balance the freed pages across the 2269 * on-line nodes with memory and will handle the hstate accounting. 2270 */ 2271 while (nr_pages--) { 2272 page = remove_pool_huge_page(h, &node_states[N_MEMORY], 1); 2273 if (!page) 2274 goto out; 2275 2276 list_add(&page->lru, &page_list); 2277 } 2278 2279 out: 2280 spin_unlock_irq(&hugetlb_lock); 2281 update_and_free_pages_bulk(h, &page_list); 2282 spin_lock_irq(&hugetlb_lock); 2283 } 2284 2285 2286 /* 2287 * vma_needs_reservation, vma_commit_reservation and vma_end_reservation 2288 * are used by the huge page allocation routines to manage reservations. 2289 * 2290 * vma_needs_reservation is called to determine if the huge page at addr 2291 * within the vma has an associated reservation. If a reservation is 2292 * needed, the value 1 is returned. The caller is then responsible for 2293 * managing the global reservation and subpool usage counts. After 2294 * the huge page has been allocated, vma_commit_reservation is called 2295 * to add the page to the reservation map. If the page allocation fails, 2296 * the reservation must be ended instead of committed. vma_end_reservation 2297 * is called in such cases. 2298 * 2299 * In the normal case, vma_commit_reservation returns the same value 2300 * as the preceding vma_needs_reservation call. The only time this 2301 * is not the case is if a reserve map was changed between calls. It 2302 * is the responsibility of the caller to notice the difference and 2303 * take appropriate action. 2304 * 2305 * vma_add_reservation is used in error paths where a reservation must 2306 * be restored when a newly allocated huge page must be freed. It is 2307 * to be called after calling vma_needs_reservation to determine if a 2308 * reservation exists. 2309 * 2310 * vma_del_reservation is used in error paths where an entry in the reserve 2311 * map was created during huge page allocation and must be removed. It is to 2312 * be called after calling vma_needs_reservation to determine if a reservation 2313 * exists. 2314 */ 2315 enum vma_resv_mode { 2316 VMA_NEEDS_RESV, 2317 VMA_COMMIT_RESV, 2318 VMA_END_RESV, 2319 VMA_ADD_RESV, 2320 VMA_DEL_RESV, 2321 }; 2322 static long __vma_reservation_common(struct hstate *h, 2323 struct vm_area_struct *vma, unsigned long addr, 2324 enum vma_resv_mode mode) 2325 { 2326 struct resv_map *resv; 2327 pgoff_t idx; 2328 long ret; 2329 long dummy_out_regions_needed; 2330 2331 resv = vma_resv_map(vma); 2332 if (!resv) 2333 return 1; 2334 2335 idx = vma_hugecache_offset(h, vma, addr); 2336 switch (mode) { 2337 case VMA_NEEDS_RESV: 2338 ret = region_chg(resv, idx, idx + 1, &dummy_out_regions_needed); 2339 /* We assume that vma_reservation_* routines always operate on 2340 * 1 page, and that adding to resv map a 1 page entry can only 2341 * ever require 1 region. 2342 */ 2343 VM_BUG_ON(dummy_out_regions_needed != 1); 2344 break; 2345 case VMA_COMMIT_RESV: 2346 ret = region_add(resv, idx, idx + 1, 1, NULL, NULL); 2347 /* region_add calls of range 1 should never fail. */ 2348 VM_BUG_ON(ret < 0); 2349 break; 2350 case VMA_END_RESV: 2351 region_abort(resv, idx, idx + 1, 1); 2352 ret = 0; 2353 break; 2354 case VMA_ADD_RESV: 2355 if (vma->vm_flags & VM_MAYSHARE) { 2356 ret = region_add(resv, idx, idx + 1, 1, NULL, NULL); 2357 /* region_add calls of range 1 should never fail. */ 2358 VM_BUG_ON(ret < 0); 2359 } else { 2360 region_abort(resv, idx, idx + 1, 1); 2361 ret = region_del(resv, idx, idx + 1); 2362 } 2363 break; 2364 case VMA_DEL_RESV: 2365 if (vma->vm_flags & VM_MAYSHARE) { 2366 region_abort(resv, idx, idx + 1, 1); 2367 ret = region_del(resv, idx, idx + 1); 2368 } else { 2369 ret = region_add(resv, idx, idx + 1, 1, NULL, NULL); 2370 /* region_add calls of range 1 should never fail. */ 2371 VM_BUG_ON(ret < 0); 2372 } 2373 break; 2374 default: 2375 BUG(); 2376 } 2377 2378 if (vma->vm_flags & VM_MAYSHARE || mode == VMA_DEL_RESV) 2379 return ret; 2380 /* 2381 * We know private mapping must have HPAGE_RESV_OWNER set. 2382 * 2383 * In most cases, reserves always exist for private mappings. 2384 * However, a file associated with mapping could have been 2385 * hole punched or truncated after reserves were consumed. 2386 * As subsequent fault on such a range will not use reserves. 2387 * Subtle - The reserve map for private mappings has the 2388 * opposite meaning than that of shared mappings. If NO 2389 * entry is in the reserve map, it means a reservation exists. 2390 * If an entry exists in the reserve map, it means the 2391 * reservation has already been consumed. As a result, the 2392 * return value of this routine is the opposite of the 2393 * value returned from reserve map manipulation routines above. 2394 */ 2395 if (ret > 0) 2396 return 0; 2397 if (ret == 0) 2398 return 1; 2399 return ret; 2400 } 2401 2402 static long vma_needs_reservation(struct hstate *h, 2403 struct vm_area_struct *vma, unsigned long addr) 2404 { 2405 return __vma_reservation_common(h, vma, addr, VMA_NEEDS_RESV); 2406 } 2407 2408 static long vma_commit_reservation(struct hstate *h, 2409 struct vm_area_struct *vma, unsigned long addr) 2410 { 2411 return __vma_reservation_common(h, vma, addr, VMA_COMMIT_RESV); 2412 } 2413 2414 static void vma_end_reservation(struct hstate *h, 2415 struct vm_area_struct *vma, unsigned long addr) 2416 { 2417 (void)__vma_reservation_common(h, vma, addr, VMA_END_RESV); 2418 } 2419 2420 static long vma_add_reservation(struct hstate *h, 2421 struct vm_area_struct *vma, unsigned long addr) 2422 { 2423 return __vma_reservation_common(h, vma, addr, VMA_ADD_RESV); 2424 } 2425 2426 static long vma_del_reservation(struct hstate *h, 2427 struct vm_area_struct *vma, unsigned long addr) 2428 { 2429 return __vma_reservation_common(h, vma, addr, VMA_DEL_RESV); 2430 } 2431 2432 /* 2433 * This routine is called to restore reservation information on error paths. 2434 * It should ONLY be called for pages allocated via alloc_huge_page(), and 2435 * the hugetlb mutex should remain held when calling this routine. 2436 * 2437 * It handles two specific cases: 2438 * 1) A reservation was in place and the page consumed the reservation. 2439 * HPageRestoreReserve is set in the page. 2440 * 2) No reservation was in place for the page, so HPageRestoreReserve is 2441 * not set. However, alloc_huge_page always updates the reserve map. 2442 * 2443 * In case 1, free_huge_page later in the error path will increment the 2444 * global reserve count. But, free_huge_page does not have enough context 2445 * to adjust the reservation map. This case deals primarily with private 2446 * mappings. Adjust the reserve map here to be consistent with global 2447 * reserve count adjustments to be made by free_huge_page. Make sure the 2448 * reserve map indicates there is a reservation present. 2449 * 2450 * In case 2, simply undo reserve map modifications done by alloc_huge_page. 2451 */ 2452 void restore_reserve_on_error(struct hstate *h, struct vm_area_struct *vma, 2453 unsigned long address, struct page *page) 2454 { 2455 long rc = vma_needs_reservation(h, vma, address); 2456 2457 if (HPageRestoreReserve(page)) { 2458 if (unlikely(rc < 0)) 2459 /* 2460 * Rare out of memory condition in reserve map 2461 * manipulation. Clear HPageRestoreReserve so that 2462 * global reserve count will not be incremented 2463 * by free_huge_page. This will make it appear 2464 * as though the reservation for this page was 2465 * consumed. This may prevent the task from 2466 * faulting in the page at a later time. This 2467 * is better than inconsistent global huge page 2468 * accounting of reserve counts. 2469 */ 2470 ClearHPageRestoreReserve(page); 2471 else if (rc) 2472 (void)vma_add_reservation(h, vma, address); 2473 else 2474 vma_end_reservation(h, vma, address); 2475 } else { 2476 if (!rc) { 2477 /* 2478 * This indicates there is an entry in the reserve map 2479 * added by alloc_huge_page. We know it was added 2480 * before the alloc_huge_page call, otherwise 2481 * HPageRestoreReserve would be set on the page. 2482 * Remove the entry so that a subsequent allocation 2483 * does not consume a reservation. 2484 */ 2485 rc = vma_del_reservation(h, vma, address); 2486 if (rc < 0) 2487 /* 2488 * VERY rare out of memory condition. Since 2489 * we can not delete the entry, set 2490 * HPageRestoreReserve so that the reserve 2491 * count will be incremented when the page 2492 * is freed. This reserve will be consumed 2493 * on a subsequent allocation. 2494 */ 2495 SetHPageRestoreReserve(page); 2496 } else if (rc < 0) { 2497 /* 2498 * Rare out of memory condition from 2499 * vma_needs_reservation call. Memory allocation is 2500 * only attempted if a new entry is needed. Therefore, 2501 * this implies there is not an entry in the 2502 * reserve map. 2503 * 2504 * For shared mappings, no entry in the map indicates 2505 * no reservation. We are done. 2506 */ 2507 if (!(vma->vm_flags & VM_MAYSHARE)) 2508 /* 2509 * For private mappings, no entry indicates 2510 * a reservation is present. Since we can 2511 * not add an entry, set SetHPageRestoreReserve 2512 * on the page so reserve count will be 2513 * incremented when freed. This reserve will 2514 * be consumed on a subsequent allocation. 2515 */ 2516 SetHPageRestoreReserve(page); 2517 } else 2518 /* 2519 * No reservation present, do nothing 2520 */ 2521 vma_end_reservation(h, vma, address); 2522 } 2523 } 2524 2525 /* 2526 * alloc_and_dissolve_huge_page - Allocate a new page and dissolve the old one 2527 * @h: struct hstate old page belongs to 2528 * @old_page: Old page to dissolve 2529 * @list: List to isolate the page in case we need to 2530 * Returns 0 on success, otherwise negated error. 2531 */ 2532 static int alloc_and_dissolve_huge_page(struct hstate *h, struct page *old_page, 2533 struct list_head *list) 2534 { 2535 gfp_t gfp_mask = htlb_alloc_mask(h) | __GFP_THISNODE; 2536 int nid = page_to_nid(old_page); 2537 struct page *new_page; 2538 int ret = 0; 2539 2540 /* 2541 * Before dissolving the page, we need to allocate a new one for the 2542 * pool to remain stable. Here, we allocate the page and 'prep' it 2543 * by doing everything but actually updating counters and adding to 2544 * the pool. This simplifies and let us do most of the processing 2545 * under the lock. 2546 */ 2547 new_page = alloc_buddy_huge_page(h, gfp_mask, nid, NULL, NULL); 2548 if (!new_page) 2549 return -ENOMEM; 2550 __prep_new_huge_page(h, new_page); 2551 2552 retry: 2553 spin_lock_irq(&hugetlb_lock); 2554 if (!PageHuge(old_page)) { 2555 /* 2556 * Freed from under us. Drop new_page too. 2557 */ 2558 goto free_new; 2559 } else if (page_count(old_page)) { 2560 /* 2561 * Someone has grabbed the page, try to isolate it here. 2562 * Fail with -EBUSY if not possible. 2563 */ 2564 spin_unlock_irq(&hugetlb_lock); 2565 if (!isolate_huge_page(old_page, list)) 2566 ret = -EBUSY; 2567 spin_lock_irq(&hugetlb_lock); 2568 goto free_new; 2569 } else if (!HPageFreed(old_page)) { 2570 /* 2571 * Page's refcount is 0 but it has not been enqueued in the 2572 * freelist yet. Race window is small, so we can succeed here if 2573 * we retry. 2574 */ 2575 spin_unlock_irq(&hugetlb_lock); 2576 cond_resched(); 2577 goto retry; 2578 } else { 2579 /* 2580 * Ok, old_page is still a genuine free hugepage. Remove it from 2581 * the freelist and decrease the counters. These will be 2582 * incremented again when calling __prep_account_new_huge_page() 2583 * and enqueue_huge_page() for new_page. The counters will remain 2584 * stable since this happens under the lock. 2585 */ 2586 remove_hugetlb_page(h, old_page, false); 2587 2588 /* 2589 * Reference count trick is needed because allocator gives us 2590 * referenced page but the pool requires pages with 0 refcount. 2591 */ 2592 __prep_account_new_huge_page(h, nid); 2593 page_ref_dec(new_page); 2594 enqueue_huge_page(h, new_page); 2595 2596 /* 2597 * Pages have been replaced, we can safely free the old one. 2598 */ 2599 spin_unlock_irq(&hugetlb_lock); 2600 update_and_free_page(h, old_page, false); 2601 } 2602 2603 return ret; 2604 2605 free_new: 2606 spin_unlock_irq(&hugetlb_lock); 2607 update_and_free_page(h, new_page, false); 2608 2609 return ret; 2610 } 2611 2612 int isolate_or_dissolve_huge_page(struct page *page, struct list_head *list) 2613 { 2614 struct hstate *h; 2615 struct page *head; 2616 int ret = -EBUSY; 2617 2618 /* 2619 * The page might have been dissolved from under our feet, so make sure 2620 * to carefully check the state under the lock. 2621 * Return success when racing as if we dissolved the page ourselves. 2622 */ 2623 spin_lock_irq(&hugetlb_lock); 2624 if (PageHuge(page)) { 2625 head = compound_head(page); 2626 h = page_hstate(head); 2627 } else { 2628 spin_unlock_irq(&hugetlb_lock); 2629 return 0; 2630 } 2631 spin_unlock_irq(&hugetlb_lock); 2632 2633 /* 2634 * Fence off gigantic pages as there is a cyclic dependency between 2635 * alloc_contig_range and them. Return -ENOMEM as this has the effect 2636 * of bailing out right away without further retrying. 2637 */ 2638 if (hstate_is_gigantic(h)) 2639 return -ENOMEM; 2640 2641 if (page_count(head) && isolate_huge_page(head, list)) 2642 ret = 0; 2643 else if (!page_count(head)) 2644 ret = alloc_and_dissolve_huge_page(h, head, list); 2645 2646 return ret; 2647 } 2648 2649 struct page *alloc_huge_page(struct vm_area_struct *vma, 2650 unsigned long addr, int avoid_reserve) 2651 { 2652 struct hugepage_subpool *spool = subpool_vma(vma); 2653 struct hstate *h = hstate_vma(vma); 2654 struct page *page; 2655 long map_chg, map_commit; 2656 long gbl_chg; 2657 int ret, idx; 2658 struct hugetlb_cgroup *h_cg; 2659 bool deferred_reserve; 2660 2661 idx = hstate_index(h); 2662 /* 2663 * Examine the region/reserve map to determine if the process 2664 * has a reservation for the page to be allocated. A return 2665 * code of zero indicates a reservation exists (no change). 2666 */ 2667 map_chg = gbl_chg = vma_needs_reservation(h, vma, addr); 2668 if (map_chg < 0) 2669 return ERR_PTR(-ENOMEM); 2670 2671 /* 2672 * Processes that did not create the mapping will have no 2673 * reserves as indicated by the region/reserve map. Check 2674 * that the allocation will not exceed the subpool limit. 2675 * Allocations for MAP_NORESERVE mappings also need to be 2676 * checked against any subpool limit. 2677 */ 2678 if (map_chg || avoid_reserve) { 2679 gbl_chg = hugepage_subpool_get_pages(spool, 1); 2680 if (gbl_chg < 0) { 2681 vma_end_reservation(h, vma, addr); 2682 return ERR_PTR(-ENOSPC); 2683 } 2684 2685 /* 2686 * Even though there was no reservation in the region/reserve 2687 * map, there could be reservations associated with the 2688 * subpool that can be used. This would be indicated if the 2689 * return value of hugepage_subpool_get_pages() is zero. 2690 * However, if avoid_reserve is specified we still avoid even 2691 * the subpool reservations. 2692 */ 2693 if (avoid_reserve) 2694 gbl_chg = 1; 2695 } 2696 2697 /* If this allocation is not consuming a reservation, charge it now. 2698 */ 2699 deferred_reserve = map_chg || avoid_reserve; 2700 if (deferred_reserve) { 2701 ret = hugetlb_cgroup_charge_cgroup_rsvd( 2702 idx, pages_per_huge_page(h), &h_cg); 2703 if (ret) 2704 goto out_subpool_put; 2705 } 2706 2707 ret = hugetlb_cgroup_charge_cgroup(idx, pages_per_huge_page(h), &h_cg); 2708 if (ret) 2709 goto out_uncharge_cgroup_reservation; 2710 2711 spin_lock_irq(&hugetlb_lock); 2712 /* 2713 * glb_chg is passed to indicate whether or not a page must be taken 2714 * from the global free pool (global change). gbl_chg == 0 indicates 2715 * a reservation exists for the allocation. 2716 */ 2717 page = dequeue_huge_page_vma(h, vma, addr, avoid_reserve, gbl_chg); 2718 if (!page) { 2719 spin_unlock_irq(&hugetlb_lock); 2720 page = alloc_buddy_huge_page_with_mpol(h, vma, addr); 2721 if (!page) 2722 goto out_uncharge_cgroup; 2723 if (!avoid_reserve && vma_has_reserves(vma, gbl_chg)) { 2724 SetHPageRestoreReserve(page); 2725 h->resv_huge_pages--; 2726 } 2727 spin_lock_irq(&hugetlb_lock); 2728 list_add(&page->lru, &h->hugepage_activelist); 2729 /* Fall through */ 2730 } 2731 hugetlb_cgroup_commit_charge(idx, pages_per_huge_page(h), h_cg, page); 2732 /* If allocation is not consuming a reservation, also store the 2733 * hugetlb_cgroup pointer on the page. 2734 */ 2735 if (deferred_reserve) { 2736 hugetlb_cgroup_commit_charge_rsvd(idx, pages_per_huge_page(h), 2737 h_cg, page); 2738 } 2739 2740 spin_unlock_irq(&hugetlb_lock); 2741 2742 hugetlb_set_page_subpool(page, spool); 2743 2744 map_commit = vma_commit_reservation(h, vma, addr); 2745 if (unlikely(map_chg > map_commit)) { 2746 /* 2747 * The page was added to the reservation map between 2748 * vma_needs_reservation and vma_commit_reservation. 2749 * This indicates a race with hugetlb_reserve_pages. 2750 * Adjust for the subpool count incremented above AND 2751 * in hugetlb_reserve_pages for the same page. Also, 2752 * the reservation count added in hugetlb_reserve_pages 2753 * no longer applies. 2754 */ 2755 long rsv_adjust; 2756 2757 rsv_adjust = hugepage_subpool_put_pages(spool, 1); 2758 hugetlb_acct_memory(h, -rsv_adjust); 2759 if (deferred_reserve) 2760 hugetlb_cgroup_uncharge_page_rsvd(hstate_index(h), 2761 pages_per_huge_page(h), page); 2762 } 2763 return page; 2764 2765 out_uncharge_cgroup: 2766 hugetlb_cgroup_uncharge_cgroup(idx, pages_per_huge_page(h), h_cg); 2767 out_uncharge_cgroup_reservation: 2768 if (deferred_reserve) 2769 hugetlb_cgroup_uncharge_cgroup_rsvd(idx, pages_per_huge_page(h), 2770 h_cg); 2771 out_subpool_put: 2772 if (map_chg || avoid_reserve) 2773 hugepage_subpool_put_pages(spool, 1); 2774 vma_end_reservation(h, vma, addr); 2775 return ERR_PTR(-ENOSPC); 2776 } 2777 2778 int alloc_bootmem_huge_page(struct hstate *h) 2779 __attribute__ ((weak, alias("__alloc_bootmem_huge_page"))); 2780 int __alloc_bootmem_huge_page(struct hstate *h) 2781 { 2782 struct huge_bootmem_page *m; 2783 int nr_nodes, node; 2784 2785 for_each_node_mask_to_alloc(h, nr_nodes, node, &node_states[N_MEMORY]) { 2786 void *addr; 2787 2788 addr = memblock_alloc_try_nid_raw( 2789 huge_page_size(h), huge_page_size(h), 2790 0, MEMBLOCK_ALLOC_ACCESSIBLE, node); 2791 if (addr) { 2792 /* 2793 * Use the beginning of the huge page to store the 2794 * huge_bootmem_page struct (until gather_bootmem 2795 * puts them into the mem_map). 2796 */ 2797 m = addr; 2798 goto found; 2799 } 2800 } 2801 return 0; 2802 2803 found: 2804 BUG_ON(!IS_ALIGNED(virt_to_phys(m), huge_page_size(h))); 2805 /* Put them into a private list first because mem_map is not up yet */ 2806 INIT_LIST_HEAD(&m->list); 2807 list_add(&m->list, &huge_boot_pages); 2808 m->hstate = h; 2809 return 1; 2810 } 2811 2812 /* 2813 * Put bootmem huge pages into the standard lists after mem_map is up. 2814 * Note: This only applies to gigantic (order > MAX_ORDER) pages. 2815 */ 2816 static void __init gather_bootmem_prealloc(void) 2817 { 2818 struct huge_bootmem_page *m; 2819 2820 list_for_each_entry(m, &huge_boot_pages, list) { 2821 struct page *page = virt_to_page(m); 2822 struct hstate *h = m->hstate; 2823 2824 VM_BUG_ON(!hstate_is_gigantic(h)); 2825 WARN_ON(page_count(page) != 1); 2826 if (prep_compound_gigantic_page(page, huge_page_order(h))) { 2827 WARN_ON(PageReserved(page)); 2828 prep_new_huge_page(h, page, page_to_nid(page)); 2829 put_page(page); /* add to the hugepage allocator */ 2830 } else { 2831 free_gigantic_page(page, huge_page_order(h)); 2832 pr_warn("HugeTLB page can not be used due to unexpected inflated ref count\n"); 2833 } 2834 2835 /* 2836 * We need to restore the 'stolen' pages to totalram_pages 2837 * in order to fix confusing memory reports from free(1) and 2838 * other side-effects, like CommitLimit going negative. 2839 */ 2840 adjust_managed_page_count(page, pages_per_huge_page(h)); 2841 cond_resched(); 2842 } 2843 } 2844 2845 static void __init hugetlb_hstate_alloc_pages(struct hstate *h) 2846 { 2847 unsigned long i; 2848 nodemask_t *node_alloc_noretry; 2849 2850 if (!hstate_is_gigantic(h)) { 2851 /* 2852 * Bit mask controlling how hard we retry per-node allocations. 2853 * Ignore errors as lower level routines can deal with 2854 * node_alloc_noretry == NULL. If this kmalloc fails at boot 2855 * time, we are likely in bigger trouble. 2856 */ 2857 node_alloc_noretry = kmalloc(sizeof(*node_alloc_noretry), 2858 GFP_KERNEL); 2859 } else { 2860 /* allocations done at boot time */ 2861 node_alloc_noretry = NULL; 2862 } 2863 2864 /* bit mask controlling how hard we retry per-node allocations */ 2865 if (node_alloc_noretry) 2866 nodes_clear(*node_alloc_noretry); 2867 2868 for (i = 0; i < h->max_huge_pages; ++i) { 2869 if (hstate_is_gigantic(h)) { 2870 if (hugetlb_cma_size) { 2871 pr_warn_once("HugeTLB: hugetlb_cma is enabled, skip boot time allocation\n"); 2872 goto free; 2873 } 2874 if (!alloc_bootmem_huge_page(h)) 2875 break; 2876 } else if (!alloc_pool_huge_page(h, 2877 &node_states[N_MEMORY], 2878 node_alloc_noretry)) 2879 break; 2880 cond_resched(); 2881 } 2882 if (i < h->max_huge_pages) { 2883 char buf[32]; 2884 2885 string_get_size(huge_page_size(h), 1, STRING_UNITS_2, buf, 32); 2886 pr_warn("HugeTLB: allocating %lu of page size %s failed. Only allocated %lu hugepages.\n", 2887 h->max_huge_pages, buf, i); 2888 h->max_huge_pages = i; 2889 } 2890 free: 2891 kfree(node_alloc_noretry); 2892 } 2893 2894 static void __init hugetlb_init_hstates(void) 2895 { 2896 struct hstate *h; 2897 2898 for_each_hstate(h) { 2899 if (minimum_order > huge_page_order(h)) 2900 minimum_order = huge_page_order(h); 2901 2902 /* oversize hugepages were init'ed in early boot */ 2903 if (!hstate_is_gigantic(h)) 2904 hugetlb_hstate_alloc_pages(h); 2905 } 2906 VM_BUG_ON(minimum_order == UINT_MAX); 2907 } 2908 2909 static void __init report_hugepages(void) 2910 { 2911 struct hstate *h; 2912 2913 for_each_hstate(h) { 2914 char buf[32]; 2915 2916 string_get_size(huge_page_size(h), 1, STRING_UNITS_2, buf, 32); 2917 pr_info("HugeTLB registered %s page size, pre-allocated %ld pages\n", 2918 buf, h->free_huge_pages); 2919 } 2920 } 2921 2922 #ifdef CONFIG_HIGHMEM 2923 static void try_to_free_low(struct hstate *h, unsigned long count, 2924 nodemask_t *nodes_allowed) 2925 { 2926 int i; 2927 LIST_HEAD(page_list); 2928 2929 lockdep_assert_held(&hugetlb_lock); 2930 if (hstate_is_gigantic(h)) 2931 return; 2932 2933 /* 2934 * Collect pages to be freed on a list, and free after dropping lock 2935 */ 2936 for_each_node_mask(i, *nodes_allowed) { 2937 struct page *page, *next; 2938 struct list_head *freel = &h->hugepage_freelists[i]; 2939 list_for_each_entry_safe(page, next, freel, lru) { 2940 if (count >= h->nr_huge_pages) 2941 goto out; 2942 if (PageHighMem(page)) 2943 continue; 2944 remove_hugetlb_page(h, page, false); 2945 list_add(&page->lru, &page_list); 2946 } 2947 } 2948 2949 out: 2950 spin_unlock_irq(&hugetlb_lock); 2951 update_and_free_pages_bulk(h, &page_list); 2952 spin_lock_irq(&hugetlb_lock); 2953 } 2954 #else 2955 static inline void try_to_free_low(struct hstate *h, unsigned long count, 2956 nodemask_t *nodes_allowed) 2957 { 2958 } 2959 #endif 2960 2961 /* 2962 * Increment or decrement surplus_huge_pages. Keep node-specific counters 2963 * balanced by operating on them in a round-robin fashion. 2964 * Returns 1 if an adjustment was made. 2965 */ 2966 static int adjust_pool_surplus(struct hstate *h, nodemask_t *nodes_allowed, 2967 int delta) 2968 { 2969 int nr_nodes, node; 2970 2971 lockdep_assert_held(&hugetlb_lock); 2972 VM_BUG_ON(delta != -1 && delta != 1); 2973 2974 if (delta < 0) { 2975 for_each_node_mask_to_alloc(h, nr_nodes, node, nodes_allowed) { 2976 if (h->surplus_huge_pages_node[node]) 2977 goto found; 2978 } 2979 } else { 2980 for_each_node_mask_to_free(h, nr_nodes, node, nodes_allowed) { 2981 if (h->surplus_huge_pages_node[node] < 2982 h->nr_huge_pages_node[node]) 2983 goto found; 2984 } 2985 } 2986 return 0; 2987 2988 found: 2989 h->surplus_huge_pages += delta; 2990 h->surplus_huge_pages_node[node] += delta; 2991 return 1; 2992 } 2993 2994 #define persistent_huge_pages(h) (h->nr_huge_pages - h->surplus_huge_pages) 2995 static int set_max_huge_pages(struct hstate *h, unsigned long count, int nid, 2996 nodemask_t *nodes_allowed) 2997 { 2998 unsigned long min_count, ret; 2999 struct page *page; 3000 LIST_HEAD(page_list); 3001 NODEMASK_ALLOC(nodemask_t, node_alloc_noretry, GFP_KERNEL); 3002 3003 /* 3004 * Bit mask controlling how hard we retry per-node allocations. 3005 * If we can not allocate the bit mask, do not attempt to allocate 3006 * the requested huge pages. 3007 */ 3008 if (node_alloc_noretry) 3009 nodes_clear(*node_alloc_noretry); 3010 else 3011 return -ENOMEM; 3012 3013 /* 3014 * resize_lock mutex prevents concurrent adjustments to number of 3015 * pages in hstate via the proc/sysfs interfaces. 3016 */ 3017 mutex_lock(&h->resize_lock); 3018 flush_free_hpage_work(h); 3019 spin_lock_irq(&hugetlb_lock); 3020 3021 /* 3022 * Check for a node specific request. 3023 * Changing node specific huge page count may require a corresponding 3024 * change to the global count. In any case, the passed node mask 3025 * (nodes_allowed) will restrict alloc/free to the specified node. 3026 */ 3027 if (nid != NUMA_NO_NODE) { 3028 unsigned long old_count = count; 3029 3030 count += h->nr_huge_pages - h->nr_huge_pages_node[nid]; 3031 /* 3032 * User may have specified a large count value which caused the 3033 * above calculation to overflow. In this case, they wanted 3034 * to allocate as many huge pages as possible. Set count to 3035 * largest possible value to align with their intention. 3036 */ 3037 if (count < old_count) 3038 count = ULONG_MAX; 3039 } 3040 3041 /* 3042 * Gigantic pages runtime allocation depend on the capability for large 3043 * page range allocation. 3044 * If the system does not provide this feature, return an error when 3045 * the user tries to allocate gigantic pages but let the user free the 3046 * boottime allocated gigantic pages. 3047 */ 3048 if (hstate_is_gigantic(h) && !IS_ENABLED(CONFIG_CONTIG_ALLOC)) { 3049 if (count > persistent_huge_pages(h)) { 3050 spin_unlock_irq(&hugetlb_lock); 3051 mutex_unlock(&h->resize_lock); 3052 NODEMASK_FREE(node_alloc_noretry); 3053 return -EINVAL; 3054 } 3055 /* Fall through to decrease pool */ 3056 } 3057 3058 /* 3059 * Increase the pool size 3060 * First take pages out of surplus state. Then make up the 3061 * remaining difference by allocating fresh huge pages. 3062 * 3063 * We might race with alloc_surplus_huge_page() here and be unable 3064 * to convert a surplus huge page to a normal huge page. That is 3065 * not critical, though, it just means the overall size of the 3066 * pool might be one hugepage larger than it needs to be, but 3067 * within all the constraints specified by the sysctls. 3068 */ 3069 while (h->surplus_huge_pages && count > persistent_huge_pages(h)) { 3070 if (!adjust_pool_surplus(h, nodes_allowed, -1)) 3071 break; 3072 } 3073 3074 while (count > persistent_huge_pages(h)) { 3075 /* 3076 * If this allocation races such that we no longer need the 3077 * page, free_huge_page will handle it by freeing the page 3078 * and reducing the surplus. 3079 */ 3080 spin_unlock_irq(&hugetlb_lock); 3081 3082 /* yield cpu to avoid soft lockup */ 3083 cond_resched(); 3084 3085 ret = alloc_pool_huge_page(h, nodes_allowed, 3086 node_alloc_noretry); 3087 spin_lock_irq(&hugetlb_lock); 3088 if (!ret) 3089 goto out; 3090 3091 /* Bail for signals. Probably ctrl-c from user */ 3092 if (signal_pending(current)) 3093 goto out; 3094 } 3095 3096 /* 3097 * Decrease the pool size 3098 * First return free pages to the buddy allocator (being careful 3099 * to keep enough around to satisfy reservations). Then place 3100 * pages into surplus state as needed so the pool will shrink 3101 * to the desired size as pages become free. 3102 * 3103 * By placing pages into the surplus state independent of the 3104 * overcommit value, we are allowing the surplus pool size to 3105 * exceed overcommit. There are few sane options here. Since 3106 * alloc_surplus_huge_page() is checking the global counter, 3107 * though, we'll note that we're not allowed to exceed surplus 3108 * and won't grow the pool anywhere else. Not until one of the 3109 * sysctls are changed, or the surplus pages go out of use. 3110 */ 3111 min_count = h->resv_huge_pages + h->nr_huge_pages - h->free_huge_pages; 3112 min_count = max(count, min_count); 3113 try_to_free_low(h, min_count, nodes_allowed); 3114 3115 /* 3116 * Collect pages to be removed on list without dropping lock 3117 */ 3118 while (min_count < persistent_huge_pages(h)) { 3119 page = remove_pool_huge_page(h, nodes_allowed, 0); 3120 if (!page) 3121 break; 3122 3123 list_add(&page->lru, &page_list); 3124 } 3125 /* free the pages after dropping lock */ 3126 spin_unlock_irq(&hugetlb_lock); 3127 update_and_free_pages_bulk(h, &page_list); 3128 flush_free_hpage_work(h); 3129 spin_lock_irq(&hugetlb_lock); 3130 3131 while (count < persistent_huge_pages(h)) { 3132 if (!adjust_pool_surplus(h, nodes_allowed, 1)) 3133 break; 3134 } 3135 out: 3136 h->max_huge_pages = persistent_huge_pages(h); 3137 spin_unlock_irq(&hugetlb_lock); 3138 mutex_unlock(&h->resize_lock); 3139 3140 NODEMASK_FREE(node_alloc_noretry); 3141 3142 return 0; 3143 } 3144 3145 #define HSTATE_ATTR_RO(_name) \ 3146 static struct kobj_attribute _name##_attr = __ATTR_RO(_name) 3147 3148 #define HSTATE_ATTR(_name) \ 3149 static struct kobj_attribute _name##_attr = \ 3150 __ATTR(_name, 0644, _name##_show, _name##_store) 3151 3152 static struct kobject *hugepages_kobj; 3153 static struct kobject *hstate_kobjs[HUGE_MAX_HSTATE]; 3154 3155 static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp); 3156 3157 static struct hstate *kobj_to_hstate(struct kobject *kobj, int *nidp) 3158 { 3159 int i; 3160 3161 for (i = 0; i < HUGE_MAX_HSTATE; i++) 3162 if (hstate_kobjs[i] == kobj) { 3163 if (nidp) 3164 *nidp = NUMA_NO_NODE; 3165 return &hstates[i]; 3166 } 3167 3168 return kobj_to_node_hstate(kobj, nidp); 3169 } 3170 3171 static ssize_t nr_hugepages_show_common(struct kobject *kobj, 3172 struct kobj_attribute *attr, char *buf) 3173 { 3174 struct hstate *h; 3175 unsigned long nr_huge_pages; 3176 int nid; 3177 3178 h = kobj_to_hstate(kobj, &nid); 3179 if (nid == NUMA_NO_NODE) 3180 nr_huge_pages = h->nr_huge_pages; 3181 else 3182 nr_huge_pages = h->nr_huge_pages_node[nid]; 3183 3184 return sysfs_emit(buf, "%lu\n", nr_huge_pages); 3185 } 3186 3187 static ssize_t __nr_hugepages_store_common(bool obey_mempolicy, 3188 struct hstate *h, int nid, 3189 unsigned long count, size_t len) 3190 { 3191 int err; 3192 nodemask_t nodes_allowed, *n_mask; 3193 3194 if (hstate_is_gigantic(h) && !gigantic_page_runtime_supported()) 3195 return -EINVAL; 3196 3197 if (nid == NUMA_NO_NODE) { 3198 /* 3199 * global hstate attribute 3200 */ 3201 if (!(obey_mempolicy && 3202 init_nodemask_of_mempolicy(&nodes_allowed))) 3203 n_mask = &node_states[N_MEMORY]; 3204 else 3205 n_mask = &nodes_allowed; 3206 } else { 3207 /* 3208 * Node specific request. count adjustment happens in 3209 * set_max_huge_pages() after acquiring hugetlb_lock. 3210 */ 3211 init_nodemask_of_node(&nodes_allowed, nid); 3212 n_mask = &nodes_allowed; 3213 } 3214 3215 err = set_max_huge_pages(h, count, nid, n_mask); 3216 3217 return err ? err : len; 3218 } 3219 3220 static ssize_t nr_hugepages_store_common(bool obey_mempolicy, 3221 struct kobject *kobj, const char *buf, 3222 size_t len) 3223 { 3224 struct hstate *h; 3225 unsigned long count; 3226 int nid; 3227 int err; 3228 3229 err = kstrtoul(buf, 10, &count); 3230 if (err) 3231 return err; 3232 3233 h = kobj_to_hstate(kobj, &nid); 3234 return __nr_hugepages_store_common(obey_mempolicy, h, nid, count, len); 3235 } 3236 3237 static ssize_t nr_hugepages_show(struct kobject *kobj, 3238 struct kobj_attribute *attr, char *buf) 3239 { 3240 return nr_hugepages_show_common(kobj, attr, buf); 3241 } 3242 3243 static ssize_t nr_hugepages_store(struct kobject *kobj, 3244 struct kobj_attribute *attr, const char *buf, size_t len) 3245 { 3246 return nr_hugepages_store_common(false, kobj, buf, len); 3247 } 3248 HSTATE_ATTR(nr_hugepages); 3249 3250 #ifdef CONFIG_NUMA 3251 3252 /* 3253 * hstate attribute for optionally mempolicy-based constraint on persistent 3254 * huge page alloc/free. 3255 */ 3256 static ssize_t nr_hugepages_mempolicy_show(struct kobject *kobj, 3257 struct kobj_attribute *attr, 3258 char *buf) 3259 { 3260 return nr_hugepages_show_common(kobj, attr, buf); 3261 } 3262 3263 static ssize_t nr_hugepages_mempolicy_store(struct kobject *kobj, 3264 struct kobj_attribute *attr, const char *buf, size_t len) 3265 { 3266 return nr_hugepages_store_common(true, kobj, buf, len); 3267 } 3268 HSTATE_ATTR(nr_hugepages_mempolicy); 3269 #endif 3270 3271 3272 static ssize_t nr_overcommit_hugepages_show(struct kobject *kobj, 3273 struct kobj_attribute *attr, char *buf) 3274 { 3275 struct hstate *h = kobj_to_hstate(kobj, NULL); 3276 return sysfs_emit(buf, "%lu\n", h->nr_overcommit_huge_pages); 3277 } 3278 3279 static ssize_t nr_overcommit_hugepages_store(struct kobject *kobj, 3280 struct kobj_attribute *attr, const char *buf, size_t count) 3281 { 3282 int err; 3283 unsigned long input; 3284 struct hstate *h = kobj_to_hstate(kobj, NULL); 3285 3286 if (hstate_is_gigantic(h)) 3287 return -EINVAL; 3288 3289 err = kstrtoul(buf, 10, &input); 3290 if (err) 3291 return err; 3292 3293 spin_lock_irq(&hugetlb_lock); 3294 h->nr_overcommit_huge_pages = input; 3295 spin_unlock_irq(&hugetlb_lock); 3296 3297 return count; 3298 } 3299 HSTATE_ATTR(nr_overcommit_hugepages); 3300 3301 static ssize_t free_hugepages_show(struct kobject *kobj, 3302 struct kobj_attribute *attr, char *buf) 3303 { 3304 struct hstate *h; 3305 unsigned long free_huge_pages; 3306 int nid; 3307 3308 h = kobj_to_hstate(kobj, &nid); 3309 if (nid == NUMA_NO_NODE) 3310 free_huge_pages = h->free_huge_pages; 3311 else 3312 free_huge_pages = h->free_huge_pages_node[nid]; 3313 3314 return sysfs_emit(buf, "%lu\n", free_huge_pages); 3315 } 3316 HSTATE_ATTR_RO(free_hugepages); 3317 3318 static ssize_t resv_hugepages_show(struct kobject *kobj, 3319 struct kobj_attribute *attr, char *buf) 3320 { 3321 struct hstate *h = kobj_to_hstate(kobj, NULL); 3322 return sysfs_emit(buf, "%lu\n", h->resv_huge_pages); 3323 } 3324 HSTATE_ATTR_RO(resv_hugepages); 3325 3326 static ssize_t surplus_hugepages_show(struct kobject *kobj, 3327 struct kobj_attribute *attr, char *buf) 3328 { 3329 struct hstate *h; 3330 unsigned long surplus_huge_pages; 3331 int nid; 3332 3333 h = kobj_to_hstate(kobj, &nid); 3334 if (nid == NUMA_NO_NODE) 3335 surplus_huge_pages = h->surplus_huge_pages; 3336 else 3337 surplus_huge_pages = h->surplus_huge_pages_node[nid]; 3338 3339 return sysfs_emit(buf, "%lu\n", surplus_huge_pages); 3340 } 3341 HSTATE_ATTR_RO(surplus_hugepages); 3342 3343 static struct attribute *hstate_attrs[] = { 3344 &nr_hugepages_attr.attr, 3345 &nr_overcommit_hugepages_attr.attr, 3346 &free_hugepages_attr.attr, 3347 &resv_hugepages_attr.attr, 3348 &surplus_hugepages_attr.attr, 3349 #ifdef CONFIG_NUMA 3350 &nr_hugepages_mempolicy_attr.attr, 3351 #endif 3352 NULL, 3353 }; 3354 3355 static const struct attribute_group hstate_attr_group = { 3356 .attrs = hstate_attrs, 3357 }; 3358 3359 static int hugetlb_sysfs_add_hstate(struct hstate *h, struct kobject *parent, 3360 struct kobject **hstate_kobjs, 3361 const struct attribute_group *hstate_attr_group) 3362 { 3363 int retval; 3364 int hi = hstate_index(h); 3365 3366 hstate_kobjs[hi] = kobject_create_and_add(h->name, parent); 3367 if (!hstate_kobjs[hi]) 3368 return -ENOMEM; 3369 3370 retval = sysfs_create_group(hstate_kobjs[hi], hstate_attr_group); 3371 if (retval) { 3372 kobject_put(hstate_kobjs[hi]); 3373 hstate_kobjs[hi] = NULL; 3374 } 3375 3376 return retval; 3377 } 3378 3379 static void __init hugetlb_sysfs_init(void) 3380 { 3381 struct hstate *h; 3382 int err; 3383 3384 hugepages_kobj = kobject_create_and_add("hugepages", mm_kobj); 3385 if (!hugepages_kobj) 3386 return; 3387 3388 for_each_hstate(h) { 3389 err = hugetlb_sysfs_add_hstate(h, hugepages_kobj, 3390 hstate_kobjs, &hstate_attr_group); 3391 if (err) 3392 pr_err("HugeTLB: Unable to add hstate %s", h->name); 3393 } 3394 } 3395 3396 #ifdef CONFIG_NUMA 3397 3398 /* 3399 * node_hstate/s - associate per node hstate attributes, via their kobjects, 3400 * with node devices in node_devices[] using a parallel array. The array 3401 * index of a node device or _hstate == node id. 3402 * This is here to avoid any static dependency of the node device driver, in 3403 * the base kernel, on the hugetlb module. 3404 */ 3405 struct node_hstate { 3406 struct kobject *hugepages_kobj; 3407 struct kobject *hstate_kobjs[HUGE_MAX_HSTATE]; 3408 }; 3409 static struct node_hstate node_hstates[MAX_NUMNODES]; 3410 3411 /* 3412 * A subset of global hstate attributes for node devices 3413 */ 3414 static struct attribute *per_node_hstate_attrs[] = { 3415 &nr_hugepages_attr.attr, 3416 &free_hugepages_attr.attr, 3417 &surplus_hugepages_attr.attr, 3418 NULL, 3419 }; 3420 3421 static const struct attribute_group per_node_hstate_attr_group = { 3422 .attrs = per_node_hstate_attrs, 3423 }; 3424 3425 /* 3426 * kobj_to_node_hstate - lookup global hstate for node device hstate attr kobj. 3427 * Returns node id via non-NULL nidp. 3428 */ 3429 static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp) 3430 { 3431 int nid; 3432 3433 for (nid = 0; nid < nr_node_ids; nid++) { 3434 struct node_hstate *nhs = &node_hstates[nid]; 3435 int i; 3436 for (i = 0; i < HUGE_MAX_HSTATE; i++) 3437 if (nhs->hstate_kobjs[i] == kobj) { 3438 if (nidp) 3439 *nidp = nid; 3440 return &hstates[i]; 3441 } 3442 } 3443 3444 BUG(); 3445 return NULL; 3446 } 3447 3448 /* 3449 * Unregister hstate attributes from a single node device. 3450 * No-op if no hstate attributes attached. 3451 */ 3452 static void hugetlb_unregister_node(struct node *node) 3453 { 3454 struct hstate *h; 3455 struct node_hstate *nhs = &node_hstates[node->dev.id]; 3456 3457 if (!nhs->hugepages_kobj) 3458 return; /* no hstate attributes */ 3459 3460 for_each_hstate(h) { 3461 int idx = hstate_index(h); 3462 if (nhs->hstate_kobjs[idx]) { 3463 kobject_put(nhs->hstate_kobjs[idx]); 3464 nhs->hstate_kobjs[idx] = NULL; 3465 } 3466 } 3467 3468 kobject_put(nhs->hugepages_kobj); 3469 nhs->hugepages_kobj = NULL; 3470 } 3471 3472 3473 /* 3474 * Register hstate attributes for a single node device. 3475 * No-op if attributes already registered. 3476 */ 3477 static void hugetlb_register_node(struct node *node) 3478 { 3479 struct hstate *h; 3480 struct node_hstate *nhs = &node_hstates[node->dev.id]; 3481 int err; 3482 3483 if (nhs->hugepages_kobj) 3484 return; /* already allocated */ 3485 3486 nhs->hugepages_kobj = kobject_create_and_add("hugepages", 3487 &node->dev.kobj); 3488 if (!nhs->hugepages_kobj) 3489 return; 3490 3491 for_each_hstate(h) { 3492 err = hugetlb_sysfs_add_hstate(h, nhs->hugepages_kobj, 3493 nhs->hstate_kobjs, 3494 &per_node_hstate_attr_group); 3495 if (err) { 3496 pr_err("HugeTLB: Unable to add hstate %s for node %d\n", 3497 h->name, node->dev.id); 3498 hugetlb_unregister_node(node); 3499 break; 3500 } 3501 } 3502 } 3503 3504 /* 3505 * hugetlb init time: register hstate attributes for all registered node 3506 * devices of nodes that have memory. All on-line nodes should have 3507 * registered their associated device by this time. 3508 */ 3509 static void __init hugetlb_register_all_nodes(void) 3510 { 3511 int nid; 3512 3513 for_each_node_state(nid, N_MEMORY) { 3514 struct node *node = node_devices[nid]; 3515 if (node->dev.id == nid) 3516 hugetlb_register_node(node); 3517 } 3518 3519 /* 3520 * Let the node device driver know we're here so it can 3521 * [un]register hstate attributes on node hotplug. 3522 */ 3523 register_hugetlbfs_with_node(hugetlb_register_node, 3524 hugetlb_unregister_node); 3525 } 3526 #else /* !CONFIG_NUMA */ 3527 3528 static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp) 3529 { 3530 BUG(); 3531 if (nidp) 3532 *nidp = -1; 3533 return NULL; 3534 } 3535 3536 static void hugetlb_register_all_nodes(void) { } 3537 3538 #endif 3539 3540 static int __init hugetlb_init(void) 3541 { 3542 int i; 3543 3544 BUILD_BUG_ON(sizeof_field(struct page, private) * BITS_PER_BYTE < 3545 __NR_HPAGEFLAGS); 3546 3547 if (!hugepages_supported()) { 3548 if (hugetlb_max_hstate || default_hstate_max_huge_pages) 3549 pr_warn("HugeTLB: huge pages not supported, ignoring associated command-line parameters\n"); 3550 return 0; 3551 } 3552 3553 /* 3554 * Make sure HPAGE_SIZE (HUGETLB_PAGE_ORDER) hstate exists. Some 3555 * architectures depend on setup being done here. 3556 */ 3557 hugetlb_add_hstate(HUGETLB_PAGE_ORDER); 3558 if (!parsed_default_hugepagesz) { 3559 /* 3560 * If we did not parse a default huge page size, set 3561 * default_hstate_idx to HPAGE_SIZE hstate. And, if the 3562 * number of huge pages for this default size was implicitly 3563 * specified, set that here as well. 3564 * Note that the implicit setting will overwrite an explicit 3565 * setting. A warning will be printed in this case. 3566 */ 3567 default_hstate_idx = hstate_index(size_to_hstate(HPAGE_SIZE)); 3568 if (default_hstate_max_huge_pages) { 3569 if (default_hstate.max_huge_pages) { 3570 char buf[32]; 3571 3572 string_get_size(huge_page_size(&default_hstate), 3573 1, STRING_UNITS_2, buf, 32); 3574 pr_warn("HugeTLB: Ignoring hugepages=%lu associated with %s page size\n", 3575 default_hstate.max_huge_pages, buf); 3576 pr_warn("HugeTLB: Using hugepages=%lu for number of default huge pages\n", 3577 default_hstate_max_huge_pages); 3578 } 3579 default_hstate.max_huge_pages = 3580 default_hstate_max_huge_pages; 3581 } 3582 } 3583 3584 hugetlb_cma_check(); 3585 hugetlb_init_hstates(); 3586 gather_bootmem_prealloc(); 3587 report_hugepages(); 3588 3589 hugetlb_sysfs_init(); 3590 hugetlb_register_all_nodes(); 3591 hugetlb_cgroup_file_init(); 3592 3593 #ifdef CONFIG_SMP 3594 num_fault_mutexes = roundup_pow_of_two(8 * num_possible_cpus()); 3595 #else 3596 num_fault_mutexes = 1; 3597 #endif 3598 hugetlb_fault_mutex_table = 3599 kmalloc_array(num_fault_mutexes, sizeof(struct mutex), 3600 GFP_KERNEL); 3601 BUG_ON(!hugetlb_fault_mutex_table); 3602 3603 for (i = 0; i < num_fault_mutexes; i++) 3604 mutex_init(&hugetlb_fault_mutex_table[i]); 3605 return 0; 3606 } 3607 subsys_initcall(hugetlb_init); 3608 3609 /* Overwritten by architectures with more huge page sizes */ 3610 bool __init __attribute((weak)) arch_hugetlb_valid_size(unsigned long size) 3611 { 3612 return size == HPAGE_SIZE; 3613 } 3614 3615 void __init hugetlb_add_hstate(unsigned int order) 3616 { 3617 struct hstate *h; 3618 unsigned long i; 3619 3620 if (size_to_hstate(PAGE_SIZE << order)) { 3621 return; 3622 } 3623 BUG_ON(hugetlb_max_hstate >= HUGE_MAX_HSTATE); 3624 BUG_ON(order == 0); 3625 h = &hstates[hugetlb_max_hstate++]; 3626 mutex_init(&h->resize_lock); 3627 h->order = order; 3628 h->mask = ~(huge_page_size(h) - 1); 3629 for (i = 0; i < MAX_NUMNODES; ++i) 3630 INIT_LIST_HEAD(&h->hugepage_freelists[i]); 3631 INIT_LIST_HEAD(&h->hugepage_activelist); 3632 h->next_nid_to_alloc = first_memory_node; 3633 h->next_nid_to_free = first_memory_node; 3634 snprintf(h->name, HSTATE_NAME_LEN, "hugepages-%lukB", 3635 huge_page_size(h)/1024); 3636 hugetlb_vmemmap_init(h); 3637 3638 parsed_hstate = h; 3639 } 3640 3641 /* 3642 * hugepages command line processing 3643 * hugepages normally follows a valid hugepagsz or default_hugepagsz 3644 * specification. If not, ignore the hugepages value. hugepages can also 3645 * be the first huge page command line option in which case it implicitly 3646 * specifies the number of huge pages for the default size. 3647 */ 3648 static int __init hugepages_setup(char *s) 3649 { 3650 unsigned long *mhp; 3651 static unsigned long *last_mhp; 3652 3653 if (!parsed_valid_hugepagesz) { 3654 pr_warn("HugeTLB: hugepages=%s does not follow a valid hugepagesz, ignoring\n", s); 3655 parsed_valid_hugepagesz = true; 3656 return 0; 3657 } 3658 3659 /* 3660 * !hugetlb_max_hstate means we haven't parsed a hugepagesz= parameter 3661 * yet, so this hugepages= parameter goes to the "default hstate". 3662 * Otherwise, it goes with the previously parsed hugepagesz or 3663 * default_hugepagesz. 3664 */ 3665 else if (!hugetlb_max_hstate) 3666 mhp = &default_hstate_max_huge_pages; 3667 else 3668 mhp = &parsed_hstate->max_huge_pages; 3669 3670 if (mhp == last_mhp) { 3671 pr_warn("HugeTLB: hugepages= specified twice without interleaving hugepagesz=, ignoring hugepages=%s\n", s); 3672 return 0; 3673 } 3674 3675 if (sscanf(s, "%lu", mhp) <= 0) 3676 *mhp = 0; 3677 3678 /* 3679 * Global state is always initialized later in hugetlb_init. 3680 * But we need to allocate gigantic hstates here early to still 3681 * use the bootmem allocator. 3682 */ 3683 if (hugetlb_max_hstate && hstate_is_gigantic(parsed_hstate)) 3684 hugetlb_hstate_alloc_pages(parsed_hstate); 3685 3686 last_mhp = mhp; 3687 3688 return 1; 3689 } 3690 __setup("hugepages=", hugepages_setup); 3691 3692 /* 3693 * hugepagesz command line processing 3694 * A specific huge page size can only be specified once with hugepagesz. 3695 * hugepagesz is followed by hugepages on the command line. The global 3696 * variable 'parsed_valid_hugepagesz' is used to determine if prior 3697 * hugepagesz argument was valid. 3698 */ 3699 static int __init hugepagesz_setup(char *s) 3700 { 3701 unsigned long size; 3702 struct hstate *h; 3703 3704 parsed_valid_hugepagesz = false; 3705 size = (unsigned long)memparse(s, NULL); 3706 3707 if (!arch_hugetlb_valid_size(size)) { 3708 pr_err("HugeTLB: unsupported hugepagesz=%s\n", s); 3709 return 0; 3710 } 3711 3712 h = size_to_hstate(size); 3713 if (h) { 3714 /* 3715 * hstate for this size already exists. This is normally 3716 * an error, but is allowed if the existing hstate is the 3717 * default hstate. More specifically, it is only allowed if 3718 * the number of huge pages for the default hstate was not 3719 * previously specified. 3720 */ 3721 if (!parsed_default_hugepagesz || h != &default_hstate || 3722 default_hstate.max_huge_pages) { 3723 pr_warn("HugeTLB: hugepagesz=%s specified twice, ignoring\n", s); 3724 return 0; 3725 } 3726 3727 /* 3728 * No need to call hugetlb_add_hstate() as hstate already 3729 * exists. But, do set parsed_hstate so that a following 3730 * hugepages= parameter will be applied to this hstate. 3731 */ 3732 parsed_hstate = h; 3733 parsed_valid_hugepagesz = true; 3734 return 1; 3735 } 3736 3737 hugetlb_add_hstate(ilog2(size) - PAGE_SHIFT); 3738 parsed_valid_hugepagesz = true; 3739 return 1; 3740 } 3741 __setup("hugepagesz=", hugepagesz_setup); 3742 3743 /* 3744 * default_hugepagesz command line input 3745 * Only one instance of default_hugepagesz allowed on command line. 3746 */ 3747 static int __init default_hugepagesz_setup(char *s) 3748 { 3749 unsigned long size; 3750 3751 parsed_valid_hugepagesz = false; 3752 if (parsed_default_hugepagesz) { 3753 pr_err("HugeTLB: default_hugepagesz previously specified, ignoring %s\n", s); 3754 return 0; 3755 } 3756 3757 size = (unsigned long)memparse(s, NULL); 3758 3759 if (!arch_hugetlb_valid_size(size)) { 3760 pr_err("HugeTLB: unsupported default_hugepagesz=%s\n", s); 3761 return 0; 3762 } 3763 3764 hugetlb_add_hstate(ilog2(size) - PAGE_SHIFT); 3765 parsed_valid_hugepagesz = true; 3766 parsed_default_hugepagesz = true; 3767 default_hstate_idx = hstate_index(size_to_hstate(size)); 3768 3769 /* 3770 * The number of default huge pages (for this size) could have been 3771 * specified as the first hugetlb parameter: hugepages=X. If so, 3772 * then default_hstate_max_huge_pages is set. If the default huge 3773 * page size is gigantic (>= MAX_ORDER), then the pages must be 3774 * allocated here from bootmem allocator. 3775 */ 3776 if (default_hstate_max_huge_pages) { 3777 default_hstate.max_huge_pages = default_hstate_max_huge_pages; 3778 if (hstate_is_gigantic(&default_hstate)) 3779 hugetlb_hstate_alloc_pages(&default_hstate); 3780 default_hstate_max_huge_pages = 0; 3781 } 3782 3783 return 1; 3784 } 3785 __setup("default_hugepagesz=", default_hugepagesz_setup); 3786 3787 static unsigned int allowed_mems_nr(struct hstate *h) 3788 { 3789 int node; 3790 unsigned int nr = 0; 3791 nodemask_t *mpol_allowed; 3792 unsigned int *array = h->free_huge_pages_node; 3793 gfp_t gfp_mask = htlb_alloc_mask(h); 3794 3795 mpol_allowed = policy_nodemask_current(gfp_mask); 3796 3797 for_each_node_mask(node, cpuset_current_mems_allowed) { 3798 if (!mpol_allowed || node_isset(node, *mpol_allowed)) 3799 nr += array[node]; 3800 } 3801 3802 return nr; 3803 } 3804 3805 #ifdef CONFIG_SYSCTL 3806 static int proc_hugetlb_doulongvec_minmax(struct ctl_table *table, int write, 3807 void *buffer, size_t *length, 3808 loff_t *ppos, unsigned long *out) 3809 { 3810 struct ctl_table dup_table; 3811 3812 /* 3813 * In order to avoid races with __do_proc_doulongvec_minmax(), we 3814 * can duplicate the @table and alter the duplicate of it. 3815 */ 3816 dup_table = *table; 3817 dup_table.data = out; 3818 3819 return proc_doulongvec_minmax(&dup_table, write, buffer, length, ppos); 3820 } 3821 3822 static int hugetlb_sysctl_handler_common(bool obey_mempolicy, 3823 struct ctl_table *table, int write, 3824 void *buffer, size_t *length, loff_t *ppos) 3825 { 3826 struct hstate *h = &default_hstate; 3827 unsigned long tmp = h->max_huge_pages; 3828 int ret; 3829 3830 if (!hugepages_supported()) 3831 return -EOPNOTSUPP; 3832 3833 ret = proc_hugetlb_doulongvec_minmax(table, write, buffer, length, ppos, 3834 &tmp); 3835 if (ret) 3836 goto out; 3837 3838 if (write) 3839 ret = __nr_hugepages_store_common(obey_mempolicy, h, 3840 NUMA_NO_NODE, tmp, *length); 3841 out: 3842 return ret; 3843 } 3844 3845 int hugetlb_sysctl_handler(struct ctl_table *table, int write, 3846 void *buffer, size_t *length, loff_t *ppos) 3847 { 3848 3849 return hugetlb_sysctl_handler_common(false, table, write, 3850 buffer, length, ppos); 3851 } 3852 3853 #ifdef CONFIG_NUMA 3854 int hugetlb_mempolicy_sysctl_handler(struct ctl_table *table, int write, 3855 void *buffer, size_t *length, loff_t *ppos) 3856 { 3857 return hugetlb_sysctl_handler_common(true, table, write, 3858 buffer, length, ppos); 3859 } 3860 #endif /* CONFIG_NUMA */ 3861 3862 int hugetlb_overcommit_handler(struct ctl_table *table, int write, 3863 void *buffer, size_t *length, loff_t *ppos) 3864 { 3865 struct hstate *h = &default_hstate; 3866 unsigned long tmp; 3867 int ret; 3868 3869 if (!hugepages_supported()) 3870 return -EOPNOTSUPP; 3871 3872 tmp = h->nr_overcommit_huge_pages; 3873 3874 if (write && hstate_is_gigantic(h)) 3875 return -EINVAL; 3876 3877 ret = proc_hugetlb_doulongvec_minmax(table, write, buffer, length, ppos, 3878 &tmp); 3879 if (ret) 3880 goto out; 3881 3882 if (write) { 3883 spin_lock_irq(&hugetlb_lock); 3884 h->nr_overcommit_huge_pages = tmp; 3885 spin_unlock_irq(&hugetlb_lock); 3886 } 3887 out: 3888 return ret; 3889 } 3890 3891 #endif /* CONFIG_SYSCTL */ 3892 3893 void hugetlb_report_meminfo(struct seq_file *m) 3894 { 3895 struct hstate *h; 3896 unsigned long total = 0; 3897 3898 if (!hugepages_supported()) 3899 return; 3900 3901 for_each_hstate(h) { 3902 unsigned long count = h->nr_huge_pages; 3903 3904 total += huge_page_size(h) * count; 3905 3906 if (h == &default_hstate) 3907 seq_printf(m, 3908 "HugePages_Total: %5lu\n" 3909 "HugePages_Free: %5lu\n" 3910 "HugePages_Rsvd: %5lu\n" 3911 "HugePages_Surp: %5lu\n" 3912 "Hugepagesize: %8lu kB\n", 3913 count, 3914 h->free_huge_pages, 3915 h->resv_huge_pages, 3916 h->surplus_huge_pages, 3917 huge_page_size(h) / SZ_1K); 3918 } 3919 3920 seq_printf(m, "Hugetlb: %8lu kB\n", total / SZ_1K); 3921 } 3922 3923 int hugetlb_report_node_meminfo(char *buf, int len, int nid) 3924 { 3925 struct hstate *h = &default_hstate; 3926 3927 if (!hugepages_supported()) 3928 return 0; 3929 3930 return sysfs_emit_at(buf, len, 3931 "Node %d HugePages_Total: %5u\n" 3932 "Node %d HugePages_Free: %5u\n" 3933 "Node %d HugePages_Surp: %5u\n", 3934 nid, h->nr_huge_pages_node[nid], 3935 nid, h->free_huge_pages_node[nid], 3936 nid, h->surplus_huge_pages_node[nid]); 3937 } 3938 3939 void hugetlb_show_meminfo(void) 3940 { 3941 struct hstate *h; 3942 int nid; 3943 3944 if (!hugepages_supported()) 3945 return; 3946 3947 for_each_node_state(nid, N_MEMORY) 3948 for_each_hstate(h) 3949 pr_info("Node %d hugepages_total=%u hugepages_free=%u hugepages_surp=%u hugepages_size=%lukB\n", 3950 nid, 3951 h->nr_huge_pages_node[nid], 3952 h->free_huge_pages_node[nid], 3953 h->surplus_huge_pages_node[nid], 3954 huge_page_size(h) / SZ_1K); 3955 } 3956 3957 void hugetlb_report_usage(struct seq_file *m, struct mm_struct *mm) 3958 { 3959 seq_printf(m, "HugetlbPages:\t%8lu kB\n", 3960 atomic_long_read(&mm->hugetlb_usage) << (PAGE_SHIFT - 10)); 3961 } 3962 3963 /* Return the number pages of memory we physically have, in PAGE_SIZE units. */ 3964 unsigned long hugetlb_total_pages(void) 3965 { 3966 struct hstate *h; 3967 unsigned long nr_total_pages = 0; 3968 3969 for_each_hstate(h) 3970 nr_total_pages += h->nr_huge_pages * pages_per_huge_page(h); 3971 return nr_total_pages; 3972 } 3973 3974 static int hugetlb_acct_memory(struct hstate *h, long delta) 3975 { 3976 int ret = -ENOMEM; 3977 3978 if (!delta) 3979 return 0; 3980 3981 spin_lock_irq(&hugetlb_lock); 3982 /* 3983 * When cpuset is configured, it breaks the strict hugetlb page 3984 * reservation as the accounting is done on a global variable. Such 3985 * reservation is completely rubbish in the presence of cpuset because 3986 * the reservation is not checked against page availability for the 3987 * current cpuset. Application can still potentially OOM'ed by kernel 3988 * with lack of free htlb page in cpuset that the task is in. 3989 * Attempt to enforce strict accounting with cpuset is almost 3990 * impossible (or too ugly) because cpuset is too fluid that 3991 * task or memory node can be dynamically moved between cpusets. 3992 * 3993 * The change of semantics for shared hugetlb mapping with cpuset is 3994 * undesirable. However, in order to preserve some of the semantics, 3995 * we fall back to check against current free page availability as 3996 * a best attempt and hopefully to minimize the impact of changing 3997 * semantics that cpuset has. 3998 * 3999 * Apart from cpuset, we also have memory policy mechanism that 4000 * also determines from which node the kernel will allocate memory 4001 * in a NUMA system. So similar to cpuset, we also should consider 4002 * the memory policy of the current task. Similar to the description 4003 * above. 4004 */ 4005 if (delta > 0) { 4006 if (gather_surplus_pages(h, delta) < 0) 4007 goto out; 4008 4009 if (delta > allowed_mems_nr(h)) { 4010 return_unused_surplus_pages(h, delta); 4011 goto out; 4012 } 4013 } 4014 4015 ret = 0; 4016 if (delta < 0) 4017 return_unused_surplus_pages(h, (unsigned long) -delta); 4018 4019 out: 4020 spin_unlock_irq(&hugetlb_lock); 4021 return ret; 4022 } 4023 4024 static void hugetlb_vm_op_open(struct vm_area_struct *vma) 4025 { 4026 struct resv_map *resv = vma_resv_map(vma); 4027 4028 /* 4029 * This new VMA should share its siblings reservation map if present. 4030 * The VMA will only ever have a valid reservation map pointer where 4031 * it is being copied for another still existing VMA. As that VMA 4032 * has a reference to the reservation map it cannot disappear until 4033 * after this open call completes. It is therefore safe to take a 4034 * new reference here without additional locking. 4035 */ 4036 if (resv && is_vma_resv_set(vma, HPAGE_RESV_OWNER)) 4037 kref_get(&resv->refs); 4038 } 4039 4040 static void hugetlb_vm_op_close(struct vm_area_struct *vma) 4041 { 4042 struct hstate *h = hstate_vma(vma); 4043 struct resv_map *resv = vma_resv_map(vma); 4044 struct hugepage_subpool *spool = subpool_vma(vma); 4045 unsigned long reserve, start, end; 4046 long gbl_reserve; 4047 4048 if (!resv || !is_vma_resv_set(vma, HPAGE_RESV_OWNER)) 4049 return; 4050 4051 start = vma_hugecache_offset(h, vma, vma->vm_start); 4052 end = vma_hugecache_offset(h, vma, vma->vm_end); 4053 4054 reserve = (end - start) - region_count(resv, start, end); 4055 hugetlb_cgroup_uncharge_counter(resv, start, end); 4056 if (reserve) { 4057 /* 4058 * Decrement reserve counts. The global reserve count may be 4059 * adjusted if the subpool has a minimum size. 4060 */ 4061 gbl_reserve = hugepage_subpool_put_pages(spool, reserve); 4062 hugetlb_acct_memory(h, -gbl_reserve); 4063 } 4064 4065 kref_put(&resv->refs, resv_map_release); 4066 } 4067 4068 static int hugetlb_vm_op_split(struct vm_area_struct *vma, unsigned long addr) 4069 { 4070 if (addr & ~(huge_page_mask(hstate_vma(vma)))) 4071 return -EINVAL; 4072 return 0; 4073 } 4074 4075 static unsigned long hugetlb_vm_op_pagesize(struct vm_area_struct *vma) 4076 { 4077 return huge_page_size(hstate_vma(vma)); 4078 } 4079 4080 /* 4081 * We cannot handle pagefaults against hugetlb pages at all. They cause 4082 * handle_mm_fault() to try to instantiate regular-sized pages in the 4083 * hugepage VMA. do_page_fault() is supposed to trap this, so BUG is we get 4084 * this far. 4085 */ 4086 static vm_fault_t hugetlb_vm_op_fault(struct vm_fault *vmf) 4087 { 4088 BUG(); 4089 return 0; 4090 } 4091 4092 /* 4093 * When a new function is introduced to vm_operations_struct and added 4094 * to hugetlb_vm_ops, please consider adding the function to shm_vm_ops. 4095 * This is because under System V memory model, mappings created via 4096 * shmget/shmat with "huge page" specified are backed by hugetlbfs files, 4097 * their original vm_ops are overwritten with shm_vm_ops. 4098 */ 4099 const struct vm_operations_struct hugetlb_vm_ops = { 4100 .fault = hugetlb_vm_op_fault, 4101 .open = hugetlb_vm_op_open, 4102 .close = hugetlb_vm_op_close, 4103 .may_split = hugetlb_vm_op_split, 4104 .pagesize = hugetlb_vm_op_pagesize, 4105 }; 4106 4107 static pte_t make_huge_pte(struct vm_area_struct *vma, struct page *page, 4108 int writable) 4109 { 4110 pte_t entry; 4111 unsigned int shift = huge_page_shift(hstate_vma(vma)); 4112 4113 if (writable) { 4114 entry = huge_pte_mkwrite(huge_pte_mkdirty(mk_huge_pte(page, 4115 vma->vm_page_prot))); 4116 } else { 4117 entry = huge_pte_wrprotect(mk_huge_pte(page, 4118 vma->vm_page_prot)); 4119 } 4120 entry = pte_mkyoung(entry); 4121 entry = pte_mkhuge(entry); 4122 entry = arch_make_huge_pte(entry, shift, vma->vm_flags); 4123 4124 return entry; 4125 } 4126 4127 static void set_huge_ptep_writable(struct vm_area_struct *vma, 4128 unsigned long address, pte_t *ptep) 4129 { 4130 pte_t entry; 4131 4132 entry = huge_pte_mkwrite(huge_pte_mkdirty(huge_ptep_get(ptep))); 4133 if (huge_ptep_set_access_flags(vma, address, ptep, entry, 1)) 4134 update_mmu_cache(vma, address, ptep); 4135 } 4136 4137 bool is_hugetlb_entry_migration(pte_t pte) 4138 { 4139 swp_entry_t swp; 4140 4141 if (huge_pte_none(pte) || pte_present(pte)) 4142 return false; 4143 swp = pte_to_swp_entry(pte); 4144 if (is_migration_entry(swp)) 4145 return true; 4146 else 4147 return false; 4148 } 4149 4150 static bool is_hugetlb_entry_hwpoisoned(pte_t pte) 4151 { 4152 swp_entry_t swp; 4153 4154 if (huge_pte_none(pte) || pte_present(pte)) 4155 return false; 4156 swp = pte_to_swp_entry(pte); 4157 if (is_hwpoison_entry(swp)) 4158 return true; 4159 else 4160 return false; 4161 } 4162 4163 static void 4164 hugetlb_install_page(struct vm_area_struct *vma, pte_t *ptep, unsigned long addr, 4165 struct page *new_page) 4166 { 4167 __SetPageUptodate(new_page); 4168 set_huge_pte_at(vma->vm_mm, addr, ptep, make_huge_pte(vma, new_page, 1)); 4169 hugepage_add_new_anon_rmap(new_page, vma, addr); 4170 hugetlb_count_add(pages_per_huge_page(hstate_vma(vma)), vma->vm_mm); 4171 ClearHPageRestoreReserve(new_page); 4172 SetHPageMigratable(new_page); 4173 } 4174 4175 int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src, 4176 struct vm_area_struct *vma) 4177 { 4178 pte_t *src_pte, *dst_pte, entry, dst_entry; 4179 struct page *ptepage; 4180 unsigned long addr; 4181 bool cow = is_cow_mapping(vma->vm_flags); 4182 struct hstate *h = hstate_vma(vma); 4183 unsigned long sz = huge_page_size(h); 4184 unsigned long npages = pages_per_huge_page(h); 4185 struct address_space *mapping = vma->vm_file->f_mapping; 4186 struct mmu_notifier_range range; 4187 int ret = 0; 4188 4189 if (cow) { 4190 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, src, 4191 vma->vm_start, 4192 vma->vm_end); 4193 mmu_notifier_invalidate_range_start(&range); 4194 } else { 4195 /* 4196 * For shared mappings i_mmap_rwsem must be held to call 4197 * huge_pte_alloc, otherwise the returned ptep could go 4198 * away if part of a shared pmd and another thread calls 4199 * huge_pmd_unshare. 4200 */ 4201 i_mmap_lock_read(mapping); 4202 } 4203 4204 for (addr = vma->vm_start; addr < vma->vm_end; addr += sz) { 4205 spinlock_t *src_ptl, *dst_ptl; 4206 src_pte = huge_pte_offset(src, addr, sz); 4207 if (!src_pte) 4208 continue; 4209 dst_pte = huge_pte_alloc(dst, vma, addr, sz); 4210 if (!dst_pte) { 4211 ret = -ENOMEM; 4212 break; 4213 } 4214 4215 /* 4216 * If the pagetables are shared don't copy or take references. 4217 * dst_pte == src_pte is the common case of src/dest sharing. 4218 * 4219 * However, src could have 'unshared' and dst shares with 4220 * another vma. If dst_pte !none, this implies sharing. 4221 * Check here before taking page table lock, and once again 4222 * after taking the lock below. 4223 */ 4224 dst_entry = huge_ptep_get(dst_pte); 4225 if ((dst_pte == src_pte) || !huge_pte_none(dst_entry)) 4226 continue; 4227 4228 dst_ptl = huge_pte_lock(h, dst, dst_pte); 4229 src_ptl = huge_pte_lockptr(h, src, src_pte); 4230 spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING); 4231 entry = huge_ptep_get(src_pte); 4232 dst_entry = huge_ptep_get(dst_pte); 4233 again: 4234 if (huge_pte_none(entry) || !huge_pte_none(dst_entry)) { 4235 /* 4236 * Skip if src entry none. Also, skip in the 4237 * unlikely case dst entry !none as this implies 4238 * sharing with another vma. 4239 */ 4240 ; 4241 } else if (unlikely(is_hugetlb_entry_migration(entry) || 4242 is_hugetlb_entry_hwpoisoned(entry))) { 4243 swp_entry_t swp_entry = pte_to_swp_entry(entry); 4244 4245 if (is_writable_migration_entry(swp_entry) && cow) { 4246 /* 4247 * COW mappings require pages in both 4248 * parent and child to be set to read. 4249 */ 4250 swp_entry = make_readable_migration_entry( 4251 swp_offset(swp_entry)); 4252 entry = swp_entry_to_pte(swp_entry); 4253 set_huge_swap_pte_at(src, addr, src_pte, 4254 entry, sz); 4255 } 4256 set_huge_swap_pte_at(dst, addr, dst_pte, entry, sz); 4257 } else { 4258 entry = huge_ptep_get(src_pte); 4259 ptepage = pte_page(entry); 4260 get_page(ptepage); 4261 4262 /* 4263 * This is a rare case where we see pinned hugetlb 4264 * pages while they're prone to COW. We need to do the 4265 * COW earlier during fork. 4266 * 4267 * When pre-allocating the page or copying data, we 4268 * need to be without the pgtable locks since we could 4269 * sleep during the process. 4270 */ 4271 if (unlikely(page_needs_cow_for_dma(vma, ptepage))) { 4272 pte_t src_pte_old = entry; 4273 struct page *new; 4274 4275 spin_unlock(src_ptl); 4276 spin_unlock(dst_ptl); 4277 /* Do not use reserve as it's private owned */ 4278 new = alloc_huge_page(vma, addr, 1); 4279 if (IS_ERR(new)) { 4280 put_page(ptepage); 4281 ret = PTR_ERR(new); 4282 break; 4283 } 4284 copy_user_huge_page(new, ptepage, addr, vma, 4285 npages); 4286 put_page(ptepage); 4287 4288 /* Install the new huge page if src pte stable */ 4289 dst_ptl = huge_pte_lock(h, dst, dst_pte); 4290 src_ptl = huge_pte_lockptr(h, src, src_pte); 4291 spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING); 4292 entry = huge_ptep_get(src_pte); 4293 if (!pte_same(src_pte_old, entry)) { 4294 restore_reserve_on_error(h, vma, addr, 4295 new); 4296 put_page(new); 4297 /* dst_entry won't change as in child */ 4298 goto again; 4299 } 4300 hugetlb_install_page(vma, dst_pte, addr, new); 4301 spin_unlock(src_ptl); 4302 spin_unlock(dst_ptl); 4303 continue; 4304 } 4305 4306 if (cow) { 4307 /* 4308 * No need to notify as we are downgrading page 4309 * table protection not changing it to point 4310 * to a new page. 4311 * 4312 * See Documentation/vm/mmu_notifier.rst 4313 */ 4314 huge_ptep_set_wrprotect(src, addr, src_pte); 4315 entry = huge_pte_wrprotect(entry); 4316 } 4317 4318 page_dup_rmap(ptepage, true); 4319 set_huge_pte_at(dst, addr, dst_pte, entry); 4320 hugetlb_count_add(npages, dst); 4321 } 4322 spin_unlock(src_ptl); 4323 spin_unlock(dst_ptl); 4324 } 4325 4326 if (cow) 4327 mmu_notifier_invalidate_range_end(&range); 4328 else 4329 i_mmap_unlock_read(mapping); 4330 4331 return ret; 4332 } 4333 4334 void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct *vma, 4335 unsigned long start, unsigned long end, 4336 struct page *ref_page) 4337 { 4338 struct mm_struct *mm = vma->vm_mm; 4339 unsigned long address; 4340 pte_t *ptep; 4341 pte_t pte; 4342 spinlock_t *ptl; 4343 struct page *page; 4344 struct hstate *h = hstate_vma(vma); 4345 unsigned long sz = huge_page_size(h); 4346 struct mmu_notifier_range range; 4347 4348 WARN_ON(!is_vm_hugetlb_page(vma)); 4349 BUG_ON(start & ~huge_page_mask(h)); 4350 BUG_ON(end & ~huge_page_mask(h)); 4351 4352 /* 4353 * This is a hugetlb vma, all the pte entries should point 4354 * to huge page. 4355 */ 4356 tlb_change_page_size(tlb, sz); 4357 tlb_start_vma(tlb, vma); 4358 4359 /* 4360 * If sharing possible, alert mmu notifiers of worst case. 4361 */ 4362 mmu_notifier_range_init(&range, MMU_NOTIFY_UNMAP, 0, vma, mm, start, 4363 end); 4364 adjust_range_if_pmd_sharing_possible(vma, &range.start, &range.end); 4365 mmu_notifier_invalidate_range_start(&range); 4366 address = start; 4367 for (; address < end; address += sz) { 4368 ptep = huge_pte_offset(mm, address, sz); 4369 if (!ptep) 4370 continue; 4371 4372 ptl = huge_pte_lock(h, mm, ptep); 4373 if (huge_pmd_unshare(mm, vma, &address, ptep)) { 4374 spin_unlock(ptl); 4375 /* 4376 * We just unmapped a page of PMDs by clearing a PUD. 4377 * The caller's TLB flush range should cover this area. 4378 */ 4379 continue; 4380 } 4381 4382 pte = huge_ptep_get(ptep); 4383 if (huge_pte_none(pte)) { 4384 spin_unlock(ptl); 4385 continue; 4386 } 4387 4388 /* 4389 * Migrating hugepage or HWPoisoned hugepage is already 4390 * unmapped and its refcount is dropped, so just clear pte here. 4391 */ 4392 if (unlikely(!pte_present(pte))) { 4393 huge_pte_clear(mm, address, ptep, sz); 4394 spin_unlock(ptl); 4395 continue; 4396 } 4397 4398 page = pte_page(pte); 4399 /* 4400 * If a reference page is supplied, it is because a specific 4401 * page is being unmapped, not a range. Ensure the page we 4402 * are about to unmap is the actual page of interest. 4403 */ 4404 if (ref_page) { 4405 if (page != ref_page) { 4406 spin_unlock(ptl); 4407 continue; 4408 } 4409 /* 4410 * Mark the VMA as having unmapped its page so that 4411 * future faults in this VMA will fail rather than 4412 * looking like data was lost 4413 */ 4414 set_vma_resv_flags(vma, HPAGE_RESV_UNMAPPED); 4415 } 4416 4417 pte = huge_ptep_get_and_clear(mm, address, ptep); 4418 tlb_remove_huge_tlb_entry(h, tlb, ptep, address); 4419 if (huge_pte_dirty(pte)) 4420 set_page_dirty(page); 4421 4422 hugetlb_count_sub(pages_per_huge_page(h), mm); 4423 page_remove_rmap(page, true); 4424 4425 spin_unlock(ptl); 4426 tlb_remove_page_size(tlb, page, huge_page_size(h)); 4427 /* 4428 * Bail out after unmapping reference page if supplied 4429 */ 4430 if (ref_page) 4431 break; 4432 } 4433 mmu_notifier_invalidate_range_end(&range); 4434 tlb_end_vma(tlb, vma); 4435 } 4436 4437 void __unmap_hugepage_range_final(struct mmu_gather *tlb, 4438 struct vm_area_struct *vma, unsigned long start, 4439 unsigned long end, struct page *ref_page) 4440 { 4441 __unmap_hugepage_range(tlb, vma, start, end, ref_page); 4442 4443 /* 4444 * Clear this flag so that x86's huge_pmd_share page_table_shareable 4445 * test will fail on a vma being torn down, and not grab a page table 4446 * on its way out. We're lucky that the flag has such an appropriate 4447 * name, and can in fact be safely cleared here. We could clear it 4448 * before the __unmap_hugepage_range above, but all that's necessary 4449 * is to clear it before releasing the i_mmap_rwsem. This works 4450 * because in the context this is called, the VMA is about to be 4451 * destroyed and the i_mmap_rwsem is held. 4452 */ 4453 vma->vm_flags &= ~VM_MAYSHARE; 4454 } 4455 4456 void unmap_hugepage_range(struct vm_area_struct *vma, unsigned long start, 4457 unsigned long end, struct page *ref_page) 4458 { 4459 struct mmu_gather tlb; 4460 4461 tlb_gather_mmu(&tlb, vma->vm_mm); 4462 __unmap_hugepage_range(&tlb, vma, start, end, ref_page); 4463 tlb_finish_mmu(&tlb); 4464 } 4465 4466 /* 4467 * This is called when the original mapper is failing to COW a MAP_PRIVATE 4468 * mapping it owns the reserve page for. The intention is to unmap the page 4469 * from other VMAs and let the children be SIGKILLed if they are faulting the 4470 * same region. 4471 */ 4472 static void unmap_ref_private(struct mm_struct *mm, struct vm_area_struct *vma, 4473 struct page *page, unsigned long address) 4474 { 4475 struct hstate *h = hstate_vma(vma); 4476 struct vm_area_struct *iter_vma; 4477 struct address_space *mapping; 4478 pgoff_t pgoff; 4479 4480 /* 4481 * vm_pgoff is in PAGE_SIZE units, hence the different calculation 4482 * from page cache lookup which is in HPAGE_SIZE units. 4483 */ 4484 address = address & huge_page_mask(h); 4485 pgoff = ((address - vma->vm_start) >> PAGE_SHIFT) + 4486 vma->vm_pgoff; 4487 mapping = vma->vm_file->f_mapping; 4488 4489 /* 4490 * Take the mapping lock for the duration of the table walk. As 4491 * this mapping should be shared between all the VMAs, 4492 * __unmap_hugepage_range() is called as the lock is already held 4493 */ 4494 i_mmap_lock_write(mapping); 4495 vma_interval_tree_foreach(iter_vma, &mapping->i_mmap, pgoff, pgoff) { 4496 /* Do not unmap the current VMA */ 4497 if (iter_vma == vma) 4498 continue; 4499 4500 /* 4501 * Shared VMAs have their own reserves and do not affect 4502 * MAP_PRIVATE accounting but it is possible that a shared 4503 * VMA is using the same page so check and skip such VMAs. 4504 */ 4505 if (iter_vma->vm_flags & VM_MAYSHARE) 4506 continue; 4507 4508 /* 4509 * Unmap the page from other VMAs without their own reserves. 4510 * They get marked to be SIGKILLed if they fault in these 4511 * areas. This is because a future no-page fault on this VMA 4512 * could insert a zeroed page instead of the data existing 4513 * from the time of fork. This would look like data corruption 4514 */ 4515 if (!is_vma_resv_set(iter_vma, HPAGE_RESV_OWNER)) 4516 unmap_hugepage_range(iter_vma, address, 4517 address + huge_page_size(h), page); 4518 } 4519 i_mmap_unlock_write(mapping); 4520 } 4521 4522 /* 4523 * Hugetlb_cow() should be called with page lock of the original hugepage held. 4524 * Called with hugetlb_instantiation_mutex held and pte_page locked so we 4525 * cannot race with other handlers or page migration. 4526 * Keep the pte_same checks anyway to make transition from the mutex easier. 4527 */ 4528 static vm_fault_t hugetlb_cow(struct mm_struct *mm, struct vm_area_struct *vma, 4529 unsigned long address, pte_t *ptep, 4530 struct page *pagecache_page, spinlock_t *ptl) 4531 { 4532 pte_t pte; 4533 struct hstate *h = hstate_vma(vma); 4534 struct page *old_page, *new_page; 4535 int outside_reserve = 0; 4536 vm_fault_t ret = 0; 4537 unsigned long haddr = address & huge_page_mask(h); 4538 struct mmu_notifier_range range; 4539 4540 pte = huge_ptep_get(ptep); 4541 old_page = pte_page(pte); 4542 4543 retry_avoidcopy: 4544 /* If no-one else is actually using this page, avoid the copy 4545 * and just make the page writable */ 4546 if (page_mapcount(old_page) == 1 && PageAnon(old_page)) { 4547 page_move_anon_rmap(old_page, vma); 4548 set_huge_ptep_writable(vma, haddr, ptep); 4549 return 0; 4550 } 4551 4552 /* 4553 * If the process that created a MAP_PRIVATE mapping is about to 4554 * perform a COW due to a shared page count, attempt to satisfy 4555 * the allocation without using the existing reserves. The pagecache 4556 * page is used to determine if the reserve at this address was 4557 * consumed or not. If reserves were used, a partial faulted mapping 4558 * at the time of fork() could consume its reserves on COW instead 4559 * of the full address range. 4560 */ 4561 if (is_vma_resv_set(vma, HPAGE_RESV_OWNER) && 4562 old_page != pagecache_page) 4563 outside_reserve = 1; 4564 4565 get_page(old_page); 4566 4567 /* 4568 * Drop page table lock as buddy allocator may be called. It will 4569 * be acquired again before returning to the caller, as expected. 4570 */ 4571 spin_unlock(ptl); 4572 new_page = alloc_huge_page(vma, haddr, outside_reserve); 4573 4574 if (IS_ERR(new_page)) { 4575 /* 4576 * If a process owning a MAP_PRIVATE mapping fails to COW, 4577 * it is due to references held by a child and an insufficient 4578 * huge page pool. To guarantee the original mappers 4579 * reliability, unmap the page from child processes. The child 4580 * may get SIGKILLed if it later faults. 4581 */ 4582 if (outside_reserve) { 4583 struct address_space *mapping = vma->vm_file->f_mapping; 4584 pgoff_t idx; 4585 u32 hash; 4586 4587 put_page(old_page); 4588 BUG_ON(huge_pte_none(pte)); 4589 /* 4590 * Drop hugetlb_fault_mutex and i_mmap_rwsem before 4591 * unmapping. unmapping needs to hold i_mmap_rwsem 4592 * in write mode. Dropping i_mmap_rwsem in read mode 4593 * here is OK as COW mappings do not interact with 4594 * PMD sharing. 4595 * 4596 * Reacquire both after unmap operation. 4597 */ 4598 idx = vma_hugecache_offset(h, vma, haddr); 4599 hash = hugetlb_fault_mutex_hash(mapping, idx); 4600 mutex_unlock(&hugetlb_fault_mutex_table[hash]); 4601 i_mmap_unlock_read(mapping); 4602 4603 unmap_ref_private(mm, vma, old_page, haddr); 4604 4605 i_mmap_lock_read(mapping); 4606 mutex_lock(&hugetlb_fault_mutex_table[hash]); 4607 spin_lock(ptl); 4608 ptep = huge_pte_offset(mm, haddr, huge_page_size(h)); 4609 if (likely(ptep && 4610 pte_same(huge_ptep_get(ptep), pte))) 4611 goto retry_avoidcopy; 4612 /* 4613 * race occurs while re-acquiring page table 4614 * lock, and our job is done. 4615 */ 4616 return 0; 4617 } 4618 4619 ret = vmf_error(PTR_ERR(new_page)); 4620 goto out_release_old; 4621 } 4622 4623 /* 4624 * When the original hugepage is shared one, it does not have 4625 * anon_vma prepared. 4626 */ 4627 if (unlikely(anon_vma_prepare(vma))) { 4628 ret = VM_FAULT_OOM; 4629 goto out_release_all; 4630 } 4631 4632 copy_user_huge_page(new_page, old_page, address, vma, 4633 pages_per_huge_page(h)); 4634 __SetPageUptodate(new_page); 4635 4636 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, mm, haddr, 4637 haddr + huge_page_size(h)); 4638 mmu_notifier_invalidate_range_start(&range); 4639 4640 /* 4641 * Retake the page table lock to check for racing updates 4642 * before the page tables are altered 4643 */ 4644 spin_lock(ptl); 4645 ptep = huge_pte_offset(mm, haddr, huge_page_size(h)); 4646 if (likely(ptep && pte_same(huge_ptep_get(ptep), pte))) { 4647 ClearHPageRestoreReserve(new_page); 4648 4649 /* Break COW */ 4650 huge_ptep_clear_flush(vma, haddr, ptep); 4651 mmu_notifier_invalidate_range(mm, range.start, range.end); 4652 set_huge_pte_at(mm, haddr, ptep, 4653 make_huge_pte(vma, new_page, 1)); 4654 page_remove_rmap(old_page, true); 4655 hugepage_add_new_anon_rmap(new_page, vma, haddr); 4656 SetHPageMigratable(new_page); 4657 /* Make the old page be freed below */ 4658 new_page = old_page; 4659 } 4660 spin_unlock(ptl); 4661 mmu_notifier_invalidate_range_end(&range); 4662 out_release_all: 4663 restore_reserve_on_error(h, vma, haddr, new_page); 4664 put_page(new_page); 4665 out_release_old: 4666 put_page(old_page); 4667 4668 spin_lock(ptl); /* Caller expects lock to be held */ 4669 return ret; 4670 } 4671 4672 /* Return the pagecache page at a given address within a VMA */ 4673 static struct page *hugetlbfs_pagecache_page(struct hstate *h, 4674 struct vm_area_struct *vma, unsigned long address) 4675 { 4676 struct address_space *mapping; 4677 pgoff_t idx; 4678 4679 mapping = vma->vm_file->f_mapping; 4680 idx = vma_hugecache_offset(h, vma, address); 4681 4682 return find_lock_page(mapping, idx); 4683 } 4684 4685 /* 4686 * Return whether there is a pagecache page to back given address within VMA. 4687 * Caller follow_hugetlb_page() holds page_table_lock so we cannot lock_page. 4688 */ 4689 static bool hugetlbfs_pagecache_present(struct hstate *h, 4690 struct vm_area_struct *vma, unsigned long address) 4691 { 4692 struct address_space *mapping; 4693 pgoff_t idx; 4694 struct page *page; 4695 4696 mapping = vma->vm_file->f_mapping; 4697 idx = vma_hugecache_offset(h, vma, address); 4698 4699 page = find_get_page(mapping, idx); 4700 if (page) 4701 put_page(page); 4702 return page != NULL; 4703 } 4704 4705 int huge_add_to_page_cache(struct page *page, struct address_space *mapping, 4706 pgoff_t idx) 4707 { 4708 struct inode *inode = mapping->host; 4709 struct hstate *h = hstate_inode(inode); 4710 int err = add_to_page_cache(page, mapping, idx, GFP_KERNEL); 4711 4712 if (err) 4713 return err; 4714 ClearHPageRestoreReserve(page); 4715 4716 /* 4717 * set page dirty so that it will not be removed from cache/file 4718 * by non-hugetlbfs specific code paths. 4719 */ 4720 set_page_dirty(page); 4721 4722 spin_lock(&inode->i_lock); 4723 inode->i_blocks += blocks_per_huge_page(h); 4724 spin_unlock(&inode->i_lock); 4725 return 0; 4726 } 4727 4728 static inline vm_fault_t hugetlb_handle_userfault(struct vm_area_struct *vma, 4729 struct address_space *mapping, 4730 pgoff_t idx, 4731 unsigned int flags, 4732 unsigned long haddr, 4733 unsigned long reason) 4734 { 4735 vm_fault_t ret; 4736 u32 hash; 4737 struct vm_fault vmf = { 4738 .vma = vma, 4739 .address = haddr, 4740 .flags = flags, 4741 4742 /* 4743 * Hard to debug if it ends up being 4744 * used by a callee that assumes 4745 * something about the other 4746 * uninitialized fields... same as in 4747 * memory.c 4748 */ 4749 }; 4750 4751 /* 4752 * hugetlb_fault_mutex and i_mmap_rwsem must be 4753 * dropped before handling userfault. Reacquire 4754 * after handling fault to make calling code simpler. 4755 */ 4756 hash = hugetlb_fault_mutex_hash(mapping, idx); 4757 mutex_unlock(&hugetlb_fault_mutex_table[hash]); 4758 i_mmap_unlock_read(mapping); 4759 ret = handle_userfault(&vmf, reason); 4760 i_mmap_lock_read(mapping); 4761 mutex_lock(&hugetlb_fault_mutex_table[hash]); 4762 4763 return ret; 4764 } 4765 4766 static vm_fault_t hugetlb_no_page(struct mm_struct *mm, 4767 struct vm_area_struct *vma, 4768 struct address_space *mapping, pgoff_t idx, 4769 unsigned long address, pte_t *ptep, unsigned int flags) 4770 { 4771 struct hstate *h = hstate_vma(vma); 4772 vm_fault_t ret = VM_FAULT_SIGBUS; 4773 int anon_rmap = 0; 4774 unsigned long size; 4775 struct page *page; 4776 pte_t new_pte; 4777 spinlock_t *ptl; 4778 unsigned long haddr = address & huge_page_mask(h); 4779 bool new_page = false; 4780 4781 /* 4782 * Currently, we are forced to kill the process in the event the 4783 * original mapper has unmapped pages from the child due to a failed 4784 * COW. Warn that such a situation has occurred as it may not be obvious 4785 */ 4786 if (is_vma_resv_set(vma, HPAGE_RESV_UNMAPPED)) { 4787 pr_warn_ratelimited("PID %d killed due to inadequate hugepage pool\n", 4788 current->pid); 4789 return ret; 4790 } 4791 4792 /* 4793 * We can not race with truncation due to holding i_mmap_rwsem. 4794 * i_size is modified when holding i_mmap_rwsem, so check here 4795 * once for faults beyond end of file. 4796 */ 4797 size = i_size_read(mapping->host) >> huge_page_shift(h); 4798 if (idx >= size) 4799 goto out; 4800 4801 retry: 4802 page = find_lock_page(mapping, idx); 4803 if (!page) { 4804 /* Check for page in userfault range */ 4805 if (userfaultfd_missing(vma)) { 4806 ret = hugetlb_handle_userfault(vma, mapping, idx, 4807 flags, haddr, 4808 VM_UFFD_MISSING); 4809 goto out; 4810 } 4811 4812 page = alloc_huge_page(vma, haddr, 0); 4813 if (IS_ERR(page)) { 4814 /* 4815 * Returning error will result in faulting task being 4816 * sent SIGBUS. The hugetlb fault mutex prevents two 4817 * tasks from racing to fault in the same page which 4818 * could result in false unable to allocate errors. 4819 * Page migration does not take the fault mutex, but 4820 * does a clear then write of pte's under page table 4821 * lock. Page fault code could race with migration, 4822 * notice the clear pte and try to allocate a page 4823 * here. Before returning error, get ptl and make 4824 * sure there really is no pte entry. 4825 */ 4826 ptl = huge_pte_lock(h, mm, ptep); 4827 ret = 0; 4828 if (huge_pte_none(huge_ptep_get(ptep))) 4829 ret = vmf_error(PTR_ERR(page)); 4830 spin_unlock(ptl); 4831 goto out; 4832 } 4833 clear_huge_page(page, address, pages_per_huge_page(h)); 4834 __SetPageUptodate(page); 4835 new_page = true; 4836 4837 if (vma->vm_flags & VM_MAYSHARE) { 4838 int err = huge_add_to_page_cache(page, mapping, idx); 4839 if (err) { 4840 put_page(page); 4841 if (err == -EEXIST) 4842 goto retry; 4843 goto out; 4844 } 4845 } else { 4846 lock_page(page); 4847 if (unlikely(anon_vma_prepare(vma))) { 4848 ret = VM_FAULT_OOM; 4849 goto backout_unlocked; 4850 } 4851 anon_rmap = 1; 4852 } 4853 } else { 4854 /* 4855 * If memory error occurs between mmap() and fault, some process 4856 * don't have hwpoisoned swap entry for errored virtual address. 4857 * So we need to block hugepage fault by PG_hwpoison bit check. 4858 */ 4859 if (unlikely(PageHWPoison(page))) { 4860 ret = VM_FAULT_HWPOISON_LARGE | 4861 VM_FAULT_SET_HINDEX(hstate_index(h)); 4862 goto backout_unlocked; 4863 } 4864 4865 /* Check for page in userfault range. */ 4866 if (userfaultfd_minor(vma)) { 4867 unlock_page(page); 4868 put_page(page); 4869 ret = hugetlb_handle_userfault(vma, mapping, idx, 4870 flags, haddr, 4871 VM_UFFD_MINOR); 4872 goto out; 4873 } 4874 } 4875 4876 /* 4877 * If we are going to COW a private mapping later, we examine the 4878 * pending reservations for this page now. This will ensure that 4879 * any allocations necessary to record that reservation occur outside 4880 * the spinlock. 4881 */ 4882 if ((flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED)) { 4883 if (vma_needs_reservation(h, vma, haddr) < 0) { 4884 ret = VM_FAULT_OOM; 4885 goto backout_unlocked; 4886 } 4887 /* Just decrements count, does not deallocate */ 4888 vma_end_reservation(h, vma, haddr); 4889 } 4890 4891 ptl = huge_pte_lock(h, mm, ptep); 4892 ret = 0; 4893 if (!huge_pte_none(huge_ptep_get(ptep))) 4894 goto backout; 4895 4896 if (anon_rmap) { 4897 ClearHPageRestoreReserve(page); 4898 hugepage_add_new_anon_rmap(page, vma, haddr); 4899 } else 4900 page_dup_rmap(page, true); 4901 new_pte = make_huge_pte(vma, page, ((vma->vm_flags & VM_WRITE) 4902 && (vma->vm_flags & VM_SHARED))); 4903 set_huge_pte_at(mm, haddr, ptep, new_pte); 4904 4905 hugetlb_count_add(pages_per_huge_page(h), mm); 4906 if ((flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED)) { 4907 /* Optimization, do the COW without a second fault */ 4908 ret = hugetlb_cow(mm, vma, address, ptep, page, ptl); 4909 } 4910 4911 spin_unlock(ptl); 4912 4913 /* 4914 * Only set HPageMigratable in newly allocated pages. Existing pages 4915 * found in the pagecache may not have HPageMigratableset if they have 4916 * been isolated for migration. 4917 */ 4918 if (new_page) 4919 SetHPageMigratable(page); 4920 4921 unlock_page(page); 4922 out: 4923 return ret; 4924 4925 backout: 4926 spin_unlock(ptl); 4927 backout_unlocked: 4928 unlock_page(page); 4929 restore_reserve_on_error(h, vma, haddr, page); 4930 put_page(page); 4931 goto out; 4932 } 4933 4934 #ifdef CONFIG_SMP 4935 u32 hugetlb_fault_mutex_hash(struct address_space *mapping, pgoff_t idx) 4936 { 4937 unsigned long key[2]; 4938 u32 hash; 4939 4940 key[0] = (unsigned long) mapping; 4941 key[1] = idx; 4942 4943 hash = jhash2((u32 *)&key, sizeof(key)/(sizeof(u32)), 0); 4944 4945 return hash & (num_fault_mutexes - 1); 4946 } 4947 #else 4948 /* 4949 * For uniprocessor systems we always use a single mutex, so just 4950 * return 0 and avoid the hashing overhead. 4951 */ 4952 u32 hugetlb_fault_mutex_hash(struct address_space *mapping, pgoff_t idx) 4953 { 4954 return 0; 4955 } 4956 #endif 4957 4958 vm_fault_t hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma, 4959 unsigned long address, unsigned int flags) 4960 { 4961 pte_t *ptep, entry; 4962 spinlock_t *ptl; 4963 vm_fault_t ret; 4964 u32 hash; 4965 pgoff_t idx; 4966 struct page *page = NULL; 4967 struct page *pagecache_page = NULL; 4968 struct hstate *h = hstate_vma(vma); 4969 struct address_space *mapping; 4970 int need_wait_lock = 0; 4971 unsigned long haddr = address & huge_page_mask(h); 4972 4973 ptep = huge_pte_offset(mm, haddr, huge_page_size(h)); 4974 if (ptep) { 4975 /* 4976 * Since we hold no locks, ptep could be stale. That is 4977 * OK as we are only making decisions based on content and 4978 * not actually modifying content here. 4979 */ 4980 entry = huge_ptep_get(ptep); 4981 if (unlikely(is_hugetlb_entry_migration(entry))) { 4982 migration_entry_wait_huge(vma, mm, ptep); 4983 return 0; 4984 } else if (unlikely(is_hugetlb_entry_hwpoisoned(entry))) 4985 return VM_FAULT_HWPOISON_LARGE | 4986 VM_FAULT_SET_HINDEX(hstate_index(h)); 4987 } 4988 4989 /* 4990 * Acquire i_mmap_rwsem before calling huge_pte_alloc and hold 4991 * until finished with ptep. This serves two purposes: 4992 * 1) It prevents huge_pmd_unshare from being called elsewhere 4993 * and making the ptep no longer valid. 4994 * 2) It synchronizes us with i_size modifications during truncation. 4995 * 4996 * ptep could have already be assigned via huge_pte_offset. That 4997 * is OK, as huge_pte_alloc will return the same value unless 4998 * something has changed. 4999 */ 5000 mapping = vma->vm_file->f_mapping; 5001 i_mmap_lock_read(mapping); 5002 ptep = huge_pte_alloc(mm, vma, haddr, huge_page_size(h)); 5003 if (!ptep) { 5004 i_mmap_unlock_read(mapping); 5005 return VM_FAULT_OOM; 5006 } 5007 5008 /* 5009 * Serialize hugepage allocation and instantiation, so that we don't 5010 * get spurious allocation failures if two CPUs race to instantiate 5011 * the same page in the page cache. 5012 */ 5013 idx = vma_hugecache_offset(h, vma, haddr); 5014 hash = hugetlb_fault_mutex_hash(mapping, idx); 5015 mutex_lock(&hugetlb_fault_mutex_table[hash]); 5016 5017 entry = huge_ptep_get(ptep); 5018 if (huge_pte_none(entry)) { 5019 ret = hugetlb_no_page(mm, vma, mapping, idx, address, ptep, flags); 5020 goto out_mutex; 5021 } 5022 5023 ret = 0; 5024 5025 /* 5026 * entry could be a migration/hwpoison entry at this point, so this 5027 * check prevents the kernel from going below assuming that we have 5028 * an active hugepage in pagecache. This goto expects the 2nd page 5029 * fault, and is_hugetlb_entry_(migration|hwpoisoned) check will 5030 * properly handle it. 5031 */ 5032 if (!pte_present(entry)) 5033 goto out_mutex; 5034 5035 /* 5036 * If we are going to COW the mapping later, we examine the pending 5037 * reservations for this page now. This will ensure that any 5038 * allocations necessary to record that reservation occur outside the 5039 * spinlock. For private mappings, we also lookup the pagecache 5040 * page now as it is used to determine if a reservation has been 5041 * consumed. 5042 */ 5043 if ((flags & FAULT_FLAG_WRITE) && !huge_pte_write(entry)) { 5044 if (vma_needs_reservation(h, vma, haddr) < 0) { 5045 ret = VM_FAULT_OOM; 5046 goto out_mutex; 5047 } 5048 /* Just decrements count, does not deallocate */ 5049 vma_end_reservation(h, vma, haddr); 5050 5051 if (!(vma->vm_flags & VM_MAYSHARE)) 5052 pagecache_page = hugetlbfs_pagecache_page(h, 5053 vma, haddr); 5054 } 5055 5056 ptl = huge_pte_lock(h, mm, ptep); 5057 5058 /* Check for a racing update before calling hugetlb_cow */ 5059 if (unlikely(!pte_same(entry, huge_ptep_get(ptep)))) 5060 goto out_ptl; 5061 5062 /* 5063 * hugetlb_cow() requires page locks of pte_page(entry) and 5064 * pagecache_page, so here we need take the former one 5065 * when page != pagecache_page or !pagecache_page. 5066 */ 5067 page = pte_page(entry); 5068 if (page != pagecache_page) 5069 if (!trylock_page(page)) { 5070 need_wait_lock = 1; 5071 goto out_ptl; 5072 } 5073 5074 get_page(page); 5075 5076 if (flags & FAULT_FLAG_WRITE) { 5077 if (!huge_pte_write(entry)) { 5078 ret = hugetlb_cow(mm, vma, address, ptep, 5079 pagecache_page, ptl); 5080 goto out_put_page; 5081 } 5082 entry = huge_pte_mkdirty(entry); 5083 } 5084 entry = pte_mkyoung(entry); 5085 if (huge_ptep_set_access_flags(vma, haddr, ptep, entry, 5086 flags & FAULT_FLAG_WRITE)) 5087 update_mmu_cache(vma, haddr, ptep); 5088 out_put_page: 5089 if (page != pagecache_page) 5090 unlock_page(page); 5091 put_page(page); 5092 out_ptl: 5093 spin_unlock(ptl); 5094 5095 if (pagecache_page) { 5096 unlock_page(pagecache_page); 5097 put_page(pagecache_page); 5098 } 5099 out_mutex: 5100 mutex_unlock(&hugetlb_fault_mutex_table[hash]); 5101 i_mmap_unlock_read(mapping); 5102 /* 5103 * Generally it's safe to hold refcount during waiting page lock. But 5104 * here we just wait to defer the next page fault to avoid busy loop and 5105 * the page is not used after unlocked before returning from the current 5106 * page fault. So we are safe from accessing freed page, even if we wait 5107 * here without taking refcount. 5108 */ 5109 if (need_wait_lock) 5110 wait_on_page_locked(page); 5111 return ret; 5112 } 5113 5114 #ifdef CONFIG_USERFAULTFD 5115 /* 5116 * Used by userfaultfd UFFDIO_COPY. Based on mcopy_atomic_pte with 5117 * modifications for huge pages. 5118 */ 5119 int hugetlb_mcopy_atomic_pte(struct mm_struct *dst_mm, 5120 pte_t *dst_pte, 5121 struct vm_area_struct *dst_vma, 5122 unsigned long dst_addr, 5123 unsigned long src_addr, 5124 enum mcopy_atomic_mode mode, 5125 struct page **pagep) 5126 { 5127 bool is_continue = (mode == MCOPY_ATOMIC_CONTINUE); 5128 struct hstate *h = hstate_vma(dst_vma); 5129 struct address_space *mapping = dst_vma->vm_file->f_mapping; 5130 pgoff_t idx = vma_hugecache_offset(h, dst_vma, dst_addr); 5131 unsigned long size; 5132 int vm_shared = dst_vma->vm_flags & VM_SHARED; 5133 pte_t _dst_pte; 5134 spinlock_t *ptl; 5135 int ret = -ENOMEM; 5136 struct page *page; 5137 int writable; 5138 5139 if (is_continue) { 5140 ret = -EFAULT; 5141 page = find_lock_page(mapping, idx); 5142 if (!page) 5143 goto out; 5144 } else if (!*pagep) { 5145 /* If a page already exists, then it's UFFDIO_COPY for 5146 * a non-missing case. Return -EEXIST. 5147 */ 5148 if (vm_shared && 5149 hugetlbfs_pagecache_present(h, dst_vma, dst_addr)) { 5150 ret = -EEXIST; 5151 goto out; 5152 } 5153 5154 page = alloc_huge_page(dst_vma, dst_addr, 0); 5155 if (IS_ERR(page)) { 5156 ret = -ENOMEM; 5157 goto out; 5158 } 5159 5160 ret = copy_huge_page_from_user(page, 5161 (const void __user *) src_addr, 5162 pages_per_huge_page(h), false); 5163 5164 /* fallback to copy_from_user outside mmap_lock */ 5165 if (unlikely(ret)) { 5166 ret = -ENOENT; 5167 /* Free the allocated page which may have 5168 * consumed a reservation. 5169 */ 5170 restore_reserve_on_error(h, dst_vma, dst_addr, page); 5171 put_page(page); 5172 5173 /* Allocate a temporary page to hold the copied 5174 * contents. 5175 */ 5176 page = alloc_huge_page_vma(h, dst_vma, dst_addr); 5177 if (!page) { 5178 ret = -ENOMEM; 5179 goto out; 5180 } 5181 *pagep = page; 5182 /* Set the outparam pagep and return to the caller to 5183 * copy the contents outside the lock. Don't free the 5184 * page. 5185 */ 5186 goto out; 5187 } 5188 } else { 5189 if (vm_shared && 5190 hugetlbfs_pagecache_present(h, dst_vma, dst_addr)) { 5191 put_page(*pagep); 5192 ret = -EEXIST; 5193 *pagep = NULL; 5194 goto out; 5195 } 5196 5197 page = alloc_huge_page(dst_vma, dst_addr, 0); 5198 if (IS_ERR(page)) { 5199 ret = -ENOMEM; 5200 *pagep = NULL; 5201 goto out; 5202 } 5203 copy_huge_page(page, *pagep); 5204 put_page(*pagep); 5205 *pagep = NULL; 5206 } 5207 5208 /* 5209 * The memory barrier inside __SetPageUptodate makes sure that 5210 * preceding stores to the page contents become visible before 5211 * the set_pte_at() write. 5212 */ 5213 __SetPageUptodate(page); 5214 5215 /* Add shared, newly allocated pages to the page cache. */ 5216 if (vm_shared && !is_continue) { 5217 size = i_size_read(mapping->host) >> huge_page_shift(h); 5218 ret = -EFAULT; 5219 if (idx >= size) 5220 goto out_release_nounlock; 5221 5222 /* 5223 * Serialization between remove_inode_hugepages() and 5224 * huge_add_to_page_cache() below happens through the 5225 * hugetlb_fault_mutex_table that here must be hold by 5226 * the caller. 5227 */ 5228 ret = huge_add_to_page_cache(page, mapping, idx); 5229 if (ret) 5230 goto out_release_nounlock; 5231 } 5232 5233 ptl = huge_pte_lockptr(h, dst_mm, dst_pte); 5234 spin_lock(ptl); 5235 5236 /* 5237 * Recheck the i_size after holding PT lock to make sure not 5238 * to leave any page mapped (as page_mapped()) beyond the end 5239 * of the i_size (remove_inode_hugepages() is strict about 5240 * enforcing that). If we bail out here, we'll also leave a 5241 * page in the radix tree in the vm_shared case beyond the end 5242 * of the i_size, but remove_inode_hugepages() will take care 5243 * of it as soon as we drop the hugetlb_fault_mutex_table. 5244 */ 5245 size = i_size_read(mapping->host) >> huge_page_shift(h); 5246 ret = -EFAULT; 5247 if (idx >= size) 5248 goto out_release_unlock; 5249 5250 ret = -EEXIST; 5251 if (!huge_pte_none(huge_ptep_get(dst_pte))) 5252 goto out_release_unlock; 5253 5254 if (vm_shared) { 5255 page_dup_rmap(page, true); 5256 } else { 5257 ClearHPageRestoreReserve(page); 5258 hugepage_add_new_anon_rmap(page, dst_vma, dst_addr); 5259 } 5260 5261 /* For CONTINUE on a non-shared VMA, don't set VM_WRITE for CoW. */ 5262 if (is_continue && !vm_shared) 5263 writable = 0; 5264 else 5265 writable = dst_vma->vm_flags & VM_WRITE; 5266 5267 _dst_pte = make_huge_pte(dst_vma, page, writable); 5268 if (writable) 5269 _dst_pte = huge_pte_mkdirty(_dst_pte); 5270 _dst_pte = pte_mkyoung(_dst_pte); 5271 5272 set_huge_pte_at(dst_mm, dst_addr, dst_pte, _dst_pte); 5273 5274 (void)huge_ptep_set_access_flags(dst_vma, dst_addr, dst_pte, _dst_pte, 5275 dst_vma->vm_flags & VM_WRITE); 5276 hugetlb_count_add(pages_per_huge_page(h), dst_mm); 5277 5278 /* No need to invalidate - it was non-present before */ 5279 update_mmu_cache(dst_vma, dst_addr, dst_pte); 5280 5281 spin_unlock(ptl); 5282 if (!is_continue) 5283 SetHPageMigratable(page); 5284 if (vm_shared || is_continue) 5285 unlock_page(page); 5286 ret = 0; 5287 out: 5288 return ret; 5289 out_release_unlock: 5290 spin_unlock(ptl); 5291 if (vm_shared || is_continue) 5292 unlock_page(page); 5293 out_release_nounlock: 5294 restore_reserve_on_error(h, dst_vma, dst_addr, page); 5295 put_page(page); 5296 goto out; 5297 } 5298 #endif /* CONFIG_USERFAULTFD */ 5299 5300 static void record_subpages_vmas(struct page *page, struct vm_area_struct *vma, 5301 int refs, struct page **pages, 5302 struct vm_area_struct **vmas) 5303 { 5304 int nr; 5305 5306 for (nr = 0; nr < refs; nr++) { 5307 if (likely(pages)) 5308 pages[nr] = mem_map_offset(page, nr); 5309 if (vmas) 5310 vmas[nr] = vma; 5311 } 5312 } 5313 5314 long follow_hugetlb_page(struct mm_struct *mm, struct vm_area_struct *vma, 5315 struct page **pages, struct vm_area_struct **vmas, 5316 unsigned long *position, unsigned long *nr_pages, 5317 long i, unsigned int flags, int *locked) 5318 { 5319 unsigned long pfn_offset; 5320 unsigned long vaddr = *position; 5321 unsigned long remainder = *nr_pages; 5322 struct hstate *h = hstate_vma(vma); 5323 int err = -EFAULT, refs; 5324 5325 while (vaddr < vma->vm_end && remainder) { 5326 pte_t *pte; 5327 spinlock_t *ptl = NULL; 5328 int absent; 5329 struct page *page; 5330 5331 /* 5332 * If we have a pending SIGKILL, don't keep faulting pages and 5333 * potentially allocating memory. 5334 */ 5335 if (fatal_signal_pending(current)) { 5336 remainder = 0; 5337 break; 5338 } 5339 5340 /* 5341 * Some archs (sparc64, sh*) have multiple pte_ts to 5342 * each hugepage. We have to make sure we get the 5343 * first, for the page indexing below to work. 5344 * 5345 * Note that page table lock is not held when pte is null. 5346 */ 5347 pte = huge_pte_offset(mm, vaddr & huge_page_mask(h), 5348 huge_page_size(h)); 5349 if (pte) 5350 ptl = huge_pte_lock(h, mm, pte); 5351 absent = !pte || huge_pte_none(huge_ptep_get(pte)); 5352 5353 /* 5354 * When coredumping, it suits get_dump_page if we just return 5355 * an error where there's an empty slot with no huge pagecache 5356 * to back it. This way, we avoid allocating a hugepage, and 5357 * the sparse dumpfile avoids allocating disk blocks, but its 5358 * huge holes still show up with zeroes where they need to be. 5359 */ 5360 if (absent && (flags & FOLL_DUMP) && 5361 !hugetlbfs_pagecache_present(h, vma, vaddr)) { 5362 if (pte) 5363 spin_unlock(ptl); 5364 remainder = 0; 5365 break; 5366 } 5367 5368 /* 5369 * We need call hugetlb_fault for both hugepages under migration 5370 * (in which case hugetlb_fault waits for the migration,) and 5371 * hwpoisoned hugepages (in which case we need to prevent the 5372 * caller from accessing to them.) In order to do this, we use 5373 * here is_swap_pte instead of is_hugetlb_entry_migration and 5374 * is_hugetlb_entry_hwpoisoned. This is because it simply covers 5375 * both cases, and because we can't follow correct pages 5376 * directly from any kind of swap entries. 5377 */ 5378 if (absent || is_swap_pte(huge_ptep_get(pte)) || 5379 ((flags & FOLL_WRITE) && 5380 !huge_pte_write(huge_ptep_get(pte)))) { 5381 vm_fault_t ret; 5382 unsigned int fault_flags = 0; 5383 5384 if (pte) 5385 spin_unlock(ptl); 5386 if (flags & FOLL_WRITE) 5387 fault_flags |= FAULT_FLAG_WRITE; 5388 if (locked) 5389 fault_flags |= FAULT_FLAG_ALLOW_RETRY | 5390 FAULT_FLAG_KILLABLE; 5391 if (flags & FOLL_NOWAIT) 5392 fault_flags |= FAULT_FLAG_ALLOW_RETRY | 5393 FAULT_FLAG_RETRY_NOWAIT; 5394 if (flags & FOLL_TRIED) { 5395 /* 5396 * Note: FAULT_FLAG_ALLOW_RETRY and 5397 * FAULT_FLAG_TRIED can co-exist 5398 */ 5399 fault_flags |= FAULT_FLAG_TRIED; 5400 } 5401 ret = hugetlb_fault(mm, vma, vaddr, fault_flags); 5402 if (ret & VM_FAULT_ERROR) { 5403 err = vm_fault_to_errno(ret, flags); 5404 remainder = 0; 5405 break; 5406 } 5407 if (ret & VM_FAULT_RETRY) { 5408 if (locked && 5409 !(fault_flags & FAULT_FLAG_RETRY_NOWAIT)) 5410 *locked = 0; 5411 *nr_pages = 0; 5412 /* 5413 * VM_FAULT_RETRY must not return an 5414 * error, it will return zero 5415 * instead. 5416 * 5417 * No need to update "position" as the 5418 * caller will not check it after 5419 * *nr_pages is set to 0. 5420 */ 5421 return i; 5422 } 5423 continue; 5424 } 5425 5426 pfn_offset = (vaddr & ~huge_page_mask(h)) >> PAGE_SHIFT; 5427 page = pte_page(huge_ptep_get(pte)); 5428 5429 /* 5430 * If subpage information not requested, update counters 5431 * and skip the same_page loop below. 5432 */ 5433 if (!pages && !vmas && !pfn_offset && 5434 (vaddr + huge_page_size(h) < vma->vm_end) && 5435 (remainder >= pages_per_huge_page(h))) { 5436 vaddr += huge_page_size(h); 5437 remainder -= pages_per_huge_page(h); 5438 i += pages_per_huge_page(h); 5439 spin_unlock(ptl); 5440 continue; 5441 } 5442 5443 refs = min3(pages_per_huge_page(h) - pfn_offset, 5444 (vma->vm_end - vaddr) >> PAGE_SHIFT, remainder); 5445 5446 if (pages || vmas) 5447 record_subpages_vmas(mem_map_offset(page, pfn_offset), 5448 vma, refs, 5449 likely(pages) ? pages + i : NULL, 5450 vmas ? vmas + i : NULL); 5451 5452 if (pages) { 5453 /* 5454 * try_grab_compound_head() should always succeed here, 5455 * because: a) we hold the ptl lock, and b) we've just 5456 * checked that the huge page is present in the page 5457 * tables. If the huge page is present, then the tail 5458 * pages must also be present. The ptl prevents the 5459 * head page and tail pages from being rearranged in 5460 * any way. So this page must be available at this 5461 * point, unless the page refcount overflowed: 5462 */ 5463 if (WARN_ON_ONCE(!try_grab_compound_head(pages[i], 5464 refs, 5465 flags))) { 5466 spin_unlock(ptl); 5467 remainder = 0; 5468 err = -ENOMEM; 5469 break; 5470 } 5471 } 5472 5473 vaddr += (refs << PAGE_SHIFT); 5474 remainder -= refs; 5475 i += refs; 5476 5477 spin_unlock(ptl); 5478 } 5479 *nr_pages = remainder; 5480 /* 5481 * setting position is actually required only if remainder is 5482 * not zero but it's faster not to add a "if (remainder)" 5483 * branch. 5484 */ 5485 *position = vaddr; 5486 5487 return i ? i : err; 5488 } 5489 5490 unsigned long hugetlb_change_protection(struct vm_area_struct *vma, 5491 unsigned long address, unsigned long end, pgprot_t newprot) 5492 { 5493 struct mm_struct *mm = vma->vm_mm; 5494 unsigned long start = address; 5495 pte_t *ptep; 5496 pte_t pte; 5497 struct hstate *h = hstate_vma(vma); 5498 unsigned long pages = 0; 5499 bool shared_pmd = false; 5500 struct mmu_notifier_range range; 5501 5502 /* 5503 * In the case of shared PMDs, the area to flush could be beyond 5504 * start/end. Set range.start/range.end to cover the maximum possible 5505 * range if PMD sharing is possible. 5506 */ 5507 mmu_notifier_range_init(&range, MMU_NOTIFY_PROTECTION_VMA, 5508 0, vma, mm, start, end); 5509 adjust_range_if_pmd_sharing_possible(vma, &range.start, &range.end); 5510 5511 BUG_ON(address >= end); 5512 flush_cache_range(vma, range.start, range.end); 5513 5514 mmu_notifier_invalidate_range_start(&range); 5515 i_mmap_lock_write(vma->vm_file->f_mapping); 5516 for (; address < end; address += huge_page_size(h)) { 5517 spinlock_t *ptl; 5518 ptep = huge_pte_offset(mm, address, huge_page_size(h)); 5519 if (!ptep) 5520 continue; 5521 ptl = huge_pte_lock(h, mm, ptep); 5522 if (huge_pmd_unshare(mm, vma, &address, ptep)) { 5523 pages++; 5524 spin_unlock(ptl); 5525 shared_pmd = true; 5526 continue; 5527 } 5528 pte = huge_ptep_get(ptep); 5529 if (unlikely(is_hugetlb_entry_hwpoisoned(pte))) { 5530 spin_unlock(ptl); 5531 continue; 5532 } 5533 if (unlikely(is_hugetlb_entry_migration(pte))) { 5534 swp_entry_t entry = pte_to_swp_entry(pte); 5535 5536 if (is_writable_migration_entry(entry)) { 5537 pte_t newpte; 5538 5539 entry = make_readable_migration_entry( 5540 swp_offset(entry)); 5541 newpte = swp_entry_to_pte(entry); 5542 set_huge_swap_pte_at(mm, address, ptep, 5543 newpte, huge_page_size(h)); 5544 pages++; 5545 } 5546 spin_unlock(ptl); 5547 continue; 5548 } 5549 if (!huge_pte_none(pte)) { 5550 pte_t old_pte; 5551 unsigned int shift = huge_page_shift(hstate_vma(vma)); 5552 5553 old_pte = huge_ptep_modify_prot_start(vma, address, ptep); 5554 pte = pte_mkhuge(huge_pte_modify(old_pte, newprot)); 5555 pte = arch_make_huge_pte(pte, shift, vma->vm_flags); 5556 huge_ptep_modify_prot_commit(vma, address, ptep, old_pte, pte); 5557 pages++; 5558 } 5559 spin_unlock(ptl); 5560 } 5561 /* 5562 * Must flush TLB before releasing i_mmap_rwsem: x86's huge_pmd_unshare 5563 * may have cleared our pud entry and done put_page on the page table: 5564 * once we release i_mmap_rwsem, another task can do the final put_page 5565 * and that page table be reused and filled with junk. If we actually 5566 * did unshare a page of pmds, flush the range corresponding to the pud. 5567 */ 5568 if (shared_pmd) 5569 flush_hugetlb_tlb_range(vma, range.start, range.end); 5570 else 5571 flush_hugetlb_tlb_range(vma, start, end); 5572 /* 5573 * No need to call mmu_notifier_invalidate_range() we are downgrading 5574 * page table protection not changing it to point to a new page. 5575 * 5576 * See Documentation/vm/mmu_notifier.rst 5577 */ 5578 i_mmap_unlock_write(vma->vm_file->f_mapping); 5579 mmu_notifier_invalidate_range_end(&range); 5580 5581 return pages << h->order; 5582 } 5583 5584 /* Return true if reservation was successful, false otherwise. */ 5585 bool hugetlb_reserve_pages(struct inode *inode, 5586 long from, long to, 5587 struct vm_area_struct *vma, 5588 vm_flags_t vm_flags) 5589 { 5590 long chg, add = -1; 5591 struct hstate *h = hstate_inode(inode); 5592 struct hugepage_subpool *spool = subpool_inode(inode); 5593 struct resv_map *resv_map; 5594 struct hugetlb_cgroup *h_cg = NULL; 5595 long gbl_reserve, regions_needed = 0; 5596 5597 /* This should never happen */ 5598 if (from > to) { 5599 VM_WARN(1, "%s called with a negative range\n", __func__); 5600 return false; 5601 } 5602 5603 /* 5604 * Only apply hugepage reservation if asked. At fault time, an 5605 * attempt will be made for VM_NORESERVE to allocate a page 5606 * without using reserves 5607 */ 5608 if (vm_flags & VM_NORESERVE) 5609 return true; 5610 5611 /* 5612 * Shared mappings base their reservation on the number of pages that 5613 * are already allocated on behalf of the file. Private mappings need 5614 * to reserve the full area even if read-only as mprotect() may be 5615 * called to make the mapping read-write. Assume !vma is a shm mapping 5616 */ 5617 if (!vma || vma->vm_flags & VM_MAYSHARE) { 5618 /* 5619 * resv_map can not be NULL as hugetlb_reserve_pages is only 5620 * called for inodes for which resv_maps were created (see 5621 * hugetlbfs_get_inode). 5622 */ 5623 resv_map = inode_resv_map(inode); 5624 5625 chg = region_chg(resv_map, from, to, ®ions_needed); 5626 5627 } else { 5628 /* Private mapping. */ 5629 resv_map = resv_map_alloc(); 5630 if (!resv_map) 5631 return false; 5632 5633 chg = to - from; 5634 5635 set_vma_resv_map(vma, resv_map); 5636 set_vma_resv_flags(vma, HPAGE_RESV_OWNER); 5637 } 5638 5639 if (chg < 0) 5640 goto out_err; 5641 5642 if (hugetlb_cgroup_charge_cgroup_rsvd(hstate_index(h), 5643 chg * pages_per_huge_page(h), &h_cg) < 0) 5644 goto out_err; 5645 5646 if (vma && !(vma->vm_flags & VM_MAYSHARE) && h_cg) { 5647 /* For private mappings, the hugetlb_cgroup uncharge info hangs 5648 * of the resv_map. 5649 */ 5650 resv_map_set_hugetlb_cgroup_uncharge_info(resv_map, h_cg, h); 5651 } 5652 5653 /* 5654 * There must be enough pages in the subpool for the mapping. If 5655 * the subpool has a minimum size, there may be some global 5656 * reservations already in place (gbl_reserve). 5657 */ 5658 gbl_reserve = hugepage_subpool_get_pages(spool, chg); 5659 if (gbl_reserve < 0) 5660 goto out_uncharge_cgroup; 5661 5662 /* 5663 * Check enough hugepages are available for the reservation. 5664 * Hand the pages back to the subpool if there are not 5665 */ 5666 if (hugetlb_acct_memory(h, gbl_reserve) < 0) 5667 goto out_put_pages; 5668 5669 /* 5670 * Account for the reservations made. Shared mappings record regions 5671 * that have reservations as they are shared by multiple VMAs. 5672 * When the last VMA disappears, the region map says how much 5673 * the reservation was and the page cache tells how much of 5674 * the reservation was consumed. Private mappings are per-VMA and 5675 * only the consumed reservations are tracked. When the VMA 5676 * disappears, the original reservation is the VMA size and the 5677 * consumed reservations are stored in the map. Hence, nothing 5678 * else has to be done for private mappings here 5679 */ 5680 if (!vma || vma->vm_flags & VM_MAYSHARE) { 5681 add = region_add(resv_map, from, to, regions_needed, h, h_cg); 5682 5683 if (unlikely(add < 0)) { 5684 hugetlb_acct_memory(h, -gbl_reserve); 5685 goto out_put_pages; 5686 } else if (unlikely(chg > add)) { 5687 /* 5688 * pages in this range were added to the reserve 5689 * map between region_chg and region_add. This 5690 * indicates a race with alloc_huge_page. Adjust 5691 * the subpool and reserve counts modified above 5692 * based on the difference. 5693 */ 5694 long rsv_adjust; 5695 5696 /* 5697 * hugetlb_cgroup_uncharge_cgroup_rsvd() will put the 5698 * reference to h_cg->css. See comment below for detail. 5699 */ 5700 hugetlb_cgroup_uncharge_cgroup_rsvd( 5701 hstate_index(h), 5702 (chg - add) * pages_per_huge_page(h), h_cg); 5703 5704 rsv_adjust = hugepage_subpool_put_pages(spool, 5705 chg - add); 5706 hugetlb_acct_memory(h, -rsv_adjust); 5707 } else if (h_cg) { 5708 /* 5709 * The file_regions will hold their own reference to 5710 * h_cg->css. So we should release the reference held 5711 * via hugetlb_cgroup_charge_cgroup_rsvd() when we are 5712 * done. 5713 */ 5714 hugetlb_cgroup_put_rsvd_cgroup(h_cg); 5715 } 5716 } 5717 return true; 5718 5719 out_put_pages: 5720 /* put back original number of pages, chg */ 5721 (void)hugepage_subpool_put_pages(spool, chg); 5722 out_uncharge_cgroup: 5723 hugetlb_cgroup_uncharge_cgroup_rsvd(hstate_index(h), 5724 chg * pages_per_huge_page(h), h_cg); 5725 out_err: 5726 if (!vma || vma->vm_flags & VM_MAYSHARE) 5727 /* Only call region_abort if the region_chg succeeded but the 5728 * region_add failed or didn't run. 5729 */ 5730 if (chg >= 0 && add < 0) 5731 region_abort(resv_map, from, to, regions_needed); 5732 if (vma && is_vma_resv_set(vma, HPAGE_RESV_OWNER)) 5733 kref_put(&resv_map->refs, resv_map_release); 5734 return false; 5735 } 5736 5737 long hugetlb_unreserve_pages(struct inode *inode, long start, long end, 5738 long freed) 5739 { 5740 struct hstate *h = hstate_inode(inode); 5741 struct resv_map *resv_map = inode_resv_map(inode); 5742 long chg = 0; 5743 struct hugepage_subpool *spool = subpool_inode(inode); 5744 long gbl_reserve; 5745 5746 /* 5747 * Since this routine can be called in the evict inode path for all 5748 * hugetlbfs inodes, resv_map could be NULL. 5749 */ 5750 if (resv_map) { 5751 chg = region_del(resv_map, start, end); 5752 /* 5753 * region_del() can fail in the rare case where a region 5754 * must be split and another region descriptor can not be 5755 * allocated. If end == LONG_MAX, it will not fail. 5756 */ 5757 if (chg < 0) 5758 return chg; 5759 } 5760 5761 spin_lock(&inode->i_lock); 5762 inode->i_blocks -= (blocks_per_huge_page(h) * freed); 5763 spin_unlock(&inode->i_lock); 5764 5765 /* 5766 * If the subpool has a minimum size, the number of global 5767 * reservations to be released may be adjusted. 5768 * 5769 * Note that !resv_map implies freed == 0. So (chg - freed) 5770 * won't go negative. 5771 */ 5772 gbl_reserve = hugepage_subpool_put_pages(spool, (chg - freed)); 5773 hugetlb_acct_memory(h, -gbl_reserve); 5774 5775 return 0; 5776 } 5777 5778 #ifdef CONFIG_ARCH_WANT_HUGE_PMD_SHARE 5779 static unsigned long page_table_shareable(struct vm_area_struct *svma, 5780 struct vm_area_struct *vma, 5781 unsigned long addr, pgoff_t idx) 5782 { 5783 unsigned long saddr = ((idx - svma->vm_pgoff) << PAGE_SHIFT) + 5784 svma->vm_start; 5785 unsigned long sbase = saddr & PUD_MASK; 5786 unsigned long s_end = sbase + PUD_SIZE; 5787 5788 /* Allow segments to share if only one is marked locked */ 5789 unsigned long vm_flags = vma->vm_flags & VM_LOCKED_CLEAR_MASK; 5790 unsigned long svm_flags = svma->vm_flags & VM_LOCKED_CLEAR_MASK; 5791 5792 /* 5793 * match the virtual addresses, permission and the alignment of the 5794 * page table page. 5795 */ 5796 if (pmd_index(addr) != pmd_index(saddr) || 5797 vm_flags != svm_flags || 5798 !range_in_vma(svma, sbase, s_end)) 5799 return 0; 5800 5801 return saddr; 5802 } 5803 5804 static bool vma_shareable(struct vm_area_struct *vma, unsigned long addr) 5805 { 5806 unsigned long base = addr & PUD_MASK; 5807 unsigned long end = base + PUD_SIZE; 5808 5809 /* 5810 * check on proper vm_flags and page table alignment 5811 */ 5812 if (vma->vm_flags & VM_MAYSHARE && range_in_vma(vma, base, end)) 5813 return true; 5814 return false; 5815 } 5816 5817 bool want_pmd_share(struct vm_area_struct *vma, unsigned long addr) 5818 { 5819 #ifdef CONFIG_USERFAULTFD 5820 if (uffd_disable_huge_pmd_share(vma)) 5821 return false; 5822 #endif 5823 return vma_shareable(vma, addr); 5824 } 5825 5826 /* 5827 * Determine if start,end range within vma could be mapped by shared pmd. 5828 * If yes, adjust start and end to cover range associated with possible 5829 * shared pmd mappings. 5830 */ 5831 void adjust_range_if_pmd_sharing_possible(struct vm_area_struct *vma, 5832 unsigned long *start, unsigned long *end) 5833 { 5834 unsigned long v_start = ALIGN(vma->vm_start, PUD_SIZE), 5835 v_end = ALIGN_DOWN(vma->vm_end, PUD_SIZE); 5836 5837 /* 5838 * vma needs to span at least one aligned PUD size, and the range 5839 * must be at least partially within in. 5840 */ 5841 if (!(vma->vm_flags & VM_MAYSHARE) || !(v_end > v_start) || 5842 (*end <= v_start) || (*start >= v_end)) 5843 return; 5844 5845 /* Extend the range to be PUD aligned for a worst case scenario */ 5846 if (*start > v_start) 5847 *start = ALIGN_DOWN(*start, PUD_SIZE); 5848 5849 if (*end < v_end) 5850 *end = ALIGN(*end, PUD_SIZE); 5851 } 5852 5853 /* 5854 * Search for a shareable pmd page for hugetlb. In any case calls pmd_alloc() 5855 * and returns the corresponding pte. While this is not necessary for the 5856 * !shared pmd case because we can allocate the pmd later as well, it makes the 5857 * code much cleaner. 5858 * 5859 * This routine must be called with i_mmap_rwsem held in at least read mode if 5860 * sharing is possible. For hugetlbfs, this prevents removal of any page 5861 * table entries associated with the address space. This is important as we 5862 * are setting up sharing based on existing page table entries (mappings). 5863 * 5864 * NOTE: This routine is only called from huge_pte_alloc. Some callers of 5865 * huge_pte_alloc know that sharing is not possible and do not take 5866 * i_mmap_rwsem as a performance optimization. This is handled by the 5867 * if !vma_shareable check at the beginning of the routine. i_mmap_rwsem is 5868 * only required for subsequent processing. 5869 */ 5870 pte_t *huge_pmd_share(struct mm_struct *mm, struct vm_area_struct *vma, 5871 unsigned long addr, pud_t *pud) 5872 { 5873 struct address_space *mapping = vma->vm_file->f_mapping; 5874 pgoff_t idx = ((addr - vma->vm_start) >> PAGE_SHIFT) + 5875 vma->vm_pgoff; 5876 struct vm_area_struct *svma; 5877 unsigned long saddr; 5878 pte_t *spte = NULL; 5879 pte_t *pte; 5880 spinlock_t *ptl; 5881 5882 i_mmap_assert_locked(mapping); 5883 vma_interval_tree_foreach(svma, &mapping->i_mmap, idx, idx) { 5884 if (svma == vma) 5885 continue; 5886 5887 saddr = page_table_shareable(svma, vma, addr, idx); 5888 if (saddr) { 5889 spte = huge_pte_offset(svma->vm_mm, saddr, 5890 vma_mmu_pagesize(svma)); 5891 if (spte) { 5892 get_page(virt_to_page(spte)); 5893 break; 5894 } 5895 } 5896 } 5897 5898 if (!spte) 5899 goto out; 5900 5901 ptl = huge_pte_lock(hstate_vma(vma), mm, spte); 5902 if (pud_none(*pud)) { 5903 pud_populate(mm, pud, 5904 (pmd_t *)((unsigned long)spte & PAGE_MASK)); 5905 mm_inc_nr_pmds(mm); 5906 } else { 5907 put_page(virt_to_page(spte)); 5908 } 5909 spin_unlock(ptl); 5910 out: 5911 pte = (pte_t *)pmd_alloc(mm, pud, addr); 5912 return pte; 5913 } 5914 5915 /* 5916 * unmap huge page backed by shared pte. 5917 * 5918 * Hugetlb pte page is ref counted at the time of mapping. If pte is shared 5919 * indicated by page_count > 1, unmap is achieved by clearing pud and 5920 * decrementing the ref count. If count == 1, the pte page is not shared. 5921 * 5922 * Called with page table lock held and i_mmap_rwsem held in write mode. 5923 * 5924 * returns: 1 successfully unmapped a shared pte page 5925 * 0 the underlying pte page is not shared, or it is the last user 5926 */ 5927 int huge_pmd_unshare(struct mm_struct *mm, struct vm_area_struct *vma, 5928 unsigned long *addr, pte_t *ptep) 5929 { 5930 pgd_t *pgd = pgd_offset(mm, *addr); 5931 p4d_t *p4d = p4d_offset(pgd, *addr); 5932 pud_t *pud = pud_offset(p4d, *addr); 5933 5934 i_mmap_assert_write_locked(vma->vm_file->f_mapping); 5935 BUG_ON(page_count(virt_to_page(ptep)) == 0); 5936 if (page_count(virt_to_page(ptep)) == 1) 5937 return 0; 5938 5939 pud_clear(pud); 5940 put_page(virt_to_page(ptep)); 5941 mm_dec_nr_pmds(mm); 5942 *addr = ALIGN(*addr, HPAGE_SIZE * PTRS_PER_PTE) - HPAGE_SIZE; 5943 return 1; 5944 } 5945 5946 #else /* !CONFIG_ARCH_WANT_HUGE_PMD_SHARE */ 5947 pte_t *huge_pmd_share(struct mm_struct *mm, struct vm_area_struct *vma, 5948 unsigned long addr, pud_t *pud) 5949 { 5950 return NULL; 5951 } 5952 5953 int huge_pmd_unshare(struct mm_struct *mm, struct vm_area_struct *vma, 5954 unsigned long *addr, pte_t *ptep) 5955 { 5956 return 0; 5957 } 5958 5959 void adjust_range_if_pmd_sharing_possible(struct vm_area_struct *vma, 5960 unsigned long *start, unsigned long *end) 5961 { 5962 } 5963 5964 bool want_pmd_share(struct vm_area_struct *vma, unsigned long addr) 5965 { 5966 return false; 5967 } 5968 #endif /* CONFIG_ARCH_WANT_HUGE_PMD_SHARE */ 5969 5970 #ifdef CONFIG_ARCH_WANT_GENERAL_HUGETLB 5971 pte_t *huge_pte_alloc(struct mm_struct *mm, struct vm_area_struct *vma, 5972 unsigned long addr, unsigned long sz) 5973 { 5974 pgd_t *pgd; 5975 p4d_t *p4d; 5976 pud_t *pud; 5977 pte_t *pte = NULL; 5978 5979 pgd = pgd_offset(mm, addr); 5980 p4d = p4d_alloc(mm, pgd, addr); 5981 if (!p4d) 5982 return NULL; 5983 pud = pud_alloc(mm, p4d, addr); 5984 if (pud) { 5985 if (sz == PUD_SIZE) { 5986 pte = (pte_t *)pud; 5987 } else { 5988 BUG_ON(sz != PMD_SIZE); 5989 if (want_pmd_share(vma, addr) && pud_none(*pud)) 5990 pte = huge_pmd_share(mm, vma, addr, pud); 5991 else 5992 pte = (pte_t *)pmd_alloc(mm, pud, addr); 5993 } 5994 } 5995 BUG_ON(pte && pte_present(*pte) && !pte_huge(*pte)); 5996 5997 return pte; 5998 } 5999 6000 /* 6001 * huge_pte_offset() - Walk the page table to resolve the hugepage 6002 * entry at address @addr 6003 * 6004 * Return: Pointer to page table entry (PUD or PMD) for 6005 * address @addr, or NULL if a !p*d_present() entry is encountered and the 6006 * size @sz doesn't match the hugepage size at this level of the page 6007 * table. 6008 */ 6009 pte_t *huge_pte_offset(struct mm_struct *mm, 6010 unsigned long addr, unsigned long sz) 6011 { 6012 pgd_t *pgd; 6013 p4d_t *p4d; 6014 pud_t *pud; 6015 pmd_t *pmd; 6016 6017 pgd = pgd_offset(mm, addr); 6018 if (!pgd_present(*pgd)) 6019 return NULL; 6020 p4d = p4d_offset(pgd, addr); 6021 if (!p4d_present(*p4d)) 6022 return NULL; 6023 6024 pud = pud_offset(p4d, addr); 6025 if (sz == PUD_SIZE) 6026 /* must be pud huge, non-present or none */ 6027 return (pte_t *)pud; 6028 if (!pud_present(*pud)) 6029 return NULL; 6030 /* must have a valid entry and size to go further */ 6031 6032 pmd = pmd_offset(pud, addr); 6033 /* must be pmd huge, non-present or none */ 6034 return (pte_t *)pmd; 6035 } 6036 6037 #endif /* CONFIG_ARCH_WANT_GENERAL_HUGETLB */ 6038 6039 /* 6040 * These functions are overwritable if your architecture needs its own 6041 * behavior. 6042 */ 6043 struct page * __weak 6044 follow_huge_addr(struct mm_struct *mm, unsigned long address, 6045 int write) 6046 { 6047 return ERR_PTR(-EINVAL); 6048 } 6049 6050 struct page * __weak 6051 follow_huge_pd(struct vm_area_struct *vma, 6052 unsigned long address, hugepd_t hpd, int flags, int pdshift) 6053 { 6054 WARN(1, "hugepd follow called with no support for hugepage directory format\n"); 6055 return NULL; 6056 } 6057 6058 struct page * __weak 6059 follow_huge_pmd(struct mm_struct *mm, unsigned long address, 6060 pmd_t *pmd, int flags) 6061 { 6062 struct page *page = NULL; 6063 spinlock_t *ptl; 6064 pte_t pte; 6065 6066 /* FOLL_GET and FOLL_PIN are mutually exclusive. */ 6067 if (WARN_ON_ONCE((flags & (FOLL_PIN | FOLL_GET)) == 6068 (FOLL_PIN | FOLL_GET))) 6069 return NULL; 6070 6071 retry: 6072 ptl = pmd_lockptr(mm, pmd); 6073 spin_lock(ptl); 6074 /* 6075 * make sure that the address range covered by this pmd is not 6076 * unmapped from other threads. 6077 */ 6078 if (!pmd_huge(*pmd)) 6079 goto out; 6080 pte = huge_ptep_get((pte_t *)pmd); 6081 if (pte_present(pte)) { 6082 page = pmd_page(*pmd) + ((address & ~PMD_MASK) >> PAGE_SHIFT); 6083 /* 6084 * try_grab_page() should always succeed here, because: a) we 6085 * hold the pmd (ptl) lock, and b) we've just checked that the 6086 * huge pmd (head) page is present in the page tables. The ptl 6087 * prevents the head page and tail pages from being rearranged 6088 * in any way. So this page must be available at this point, 6089 * unless the page refcount overflowed: 6090 */ 6091 if (WARN_ON_ONCE(!try_grab_page(page, flags))) { 6092 page = NULL; 6093 goto out; 6094 } 6095 } else { 6096 if (is_hugetlb_entry_migration(pte)) { 6097 spin_unlock(ptl); 6098 __migration_entry_wait(mm, (pte_t *)pmd, ptl); 6099 goto retry; 6100 } 6101 /* 6102 * hwpoisoned entry is treated as no_page_table in 6103 * follow_page_mask(). 6104 */ 6105 } 6106 out: 6107 spin_unlock(ptl); 6108 return page; 6109 } 6110 6111 struct page * __weak 6112 follow_huge_pud(struct mm_struct *mm, unsigned long address, 6113 pud_t *pud, int flags) 6114 { 6115 if (flags & (FOLL_GET | FOLL_PIN)) 6116 return NULL; 6117 6118 return pte_page(*(pte_t *)pud) + ((address & ~PUD_MASK) >> PAGE_SHIFT); 6119 } 6120 6121 struct page * __weak 6122 follow_huge_pgd(struct mm_struct *mm, unsigned long address, pgd_t *pgd, int flags) 6123 { 6124 if (flags & (FOLL_GET | FOLL_PIN)) 6125 return NULL; 6126 6127 return pte_page(*(pte_t *)pgd) + ((address & ~PGDIR_MASK) >> PAGE_SHIFT); 6128 } 6129 6130 bool isolate_huge_page(struct page *page, struct list_head *list) 6131 { 6132 bool ret = true; 6133 6134 spin_lock_irq(&hugetlb_lock); 6135 if (!PageHeadHuge(page) || 6136 !HPageMigratable(page) || 6137 !get_page_unless_zero(page)) { 6138 ret = false; 6139 goto unlock; 6140 } 6141 ClearHPageMigratable(page); 6142 list_move_tail(&page->lru, list); 6143 unlock: 6144 spin_unlock_irq(&hugetlb_lock); 6145 return ret; 6146 } 6147 6148 int get_hwpoison_huge_page(struct page *page, bool *hugetlb) 6149 { 6150 int ret = 0; 6151 6152 *hugetlb = false; 6153 spin_lock_irq(&hugetlb_lock); 6154 if (PageHeadHuge(page)) { 6155 *hugetlb = true; 6156 if (HPageFreed(page) || HPageMigratable(page)) 6157 ret = get_page_unless_zero(page); 6158 else 6159 ret = -EBUSY; 6160 } 6161 spin_unlock_irq(&hugetlb_lock); 6162 return ret; 6163 } 6164 6165 void putback_active_hugepage(struct page *page) 6166 { 6167 spin_lock_irq(&hugetlb_lock); 6168 SetHPageMigratable(page); 6169 list_move_tail(&page->lru, &(page_hstate(page))->hugepage_activelist); 6170 spin_unlock_irq(&hugetlb_lock); 6171 put_page(page); 6172 } 6173 6174 void move_hugetlb_state(struct page *oldpage, struct page *newpage, int reason) 6175 { 6176 struct hstate *h = page_hstate(oldpage); 6177 6178 hugetlb_cgroup_migrate(oldpage, newpage); 6179 set_page_owner_migrate_reason(newpage, reason); 6180 6181 /* 6182 * transfer temporary state of the new huge page. This is 6183 * reverse to other transitions because the newpage is going to 6184 * be final while the old one will be freed so it takes over 6185 * the temporary status. 6186 * 6187 * Also note that we have to transfer the per-node surplus state 6188 * here as well otherwise the global surplus count will not match 6189 * the per-node's. 6190 */ 6191 if (HPageTemporary(newpage)) { 6192 int old_nid = page_to_nid(oldpage); 6193 int new_nid = page_to_nid(newpage); 6194 6195 SetHPageTemporary(oldpage); 6196 ClearHPageTemporary(newpage); 6197 6198 /* 6199 * There is no need to transfer the per-node surplus state 6200 * when we do not cross the node. 6201 */ 6202 if (new_nid == old_nid) 6203 return; 6204 spin_lock_irq(&hugetlb_lock); 6205 if (h->surplus_huge_pages_node[old_nid]) { 6206 h->surplus_huge_pages_node[old_nid]--; 6207 h->surplus_huge_pages_node[new_nid]++; 6208 } 6209 spin_unlock_irq(&hugetlb_lock); 6210 } 6211 } 6212 6213 /* 6214 * This function will unconditionally remove all the shared pmd pgtable entries 6215 * within the specific vma for a hugetlbfs memory range. 6216 */ 6217 void hugetlb_unshare_all_pmds(struct vm_area_struct *vma) 6218 { 6219 struct hstate *h = hstate_vma(vma); 6220 unsigned long sz = huge_page_size(h); 6221 struct mm_struct *mm = vma->vm_mm; 6222 struct mmu_notifier_range range; 6223 unsigned long address, start, end; 6224 spinlock_t *ptl; 6225 pte_t *ptep; 6226 6227 if (!(vma->vm_flags & VM_MAYSHARE)) 6228 return; 6229 6230 start = ALIGN(vma->vm_start, PUD_SIZE); 6231 end = ALIGN_DOWN(vma->vm_end, PUD_SIZE); 6232 6233 if (start >= end) 6234 return; 6235 6236 /* 6237 * No need to call adjust_range_if_pmd_sharing_possible(), because 6238 * we have already done the PUD_SIZE alignment. 6239 */ 6240 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, mm, 6241 start, end); 6242 mmu_notifier_invalidate_range_start(&range); 6243 i_mmap_lock_write(vma->vm_file->f_mapping); 6244 for (address = start; address < end; address += PUD_SIZE) { 6245 unsigned long tmp = address; 6246 6247 ptep = huge_pte_offset(mm, address, sz); 6248 if (!ptep) 6249 continue; 6250 ptl = huge_pte_lock(h, mm, ptep); 6251 /* We don't want 'address' to be changed */ 6252 huge_pmd_unshare(mm, vma, &tmp, ptep); 6253 spin_unlock(ptl); 6254 } 6255 flush_hugetlb_tlb_range(vma, start, end); 6256 i_mmap_unlock_write(vma->vm_file->f_mapping); 6257 /* 6258 * No need to call mmu_notifier_invalidate_range(), see 6259 * Documentation/vm/mmu_notifier.rst. 6260 */ 6261 mmu_notifier_invalidate_range_end(&range); 6262 } 6263 6264 #ifdef CONFIG_CMA 6265 static bool cma_reserve_called __initdata; 6266 6267 static int __init cmdline_parse_hugetlb_cma(char *p) 6268 { 6269 hugetlb_cma_size = memparse(p, &p); 6270 return 0; 6271 } 6272 6273 early_param("hugetlb_cma", cmdline_parse_hugetlb_cma); 6274 6275 void __init hugetlb_cma_reserve(int order) 6276 { 6277 unsigned long size, reserved, per_node; 6278 int nid; 6279 6280 cma_reserve_called = true; 6281 6282 if (!hugetlb_cma_size) 6283 return; 6284 6285 if (hugetlb_cma_size < (PAGE_SIZE << order)) { 6286 pr_warn("hugetlb_cma: cma area should be at least %lu MiB\n", 6287 (PAGE_SIZE << order) / SZ_1M); 6288 return; 6289 } 6290 6291 /* 6292 * If 3 GB area is requested on a machine with 4 numa nodes, 6293 * let's allocate 1 GB on first three nodes and ignore the last one. 6294 */ 6295 per_node = DIV_ROUND_UP(hugetlb_cma_size, nr_online_nodes); 6296 pr_info("hugetlb_cma: reserve %lu MiB, up to %lu MiB per node\n", 6297 hugetlb_cma_size / SZ_1M, per_node / SZ_1M); 6298 6299 reserved = 0; 6300 for_each_node_state(nid, N_ONLINE) { 6301 int res; 6302 char name[CMA_MAX_NAME]; 6303 6304 size = min(per_node, hugetlb_cma_size - reserved); 6305 size = round_up(size, PAGE_SIZE << order); 6306 6307 snprintf(name, sizeof(name), "hugetlb%d", nid); 6308 res = cma_declare_contiguous_nid(0, size, 0, PAGE_SIZE << order, 6309 0, false, name, 6310 &hugetlb_cma[nid], nid); 6311 if (res) { 6312 pr_warn("hugetlb_cma: reservation failed: err %d, node %d", 6313 res, nid); 6314 continue; 6315 } 6316 6317 reserved += size; 6318 pr_info("hugetlb_cma: reserved %lu MiB on node %d\n", 6319 size / SZ_1M, nid); 6320 6321 if (reserved >= hugetlb_cma_size) 6322 break; 6323 } 6324 } 6325 6326 void __init hugetlb_cma_check(void) 6327 { 6328 if (!hugetlb_cma_size || cma_reserve_called) 6329 return; 6330 6331 pr_warn("hugetlb_cma: the option isn't supported by current arch\n"); 6332 } 6333 6334 #endif /* CONFIG_CMA */ 6335