1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Generic hugetlb support. 4 * (C) Nadia Yvette Chambers, April 2004 5 */ 6 #include <linux/list.h> 7 #include <linux/init.h> 8 #include <linux/mm.h> 9 #include <linux/seq_file.h> 10 #include <linux/sysctl.h> 11 #include <linux/highmem.h> 12 #include <linux/mmu_notifier.h> 13 #include <linux/nodemask.h> 14 #include <linux/pagemap.h> 15 #include <linux/mempolicy.h> 16 #include <linux/compiler.h> 17 #include <linux/cpuset.h> 18 #include <linux/mutex.h> 19 #include <linux/memblock.h> 20 #include <linux/sysfs.h> 21 #include <linux/slab.h> 22 #include <linux/mmdebug.h> 23 #include <linux/sched/signal.h> 24 #include <linux/rmap.h> 25 #include <linux/string_helpers.h> 26 #include <linux/swap.h> 27 #include <linux/swapops.h> 28 #include <linux/jhash.h> 29 #include <linux/numa.h> 30 #include <linux/llist.h> 31 32 #include <asm/page.h> 33 #include <asm/pgtable.h> 34 #include <asm/tlb.h> 35 36 #include <linux/io.h> 37 #include <linux/hugetlb.h> 38 #include <linux/hugetlb_cgroup.h> 39 #include <linux/node.h> 40 #include <linux/userfaultfd_k.h> 41 #include <linux/page_owner.h> 42 #include "internal.h" 43 44 int hugetlb_max_hstate __read_mostly; 45 unsigned int default_hstate_idx; 46 struct hstate hstates[HUGE_MAX_HSTATE]; 47 /* 48 * Minimum page order among possible hugepage sizes, set to a proper value 49 * at boot time. 50 */ 51 static unsigned int minimum_order __read_mostly = UINT_MAX; 52 53 __initdata LIST_HEAD(huge_boot_pages); 54 55 /* for command line parsing */ 56 static struct hstate * __initdata parsed_hstate; 57 static unsigned long __initdata default_hstate_max_huge_pages; 58 static unsigned long __initdata default_hstate_size; 59 static bool __initdata parsed_valid_hugepagesz = true; 60 61 /* 62 * Protects updates to hugepage_freelists, hugepage_activelist, nr_huge_pages, 63 * free_huge_pages, and surplus_huge_pages. 64 */ 65 DEFINE_SPINLOCK(hugetlb_lock); 66 67 /* 68 * Serializes faults on the same logical page. This is used to 69 * prevent spurious OOMs when the hugepage pool is fully utilized. 70 */ 71 static int num_fault_mutexes; 72 struct mutex *hugetlb_fault_mutex_table ____cacheline_aligned_in_smp; 73 74 /* Forward declaration */ 75 static int hugetlb_acct_memory(struct hstate *h, long delta); 76 77 static inline void unlock_or_release_subpool(struct hugepage_subpool *spool) 78 { 79 bool free = (spool->count == 0) && (spool->used_hpages == 0); 80 81 spin_unlock(&spool->lock); 82 83 /* If no pages are used, and no other handles to the subpool 84 * remain, give up any reservations mased on minimum size and 85 * free the subpool */ 86 if (free) { 87 if (spool->min_hpages != -1) 88 hugetlb_acct_memory(spool->hstate, 89 -spool->min_hpages); 90 kfree(spool); 91 } 92 } 93 94 struct hugepage_subpool *hugepage_new_subpool(struct hstate *h, long max_hpages, 95 long min_hpages) 96 { 97 struct hugepage_subpool *spool; 98 99 spool = kzalloc(sizeof(*spool), GFP_KERNEL); 100 if (!spool) 101 return NULL; 102 103 spin_lock_init(&spool->lock); 104 spool->count = 1; 105 spool->max_hpages = max_hpages; 106 spool->hstate = h; 107 spool->min_hpages = min_hpages; 108 109 if (min_hpages != -1 && hugetlb_acct_memory(h, min_hpages)) { 110 kfree(spool); 111 return NULL; 112 } 113 spool->rsv_hpages = min_hpages; 114 115 return spool; 116 } 117 118 void hugepage_put_subpool(struct hugepage_subpool *spool) 119 { 120 spin_lock(&spool->lock); 121 BUG_ON(!spool->count); 122 spool->count--; 123 unlock_or_release_subpool(spool); 124 } 125 126 /* 127 * Subpool accounting for allocating and reserving pages. 128 * Return -ENOMEM if there are not enough resources to satisfy the 129 * the request. Otherwise, return the number of pages by which the 130 * global pools must be adjusted (upward). The returned value may 131 * only be different than the passed value (delta) in the case where 132 * a subpool minimum size must be manitained. 133 */ 134 static long hugepage_subpool_get_pages(struct hugepage_subpool *spool, 135 long delta) 136 { 137 long ret = delta; 138 139 if (!spool) 140 return ret; 141 142 spin_lock(&spool->lock); 143 144 if (spool->max_hpages != -1) { /* maximum size accounting */ 145 if ((spool->used_hpages + delta) <= spool->max_hpages) 146 spool->used_hpages += delta; 147 else { 148 ret = -ENOMEM; 149 goto unlock_ret; 150 } 151 } 152 153 /* minimum size accounting */ 154 if (spool->min_hpages != -1 && spool->rsv_hpages) { 155 if (delta > spool->rsv_hpages) { 156 /* 157 * Asking for more reserves than those already taken on 158 * behalf of subpool. Return difference. 159 */ 160 ret = delta - spool->rsv_hpages; 161 spool->rsv_hpages = 0; 162 } else { 163 ret = 0; /* reserves already accounted for */ 164 spool->rsv_hpages -= delta; 165 } 166 } 167 168 unlock_ret: 169 spin_unlock(&spool->lock); 170 return ret; 171 } 172 173 /* 174 * Subpool accounting for freeing and unreserving pages. 175 * Return the number of global page reservations that must be dropped. 176 * The return value may only be different than the passed value (delta) 177 * in the case where a subpool minimum size must be maintained. 178 */ 179 static long hugepage_subpool_put_pages(struct hugepage_subpool *spool, 180 long delta) 181 { 182 long ret = delta; 183 184 if (!spool) 185 return delta; 186 187 spin_lock(&spool->lock); 188 189 if (spool->max_hpages != -1) /* maximum size accounting */ 190 spool->used_hpages -= delta; 191 192 /* minimum size accounting */ 193 if (spool->min_hpages != -1 && spool->used_hpages < spool->min_hpages) { 194 if (spool->rsv_hpages + delta <= spool->min_hpages) 195 ret = 0; 196 else 197 ret = spool->rsv_hpages + delta - spool->min_hpages; 198 199 spool->rsv_hpages += delta; 200 if (spool->rsv_hpages > spool->min_hpages) 201 spool->rsv_hpages = spool->min_hpages; 202 } 203 204 /* 205 * If hugetlbfs_put_super couldn't free spool due to an outstanding 206 * quota reference, free it now. 207 */ 208 unlock_or_release_subpool(spool); 209 210 return ret; 211 } 212 213 static inline struct hugepage_subpool *subpool_inode(struct inode *inode) 214 { 215 return HUGETLBFS_SB(inode->i_sb)->spool; 216 } 217 218 static inline struct hugepage_subpool *subpool_vma(struct vm_area_struct *vma) 219 { 220 return subpool_inode(file_inode(vma->vm_file)); 221 } 222 223 /* Helper that removes a struct file_region from the resv_map cache and returns 224 * it for use. 225 */ 226 static struct file_region * 227 get_file_region_entry_from_cache(struct resv_map *resv, long from, long to) 228 { 229 struct file_region *nrg = NULL; 230 231 VM_BUG_ON(resv->region_cache_count <= 0); 232 233 resv->region_cache_count--; 234 nrg = list_first_entry(&resv->region_cache, struct file_region, link); 235 VM_BUG_ON(!nrg); 236 list_del(&nrg->link); 237 238 nrg->from = from; 239 nrg->to = to; 240 241 return nrg; 242 } 243 244 static void copy_hugetlb_cgroup_uncharge_info(struct file_region *nrg, 245 struct file_region *rg) 246 { 247 #ifdef CONFIG_CGROUP_HUGETLB 248 nrg->reservation_counter = rg->reservation_counter; 249 nrg->css = rg->css; 250 if (rg->css) 251 css_get(rg->css); 252 #endif 253 } 254 255 /* Helper that records hugetlb_cgroup uncharge info. */ 256 static void record_hugetlb_cgroup_uncharge_info(struct hugetlb_cgroup *h_cg, 257 struct hstate *h, 258 struct resv_map *resv, 259 struct file_region *nrg) 260 { 261 #ifdef CONFIG_CGROUP_HUGETLB 262 if (h_cg) { 263 nrg->reservation_counter = 264 &h_cg->rsvd_hugepage[hstate_index(h)]; 265 nrg->css = &h_cg->css; 266 if (!resv->pages_per_hpage) 267 resv->pages_per_hpage = pages_per_huge_page(h); 268 /* pages_per_hpage should be the same for all entries in 269 * a resv_map. 270 */ 271 VM_BUG_ON(resv->pages_per_hpage != pages_per_huge_page(h)); 272 } else { 273 nrg->reservation_counter = NULL; 274 nrg->css = NULL; 275 } 276 #endif 277 } 278 279 static bool has_same_uncharge_info(struct file_region *rg, 280 struct file_region *org) 281 { 282 #ifdef CONFIG_CGROUP_HUGETLB 283 return rg && org && 284 rg->reservation_counter == org->reservation_counter && 285 rg->css == org->css; 286 287 #else 288 return true; 289 #endif 290 } 291 292 static void coalesce_file_region(struct resv_map *resv, struct file_region *rg) 293 { 294 struct file_region *nrg = NULL, *prg = NULL; 295 296 prg = list_prev_entry(rg, link); 297 if (&prg->link != &resv->regions && prg->to == rg->from && 298 has_same_uncharge_info(prg, rg)) { 299 prg->to = rg->to; 300 301 list_del(&rg->link); 302 kfree(rg); 303 304 coalesce_file_region(resv, prg); 305 return; 306 } 307 308 nrg = list_next_entry(rg, link); 309 if (&nrg->link != &resv->regions && nrg->from == rg->to && 310 has_same_uncharge_info(nrg, rg)) { 311 nrg->from = rg->from; 312 313 list_del(&rg->link); 314 kfree(rg); 315 316 coalesce_file_region(resv, nrg); 317 return; 318 } 319 } 320 321 /* Must be called with resv->lock held. Calling this with count_only == true 322 * will count the number of pages to be added but will not modify the linked 323 * list. If regions_needed != NULL and count_only == true, then regions_needed 324 * will indicate the number of file_regions needed in the cache to carry out to 325 * add the regions for this range. 326 */ 327 static long add_reservation_in_range(struct resv_map *resv, long f, long t, 328 struct hugetlb_cgroup *h_cg, 329 struct hstate *h, long *regions_needed, 330 bool count_only) 331 { 332 long add = 0; 333 struct list_head *head = &resv->regions; 334 long last_accounted_offset = f; 335 struct file_region *rg = NULL, *trg = NULL, *nrg = NULL; 336 337 if (regions_needed) 338 *regions_needed = 0; 339 340 /* In this loop, we essentially handle an entry for the range 341 * [last_accounted_offset, rg->from), at every iteration, with some 342 * bounds checking. 343 */ 344 list_for_each_entry_safe(rg, trg, head, link) { 345 /* Skip irrelevant regions that start before our range. */ 346 if (rg->from < f) { 347 /* If this region ends after the last accounted offset, 348 * then we need to update last_accounted_offset. 349 */ 350 if (rg->to > last_accounted_offset) 351 last_accounted_offset = rg->to; 352 continue; 353 } 354 355 /* When we find a region that starts beyond our range, we've 356 * finished. 357 */ 358 if (rg->from > t) 359 break; 360 361 /* Add an entry for last_accounted_offset -> rg->from, and 362 * update last_accounted_offset. 363 */ 364 if (rg->from > last_accounted_offset) { 365 add += rg->from - last_accounted_offset; 366 if (!count_only) { 367 nrg = get_file_region_entry_from_cache( 368 resv, last_accounted_offset, rg->from); 369 record_hugetlb_cgroup_uncharge_info(h_cg, h, 370 resv, nrg); 371 list_add(&nrg->link, rg->link.prev); 372 coalesce_file_region(resv, nrg); 373 } else if (regions_needed) 374 *regions_needed += 1; 375 } 376 377 last_accounted_offset = rg->to; 378 } 379 380 /* Handle the case where our range extends beyond 381 * last_accounted_offset. 382 */ 383 if (last_accounted_offset < t) { 384 add += t - last_accounted_offset; 385 if (!count_only) { 386 nrg = get_file_region_entry_from_cache( 387 resv, last_accounted_offset, t); 388 record_hugetlb_cgroup_uncharge_info(h_cg, h, resv, nrg); 389 list_add(&nrg->link, rg->link.prev); 390 coalesce_file_region(resv, nrg); 391 } else if (regions_needed) 392 *regions_needed += 1; 393 } 394 395 VM_BUG_ON(add < 0); 396 return add; 397 } 398 399 /* Must be called with resv->lock acquired. Will drop lock to allocate entries. 400 */ 401 static int allocate_file_region_entries(struct resv_map *resv, 402 int regions_needed) 403 __must_hold(&resv->lock) 404 { 405 struct list_head allocated_regions; 406 int to_allocate = 0, i = 0; 407 struct file_region *trg = NULL, *rg = NULL; 408 409 VM_BUG_ON(regions_needed < 0); 410 411 INIT_LIST_HEAD(&allocated_regions); 412 413 /* 414 * Check for sufficient descriptors in the cache to accommodate 415 * the number of in progress add operations plus regions_needed. 416 * 417 * This is a while loop because when we drop the lock, some other call 418 * to region_add or region_del may have consumed some region_entries, 419 * so we keep looping here until we finally have enough entries for 420 * (adds_in_progress + regions_needed). 421 */ 422 while (resv->region_cache_count < 423 (resv->adds_in_progress + regions_needed)) { 424 to_allocate = resv->adds_in_progress + regions_needed - 425 resv->region_cache_count; 426 427 /* At this point, we should have enough entries in the cache 428 * for all the existings adds_in_progress. We should only be 429 * needing to allocate for regions_needed. 430 */ 431 VM_BUG_ON(resv->region_cache_count < resv->adds_in_progress); 432 433 spin_unlock(&resv->lock); 434 for (i = 0; i < to_allocate; i++) { 435 trg = kmalloc(sizeof(*trg), GFP_KERNEL); 436 if (!trg) 437 goto out_of_memory; 438 list_add(&trg->link, &allocated_regions); 439 } 440 441 spin_lock(&resv->lock); 442 443 list_for_each_entry_safe(rg, trg, &allocated_regions, link) { 444 list_del(&rg->link); 445 list_add(&rg->link, &resv->region_cache); 446 resv->region_cache_count++; 447 } 448 } 449 450 return 0; 451 452 out_of_memory: 453 list_for_each_entry_safe(rg, trg, &allocated_regions, link) { 454 list_del(&rg->link); 455 kfree(rg); 456 } 457 return -ENOMEM; 458 } 459 460 /* 461 * Add the huge page range represented by [f, t) to the reserve 462 * map. Regions will be taken from the cache to fill in this range. 463 * Sufficient regions should exist in the cache due to the previous 464 * call to region_chg with the same range, but in some cases the cache will not 465 * have sufficient entries due to races with other code doing region_add or 466 * region_del. The extra needed entries will be allocated. 467 * 468 * regions_needed is the out value provided by a previous call to region_chg. 469 * 470 * Return the number of new huge pages added to the map. This number is greater 471 * than or equal to zero. If file_region entries needed to be allocated for 472 * this operation and we were not able to allocate, it ruturns -ENOMEM. 473 * region_add of regions of length 1 never allocate file_regions and cannot 474 * fail; region_chg will always allocate at least 1 entry and a region_add for 475 * 1 page will only require at most 1 entry. 476 */ 477 static long region_add(struct resv_map *resv, long f, long t, 478 long in_regions_needed, struct hstate *h, 479 struct hugetlb_cgroup *h_cg) 480 { 481 long add = 0, actual_regions_needed = 0; 482 483 spin_lock(&resv->lock); 484 retry: 485 486 /* Count how many regions are actually needed to execute this add. */ 487 add_reservation_in_range(resv, f, t, NULL, NULL, &actual_regions_needed, 488 true); 489 490 /* 491 * Check for sufficient descriptors in the cache to accommodate 492 * this add operation. Note that actual_regions_needed may be greater 493 * than in_regions_needed, as the resv_map may have been modified since 494 * the region_chg call. In this case, we need to make sure that we 495 * allocate extra entries, such that we have enough for all the 496 * existing adds_in_progress, plus the excess needed for this 497 * operation. 498 */ 499 if (actual_regions_needed > in_regions_needed && 500 resv->region_cache_count < 501 resv->adds_in_progress + 502 (actual_regions_needed - in_regions_needed)) { 503 /* region_add operation of range 1 should never need to 504 * allocate file_region entries. 505 */ 506 VM_BUG_ON(t - f <= 1); 507 508 if (allocate_file_region_entries( 509 resv, actual_regions_needed - in_regions_needed)) { 510 return -ENOMEM; 511 } 512 513 goto retry; 514 } 515 516 add = add_reservation_in_range(resv, f, t, h_cg, h, NULL, false); 517 518 resv->adds_in_progress -= in_regions_needed; 519 520 spin_unlock(&resv->lock); 521 VM_BUG_ON(add < 0); 522 return add; 523 } 524 525 /* 526 * Examine the existing reserve map and determine how many 527 * huge pages in the specified range [f, t) are NOT currently 528 * represented. This routine is called before a subsequent 529 * call to region_add that will actually modify the reserve 530 * map to add the specified range [f, t). region_chg does 531 * not change the number of huge pages represented by the 532 * map. A number of new file_region structures is added to the cache as a 533 * placeholder, for the subsequent region_add call to use. At least 1 534 * file_region structure is added. 535 * 536 * out_regions_needed is the number of regions added to the 537 * resv->adds_in_progress. This value needs to be provided to a follow up call 538 * to region_add or region_abort for proper accounting. 539 * 540 * Returns the number of huge pages that need to be added to the existing 541 * reservation map for the range [f, t). This number is greater or equal to 542 * zero. -ENOMEM is returned if a new file_region structure or cache entry 543 * is needed and can not be allocated. 544 */ 545 static long region_chg(struct resv_map *resv, long f, long t, 546 long *out_regions_needed) 547 { 548 long chg = 0; 549 550 spin_lock(&resv->lock); 551 552 /* Count how many hugepages in this range are NOT respresented. */ 553 chg = add_reservation_in_range(resv, f, t, NULL, NULL, 554 out_regions_needed, true); 555 556 if (*out_regions_needed == 0) 557 *out_regions_needed = 1; 558 559 if (allocate_file_region_entries(resv, *out_regions_needed)) 560 return -ENOMEM; 561 562 resv->adds_in_progress += *out_regions_needed; 563 564 spin_unlock(&resv->lock); 565 return chg; 566 } 567 568 /* 569 * Abort the in progress add operation. The adds_in_progress field 570 * of the resv_map keeps track of the operations in progress between 571 * calls to region_chg and region_add. Operations are sometimes 572 * aborted after the call to region_chg. In such cases, region_abort 573 * is called to decrement the adds_in_progress counter. regions_needed 574 * is the value returned by the region_chg call, it is used to decrement 575 * the adds_in_progress counter. 576 * 577 * NOTE: The range arguments [f, t) are not needed or used in this 578 * routine. They are kept to make reading the calling code easier as 579 * arguments will match the associated region_chg call. 580 */ 581 static void region_abort(struct resv_map *resv, long f, long t, 582 long regions_needed) 583 { 584 spin_lock(&resv->lock); 585 VM_BUG_ON(!resv->region_cache_count); 586 resv->adds_in_progress -= regions_needed; 587 spin_unlock(&resv->lock); 588 } 589 590 /* 591 * Delete the specified range [f, t) from the reserve map. If the 592 * t parameter is LONG_MAX, this indicates that ALL regions after f 593 * should be deleted. Locate the regions which intersect [f, t) 594 * and either trim, delete or split the existing regions. 595 * 596 * Returns the number of huge pages deleted from the reserve map. 597 * In the normal case, the return value is zero or more. In the 598 * case where a region must be split, a new region descriptor must 599 * be allocated. If the allocation fails, -ENOMEM will be returned. 600 * NOTE: If the parameter t == LONG_MAX, then we will never split 601 * a region and possibly return -ENOMEM. Callers specifying 602 * t == LONG_MAX do not need to check for -ENOMEM error. 603 */ 604 static long region_del(struct resv_map *resv, long f, long t) 605 { 606 struct list_head *head = &resv->regions; 607 struct file_region *rg, *trg; 608 struct file_region *nrg = NULL; 609 long del = 0; 610 611 retry: 612 spin_lock(&resv->lock); 613 list_for_each_entry_safe(rg, trg, head, link) { 614 /* 615 * Skip regions before the range to be deleted. file_region 616 * ranges are normally of the form [from, to). However, there 617 * may be a "placeholder" entry in the map which is of the form 618 * (from, to) with from == to. Check for placeholder entries 619 * at the beginning of the range to be deleted. 620 */ 621 if (rg->to <= f && (rg->to != rg->from || rg->to != f)) 622 continue; 623 624 if (rg->from >= t) 625 break; 626 627 if (f > rg->from && t < rg->to) { /* Must split region */ 628 /* 629 * Check for an entry in the cache before dropping 630 * lock and attempting allocation. 631 */ 632 if (!nrg && 633 resv->region_cache_count > resv->adds_in_progress) { 634 nrg = list_first_entry(&resv->region_cache, 635 struct file_region, 636 link); 637 list_del(&nrg->link); 638 resv->region_cache_count--; 639 } 640 641 if (!nrg) { 642 spin_unlock(&resv->lock); 643 nrg = kmalloc(sizeof(*nrg), GFP_KERNEL); 644 if (!nrg) 645 return -ENOMEM; 646 goto retry; 647 } 648 649 del += t - f; 650 651 /* New entry for end of split region */ 652 nrg->from = t; 653 nrg->to = rg->to; 654 655 copy_hugetlb_cgroup_uncharge_info(nrg, rg); 656 657 INIT_LIST_HEAD(&nrg->link); 658 659 /* Original entry is trimmed */ 660 rg->to = f; 661 662 hugetlb_cgroup_uncharge_file_region( 663 resv, rg, nrg->to - nrg->from); 664 665 list_add(&nrg->link, &rg->link); 666 nrg = NULL; 667 break; 668 } 669 670 if (f <= rg->from && t >= rg->to) { /* Remove entire region */ 671 del += rg->to - rg->from; 672 hugetlb_cgroup_uncharge_file_region(resv, rg, 673 rg->to - rg->from); 674 list_del(&rg->link); 675 kfree(rg); 676 continue; 677 } 678 679 if (f <= rg->from) { /* Trim beginning of region */ 680 del += t - rg->from; 681 rg->from = t; 682 683 hugetlb_cgroup_uncharge_file_region(resv, rg, 684 t - rg->from); 685 } else { /* Trim end of region */ 686 del += rg->to - f; 687 rg->to = f; 688 689 hugetlb_cgroup_uncharge_file_region(resv, rg, 690 rg->to - f); 691 } 692 } 693 694 spin_unlock(&resv->lock); 695 kfree(nrg); 696 return del; 697 } 698 699 /* 700 * A rare out of memory error was encountered which prevented removal of 701 * the reserve map region for a page. The huge page itself was free'ed 702 * and removed from the page cache. This routine will adjust the subpool 703 * usage count, and the global reserve count if needed. By incrementing 704 * these counts, the reserve map entry which could not be deleted will 705 * appear as a "reserved" entry instead of simply dangling with incorrect 706 * counts. 707 */ 708 void hugetlb_fix_reserve_counts(struct inode *inode) 709 { 710 struct hugepage_subpool *spool = subpool_inode(inode); 711 long rsv_adjust; 712 713 rsv_adjust = hugepage_subpool_get_pages(spool, 1); 714 if (rsv_adjust) { 715 struct hstate *h = hstate_inode(inode); 716 717 hugetlb_acct_memory(h, 1); 718 } 719 } 720 721 /* 722 * Count and return the number of huge pages in the reserve map 723 * that intersect with the range [f, t). 724 */ 725 static long region_count(struct resv_map *resv, long f, long t) 726 { 727 struct list_head *head = &resv->regions; 728 struct file_region *rg; 729 long chg = 0; 730 731 spin_lock(&resv->lock); 732 /* Locate each segment we overlap with, and count that overlap. */ 733 list_for_each_entry(rg, head, link) { 734 long seg_from; 735 long seg_to; 736 737 if (rg->to <= f) 738 continue; 739 if (rg->from >= t) 740 break; 741 742 seg_from = max(rg->from, f); 743 seg_to = min(rg->to, t); 744 745 chg += seg_to - seg_from; 746 } 747 spin_unlock(&resv->lock); 748 749 return chg; 750 } 751 752 /* 753 * Convert the address within this vma to the page offset within 754 * the mapping, in pagecache page units; huge pages here. 755 */ 756 static pgoff_t vma_hugecache_offset(struct hstate *h, 757 struct vm_area_struct *vma, unsigned long address) 758 { 759 return ((address - vma->vm_start) >> huge_page_shift(h)) + 760 (vma->vm_pgoff >> huge_page_order(h)); 761 } 762 763 pgoff_t linear_hugepage_index(struct vm_area_struct *vma, 764 unsigned long address) 765 { 766 return vma_hugecache_offset(hstate_vma(vma), vma, address); 767 } 768 EXPORT_SYMBOL_GPL(linear_hugepage_index); 769 770 /* 771 * Return the size of the pages allocated when backing a VMA. In the majority 772 * cases this will be same size as used by the page table entries. 773 */ 774 unsigned long vma_kernel_pagesize(struct vm_area_struct *vma) 775 { 776 if (vma->vm_ops && vma->vm_ops->pagesize) 777 return vma->vm_ops->pagesize(vma); 778 return PAGE_SIZE; 779 } 780 EXPORT_SYMBOL_GPL(vma_kernel_pagesize); 781 782 /* 783 * Return the page size being used by the MMU to back a VMA. In the majority 784 * of cases, the page size used by the kernel matches the MMU size. On 785 * architectures where it differs, an architecture-specific 'strong' 786 * version of this symbol is required. 787 */ 788 __weak unsigned long vma_mmu_pagesize(struct vm_area_struct *vma) 789 { 790 return vma_kernel_pagesize(vma); 791 } 792 793 /* 794 * Flags for MAP_PRIVATE reservations. These are stored in the bottom 795 * bits of the reservation map pointer, which are always clear due to 796 * alignment. 797 */ 798 #define HPAGE_RESV_OWNER (1UL << 0) 799 #define HPAGE_RESV_UNMAPPED (1UL << 1) 800 #define HPAGE_RESV_MASK (HPAGE_RESV_OWNER | HPAGE_RESV_UNMAPPED) 801 802 /* 803 * These helpers are used to track how many pages are reserved for 804 * faults in a MAP_PRIVATE mapping. Only the process that called mmap() 805 * is guaranteed to have their future faults succeed. 806 * 807 * With the exception of reset_vma_resv_huge_pages() which is called at fork(), 808 * the reserve counters are updated with the hugetlb_lock held. It is safe 809 * to reset the VMA at fork() time as it is not in use yet and there is no 810 * chance of the global counters getting corrupted as a result of the values. 811 * 812 * The private mapping reservation is represented in a subtly different 813 * manner to a shared mapping. A shared mapping has a region map associated 814 * with the underlying file, this region map represents the backing file 815 * pages which have ever had a reservation assigned which this persists even 816 * after the page is instantiated. A private mapping has a region map 817 * associated with the original mmap which is attached to all VMAs which 818 * reference it, this region map represents those offsets which have consumed 819 * reservation ie. where pages have been instantiated. 820 */ 821 static unsigned long get_vma_private_data(struct vm_area_struct *vma) 822 { 823 return (unsigned long)vma->vm_private_data; 824 } 825 826 static void set_vma_private_data(struct vm_area_struct *vma, 827 unsigned long value) 828 { 829 vma->vm_private_data = (void *)value; 830 } 831 832 static void 833 resv_map_set_hugetlb_cgroup_uncharge_info(struct resv_map *resv_map, 834 struct hugetlb_cgroup *h_cg, 835 struct hstate *h) 836 { 837 #ifdef CONFIG_CGROUP_HUGETLB 838 if (!h_cg || !h) { 839 resv_map->reservation_counter = NULL; 840 resv_map->pages_per_hpage = 0; 841 resv_map->css = NULL; 842 } else { 843 resv_map->reservation_counter = 844 &h_cg->rsvd_hugepage[hstate_index(h)]; 845 resv_map->pages_per_hpage = pages_per_huge_page(h); 846 resv_map->css = &h_cg->css; 847 } 848 #endif 849 } 850 851 struct resv_map *resv_map_alloc(void) 852 { 853 struct resv_map *resv_map = kmalloc(sizeof(*resv_map), GFP_KERNEL); 854 struct file_region *rg = kmalloc(sizeof(*rg), GFP_KERNEL); 855 856 if (!resv_map || !rg) { 857 kfree(resv_map); 858 kfree(rg); 859 return NULL; 860 } 861 862 kref_init(&resv_map->refs); 863 spin_lock_init(&resv_map->lock); 864 INIT_LIST_HEAD(&resv_map->regions); 865 866 resv_map->adds_in_progress = 0; 867 /* 868 * Initialize these to 0. On shared mappings, 0's here indicate these 869 * fields don't do cgroup accounting. On private mappings, these will be 870 * re-initialized to the proper values, to indicate that hugetlb cgroup 871 * reservations are to be un-charged from here. 872 */ 873 resv_map_set_hugetlb_cgroup_uncharge_info(resv_map, NULL, NULL); 874 875 INIT_LIST_HEAD(&resv_map->region_cache); 876 list_add(&rg->link, &resv_map->region_cache); 877 resv_map->region_cache_count = 1; 878 879 return resv_map; 880 } 881 882 void resv_map_release(struct kref *ref) 883 { 884 struct resv_map *resv_map = container_of(ref, struct resv_map, refs); 885 struct list_head *head = &resv_map->region_cache; 886 struct file_region *rg, *trg; 887 888 /* Clear out any active regions before we release the map. */ 889 region_del(resv_map, 0, LONG_MAX); 890 891 /* ... and any entries left in the cache */ 892 list_for_each_entry_safe(rg, trg, head, link) { 893 list_del(&rg->link); 894 kfree(rg); 895 } 896 897 VM_BUG_ON(resv_map->adds_in_progress); 898 899 kfree(resv_map); 900 } 901 902 static inline struct resv_map *inode_resv_map(struct inode *inode) 903 { 904 /* 905 * At inode evict time, i_mapping may not point to the original 906 * address space within the inode. This original address space 907 * contains the pointer to the resv_map. So, always use the 908 * address space embedded within the inode. 909 * The VERY common case is inode->mapping == &inode->i_data but, 910 * this may not be true for device special inodes. 911 */ 912 return (struct resv_map *)(&inode->i_data)->private_data; 913 } 914 915 static struct resv_map *vma_resv_map(struct vm_area_struct *vma) 916 { 917 VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma); 918 if (vma->vm_flags & VM_MAYSHARE) { 919 struct address_space *mapping = vma->vm_file->f_mapping; 920 struct inode *inode = mapping->host; 921 922 return inode_resv_map(inode); 923 924 } else { 925 return (struct resv_map *)(get_vma_private_data(vma) & 926 ~HPAGE_RESV_MASK); 927 } 928 } 929 930 static void set_vma_resv_map(struct vm_area_struct *vma, struct resv_map *map) 931 { 932 VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma); 933 VM_BUG_ON_VMA(vma->vm_flags & VM_MAYSHARE, vma); 934 935 set_vma_private_data(vma, (get_vma_private_data(vma) & 936 HPAGE_RESV_MASK) | (unsigned long)map); 937 } 938 939 static void set_vma_resv_flags(struct vm_area_struct *vma, unsigned long flags) 940 { 941 VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma); 942 VM_BUG_ON_VMA(vma->vm_flags & VM_MAYSHARE, vma); 943 944 set_vma_private_data(vma, get_vma_private_data(vma) | flags); 945 } 946 947 static int is_vma_resv_set(struct vm_area_struct *vma, unsigned long flag) 948 { 949 VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma); 950 951 return (get_vma_private_data(vma) & flag) != 0; 952 } 953 954 /* Reset counters to 0 and clear all HPAGE_RESV_* flags */ 955 void reset_vma_resv_huge_pages(struct vm_area_struct *vma) 956 { 957 VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma); 958 if (!(vma->vm_flags & VM_MAYSHARE)) 959 vma->vm_private_data = (void *)0; 960 } 961 962 /* Returns true if the VMA has associated reserve pages */ 963 static bool vma_has_reserves(struct vm_area_struct *vma, long chg) 964 { 965 if (vma->vm_flags & VM_NORESERVE) { 966 /* 967 * This address is already reserved by other process(chg == 0), 968 * so, we should decrement reserved count. Without decrementing, 969 * reserve count remains after releasing inode, because this 970 * allocated page will go into page cache and is regarded as 971 * coming from reserved pool in releasing step. Currently, we 972 * don't have any other solution to deal with this situation 973 * properly, so add work-around here. 974 */ 975 if (vma->vm_flags & VM_MAYSHARE && chg == 0) 976 return true; 977 else 978 return false; 979 } 980 981 /* Shared mappings always use reserves */ 982 if (vma->vm_flags & VM_MAYSHARE) { 983 /* 984 * We know VM_NORESERVE is not set. Therefore, there SHOULD 985 * be a region map for all pages. The only situation where 986 * there is no region map is if a hole was punched via 987 * fallocate. In this case, there really are no reverves to 988 * use. This situation is indicated if chg != 0. 989 */ 990 if (chg) 991 return false; 992 else 993 return true; 994 } 995 996 /* 997 * Only the process that called mmap() has reserves for 998 * private mappings. 999 */ 1000 if (is_vma_resv_set(vma, HPAGE_RESV_OWNER)) { 1001 /* 1002 * Like the shared case above, a hole punch or truncate 1003 * could have been performed on the private mapping. 1004 * Examine the value of chg to determine if reserves 1005 * actually exist or were previously consumed. 1006 * Very Subtle - The value of chg comes from a previous 1007 * call to vma_needs_reserves(). The reserve map for 1008 * private mappings has different (opposite) semantics 1009 * than that of shared mappings. vma_needs_reserves() 1010 * has already taken this difference in semantics into 1011 * account. Therefore, the meaning of chg is the same 1012 * as in the shared case above. Code could easily be 1013 * combined, but keeping it separate draws attention to 1014 * subtle differences. 1015 */ 1016 if (chg) 1017 return false; 1018 else 1019 return true; 1020 } 1021 1022 return false; 1023 } 1024 1025 static void enqueue_huge_page(struct hstate *h, struct page *page) 1026 { 1027 int nid = page_to_nid(page); 1028 list_move(&page->lru, &h->hugepage_freelists[nid]); 1029 h->free_huge_pages++; 1030 h->free_huge_pages_node[nid]++; 1031 } 1032 1033 static struct page *dequeue_huge_page_node_exact(struct hstate *h, int nid) 1034 { 1035 struct page *page; 1036 1037 list_for_each_entry(page, &h->hugepage_freelists[nid], lru) 1038 if (!PageHWPoison(page)) 1039 break; 1040 /* 1041 * if 'non-isolated free hugepage' not found on the list, 1042 * the allocation fails. 1043 */ 1044 if (&h->hugepage_freelists[nid] == &page->lru) 1045 return NULL; 1046 list_move(&page->lru, &h->hugepage_activelist); 1047 set_page_refcounted(page); 1048 h->free_huge_pages--; 1049 h->free_huge_pages_node[nid]--; 1050 return page; 1051 } 1052 1053 static struct page *dequeue_huge_page_nodemask(struct hstate *h, gfp_t gfp_mask, int nid, 1054 nodemask_t *nmask) 1055 { 1056 unsigned int cpuset_mems_cookie; 1057 struct zonelist *zonelist; 1058 struct zone *zone; 1059 struct zoneref *z; 1060 int node = NUMA_NO_NODE; 1061 1062 zonelist = node_zonelist(nid, gfp_mask); 1063 1064 retry_cpuset: 1065 cpuset_mems_cookie = read_mems_allowed_begin(); 1066 for_each_zone_zonelist_nodemask(zone, z, zonelist, gfp_zone(gfp_mask), nmask) { 1067 struct page *page; 1068 1069 if (!cpuset_zone_allowed(zone, gfp_mask)) 1070 continue; 1071 /* 1072 * no need to ask again on the same node. Pool is node rather than 1073 * zone aware 1074 */ 1075 if (zone_to_nid(zone) == node) 1076 continue; 1077 node = zone_to_nid(zone); 1078 1079 page = dequeue_huge_page_node_exact(h, node); 1080 if (page) 1081 return page; 1082 } 1083 if (unlikely(read_mems_allowed_retry(cpuset_mems_cookie))) 1084 goto retry_cpuset; 1085 1086 return NULL; 1087 } 1088 1089 /* Movability of hugepages depends on migration support. */ 1090 static inline gfp_t htlb_alloc_mask(struct hstate *h) 1091 { 1092 if (hugepage_movable_supported(h)) 1093 return GFP_HIGHUSER_MOVABLE; 1094 else 1095 return GFP_HIGHUSER; 1096 } 1097 1098 static struct page *dequeue_huge_page_vma(struct hstate *h, 1099 struct vm_area_struct *vma, 1100 unsigned long address, int avoid_reserve, 1101 long chg) 1102 { 1103 struct page *page; 1104 struct mempolicy *mpol; 1105 gfp_t gfp_mask; 1106 nodemask_t *nodemask; 1107 int nid; 1108 1109 /* 1110 * A child process with MAP_PRIVATE mappings created by their parent 1111 * have no page reserves. This check ensures that reservations are 1112 * not "stolen". The child may still get SIGKILLed 1113 */ 1114 if (!vma_has_reserves(vma, chg) && 1115 h->free_huge_pages - h->resv_huge_pages == 0) 1116 goto err; 1117 1118 /* If reserves cannot be used, ensure enough pages are in the pool */ 1119 if (avoid_reserve && h->free_huge_pages - h->resv_huge_pages == 0) 1120 goto err; 1121 1122 gfp_mask = htlb_alloc_mask(h); 1123 nid = huge_node(vma, address, gfp_mask, &mpol, &nodemask); 1124 page = dequeue_huge_page_nodemask(h, gfp_mask, nid, nodemask); 1125 if (page && !avoid_reserve && vma_has_reserves(vma, chg)) { 1126 SetPagePrivate(page); 1127 h->resv_huge_pages--; 1128 } 1129 1130 mpol_cond_put(mpol); 1131 return page; 1132 1133 err: 1134 return NULL; 1135 } 1136 1137 /* 1138 * common helper functions for hstate_next_node_to_{alloc|free}. 1139 * We may have allocated or freed a huge page based on a different 1140 * nodes_allowed previously, so h->next_node_to_{alloc|free} might 1141 * be outside of *nodes_allowed. Ensure that we use an allowed 1142 * node for alloc or free. 1143 */ 1144 static int next_node_allowed(int nid, nodemask_t *nodes_allowed) 1145 { 1146 nid = next_node_in(nid, *nodes_allowed); 1147 VM_BUG_ON(nid >= MAX_NUMNODES); 1148 1149 return nid; 1150 } 1151 1152 static int get_valid_node_allowed(int nid, nodemask_t *nodes_allowed) 1153 { 1154 if (!node_isset(nid, *nodes_allowed)) 1155 nid = next_node_allowed(nid, nodes_allowed); 1156 return nid; 1157 } 1158 1159 /* 1160 * returns the previously saved node ["this node"] from which to 1161 * allocate a persistent huge page for the pool and advance the 1162 * next node from which to allocate, handling wrap at end of node 1163 * mask. 1164 */ 1165 static int hstate_next_node_to_alloc(struct hstate *h, 1166 nodemask_t *nodes_allowed) 1167 { 1168 int nid; 1169 1170 VM_BUG_ON(!nodes_allowed); 1171 1172 nid = get_valid_node_allowed(h->next_nid_to_alloc, nodes_allowed); 1173 h->next_nid_to_alloc = next_node_allowed(nid, nodes_allowed); 1174 1175 return nid; 1176 } 1177 1178 /* 1179 * helper for free_pool_huge_page() - return the previously saved 1180 * node ["this node"] from which to free a huge page. Advance the 1181 * next node id whether or not we find a free huge page to free so 1182 * that the next attempt to free addresses the next node. 1183 */ 1184 static int hstate_next_node_to_free(struct hstate *h, nodemask_t *nodes_allowed) 1185 { 1186 int nid; 1187 1188 VM_BUG_ON(!nodes_allowed); 1189 1190 nid = get_valid_node_allowed(h->next_nid_to_free, nodes_allowed); 1191 h->next_nid_to_free = next_node_allowed(nid, nodes_allowed); 1192 1193 return nid; 1194 } 1195 1196 #define for_each_node_mask_to_alloc(hs, nr_nodes, node, mask) \ 1197 for (nr_nodes = nodes_weight(*mask); \ 1198 nr_nodes > 0 && \ 1199 ((node = hstate_next_node_to_alloc(hs, mask)) || 1); \ 1200 nr_nodes--) 1201 1202 #define for_each_node_mask_to_free(hs, nr_nodes, node, mask) \ 1203 for (nr_nodes = nodes_weight(*mask); \ 1204 nr_nodes > 0 && \ 1205 ((node = hstate_next_node_to_free(hs, mask)) || 1); \ 1206 nr_nodes--) 1207 1208 #ifdef CONFIG_ARCH_HAS_GIGANTIC_PAGE 1209 static void destroy_compound_gigantic_page(struct page *page, 1210 unsigned int order) 1211 { 1212 int i; 1213 int nr_pages = 1 << order; 1214 struct page *p = page + 1; 1215 1216 atomic_set(compound_mapcount_ptr(page), 0); 1217 if (hpage_pincount_available(page)) 1218 atomic_set(compound_pincount_ptr(page), 0); 1219 1220 for (i = 1; i < nr_pages; i++, p = mem_map_next(p, page, i)) { 1221 clear_compound_head(p); 1222 set_page_refcounted(p); 1223 } 1224 1225 set_compound_order(page, 0); 1226 __ClearPageHead(page); 1227 } 1228 1229 static void free_gigantic_page(struct page *page, unsigned int order) 1230 { 1231 free_contig_range(page_to_pfn(page), 1 << order); 1232 } 1233 1234 #ifdef CONFIG_CONTIG_ALLOC 1235 static struct page *alloc_gigantic_page(struct hstate *h, gfp_t gfp_mask, 1236 int nid, nodemask_t *nodemask) 1237 { 1238 unsigned long nr_pages = 1UL << huge_page_order(h); 1239 1240 return alloc_contig_pages(nr_pages, gfp_mask, nid, nodemask); 1241 } 1242 1243 static void prep_new_huge_page(struct hstate *h, struct page *page, int nid); 1244 static void prep_compound_gigantic_page(struct page *page, unsigned int order); 1245 #else /* !CONFIG_CONTIG_ALLOC */ 1246 static struct page *alloc_gigantic_page(struct hstate *h, gfp_t gfp_mask, 1247 int nid, nodemask_t *nodemask) 1248 { 1249 return NULL; 1250 } 1251 #endif /* CONFIG_CONTIG_ALLOC */ 1252 1253 #else /* !CONFIG_ARCH_HAS_GIGANTIC_PAGE */ 1254 static struct page *alloc_gigantic_page(struct hstate *h, gfp_t gfp_mask, 1255 int nid, nodemask_t *nodemask) 1256 { 1257 return NULL; 1258 } 1259 static inline void free_gigantic_page(struct page *page, unsigned int order) { } 1260 static inline void destroy_compound_gigantic_page(struct page *page, 1261 unsigned int order) { } 1262 #endif 1263 1264 static void update_and_free_page(struct hstate *h, struct page *page) 1265 { 1266 int i; 1267 1268 if (hstate_is_gigantic(h) && !gigantic_page_runtime_supported()) 1269 return; 1270 1271 h->nr_huge_pages--; 1272 h->nr_huge_pages_node[page_to_nid(page)]--; 1273 for (i = 0; i < pages_per_huge_page(h); i++) { 1274 page[i].flags &= ~(1 << PG_locked | 1 << PG_error | 1275 1 << PG_referenced | 1 << PG_dirty | 1276 1 << PG_active | 1 << PG_private | 1277 1 << PG_writeback); 1278 } 1279 VM_BUG_ON_PAGE(hugetlb_cgroup_from_page(page), page); 1280 VM_BUG_ON_PAGE(hugetlb_cgroup_from_page_rsvd(page), page); 1281 set_compound_page_dtor(page, NULL_COMPOUND_DTOR); 1282 set_page_refcounted(page); 1283 if (hstate_is_gigantic(h)) { 1284 destroy_compound_gigantic_page(page, huge_page_order(h)); 1285 free_gigantic_page(page, huge_page_order(h)); 1286 } else { 1287 __free_pages(page, huge_page_order(h)); 1288 } 1289 } 1290 1291 struct hstate *size_to_hstate(unsigned long size) 1292 { 1293 struct hstate *h; 1294 1295 for_each_hstate(h) { 1296 if (huge_page_size(h) == size) 1297 return h; 1298 } 1299 return NULL; 1300 } 1301 1302 /* 1303 * Test to determine whether the hugepage is "active/in-use" (i.e. being linked 1304 * to hstate->hugepage_activelist.) 1305 * 1306 * This function can be called for tail pages, but never returns true for them. 1307 */ 1308 bool page_huge_active(struct page *page) 1309 { 1310 VM_BUG_ON_PAGE(!PageHuge(page), page); 1311 return PageHead(page) && PagePrivate(&page[1]); 1312 } 1313 1314 /* never called for tail page */ 1315 static void set_page_huge_active(struct page *page) 1316 { 1317 VM_BUG_ON_PAGE(!PageHeadHuge(page), page); 1318 SetPagePrivate(&page[1]); 1319 } 1320 1321 static void clear_page_huge_active(struct page *page) 1322 { 1323 VM_BUG_ON_PAGE(!PageHeadHuge(page), page); 1324 ClearPagePrivate(&page[1]); 1325 } 1326 1327 /* 1328 * Internal hugetlb specific page flag. Do not use outside of the hugetlb 1329 * code 1330 */ 1331 static inline bool PageHugeTemporary(struct page *page) 1332 { 1333 if (!PageHuge(page)) 1334 return false; 1335 1336 return (unsigned long)page[2].mapping == -1U; 1337 } 1338 1339 static inline void SetPageHugeTemporary(struct page *page) 1340 { 1341 page[2].mapping = (void *)-1U; 1342 } 1343 1344 static inline void ClearPageHugeTemporary(struct page *page) 1345 { 1346 page[2].mapping = NULL; 1347 } 1348 1349 static void __free_huge_page(struct page *page) 1350 { 1351 /* 1352 * Can't pass hstate in here because it is called from the 1353 * compound page destructor. 1354 */ 1355 struct hstate *h = page_hstate(page); 1356 int nid = page_to_nid(page); 1357 struct hugepage_subpool *spool = 1358 (struct hugepage_subpool *)page_private(page); 1359 bool restore_reserve; 1360 1361 VM_BUG_ON_PAGE(page_count(page), page); 1362 VM_BUG_ON_PAGE(page_mapcount(page), page); 1363 1364 set_page_private(page, 0); 1365 page->mapping = NULL; 1366 restore_reserve = PagePrivate(page); 1367 ClearPagePrivate(page); 1368 1369 /* 1370 * If PagePrivate() was set on page, page allocation consumed a 1371 * reservation. If the page was associated with a subpool, there 1372 * would have been a page reserved in the subpool before allocation 1373 * via hugepage_subpool_get_pages(). Since we are 'restoring' the 1374 * reservtion, do not call hugepage_subpool_put_pages() as this will 1375 * remove the reserved page from the subpool. 1376 */ 1377 if (!restore_reserve) { 1378 /* 1379 * A return code of zero implies that the subpool will be 1380 * under its minimum size if the reservation is not restored 1381 * after page is free. Therefore, force restore_reserve 1382 * operation. 1383 */ 1384 if (hugepage_subpool_put_pages(spool, 1) == 0) 1385 restore_reserve = true; 1386 } 1387 1388 spin_lock(&hugetlb_lock); 1389 clear_page_huge_active(page); 1390 hugetlb_cgroup_uncharge_page(hstate_index(h), 1391 pages_per_huge_page(h), page); 1392 hugetlb_cgroup_uncharge_page_rsvd(hstate_index(h), 1393 pages_per_huge_page(h), page); 1394 if (restore_reserve) 1395 h->resv_huge_pages++; 1396 1397 if (PageHugeTemporary(page)) { 1398 list_del(&page->lru); 1399 ClearPageHugeTemporary(page); 1400 update_and_free_page(h, page); 1401 } else if (h->surplus_huge_pages_node[nid]) { 1402 /* remove the page from active list */ 1403 list_del(&page->lru); 1404 update_and_free_page(h, page); 1405 h->surplus_huge_pages--; 1406 h->surplus_huge_pages_node[nid]--; 1407 } else { 1408 arch_clear_hugepage_flags(page); 1409 enqueue_huge_page(h, page); 1410 } 1411 spin_unlock(&hugetlb_lock); 1412 } 1413 1414 /* 1415 * As free_huge_page() can be called from a non-task context, we have 1416 * to defer the actual freeing in a workqueue to prevent potential 1417 * hugetlb_lock deadlock. 1418 * 1419 * free_hpage_workfn() locklessly retrieves the linked list of pages to 1420 * be freed and frees them one-by-one. As the page->mapping pointer is 1421 * going to be cleared in __free_huge_page() anyway, it is reused as the 1422 * llist_node structure of a lockless linked list of huge pages to be freed. 1423 */ 1424 static LLIST_HEAD(hpage_freelist); 1425 1426 static void free_hpage_workfn(struct work_struct *work) 1427 { 1428 struct llist_node *node; 1429 struct page *page; 1430 1431 node = llist_del_all(&hpage_freelist); 1432 1433 while (node) { 1434 page = container_of((struct address_space **)node, 1435 struct page, mapping); 1436 node = node->next; 1437 __free_huge_page(page); 1438 } 1439 } 1440 static DECLARE_WORK(free_hpage_work, free_hpage_workfn); 1441 1442 void free_huge_page(struct page *page) 1443 { 1444 /* 1445 * Defer freeing if in non-task context to avoid hugetlb_lock deadlock. 1446 */ 1447 if (!in_task()) { 1448 /* 1449 * Only call schedule_work() if hpage_freelist is previously 1450 * empty. Otherwise, schedule_work() had been called but the 1451 * workfn hasn't retrieved the list yet. 1452 */ 1453 if (llist_add((struct llist_node *)&page->mapping, 1454 &hpage_freelist)) 1455 schedule_work(&free_hpage_work); 1456 return; 1457 } 1458 1459 __free_huge_page(page); 1460 } 1461 1462 static void prep_new_huge_page(struct hstate *h, struct page *page, int nid) 1463 { 1464 INIT_LIST_HEAD(&page->lru); 1465 set_compound_page_dtor(page, HUGETLB_PAGE_DTOR); 1466 spin_lock(&hugetlb_lock); 1467 set_hugetlb_cgroup(page, NULL); 1468 set_hugetlb_cgroup_rsvd(page, NULL); 1469 h->nr_huge_pages++; 1470 h->nr_huge_pages_node[nid]++; 1471 spin_unlock(&hugetlb_lock); 1472 } 1473 1474 static void prep_compound_gigantic_page(struct page *page, unsigned int order) 1475 { 1476 int i; 1477 int nr_pages = 1 << order; 1478 struct page *p = page + 1; 1479 1480 /* we rely on prep_new_huge_page to set the destructor */ 1481 set_compound_order(page, order); 1482 __ClearPageReserved(page); 1483 __SetPageHead(page); 1484 for (i = 1; i < nr_pages; i++, p = mem_map_next(p, page, i)) { 1485 /* 1486 * For gigantic hugepages allocated through bootmem at 1487 * boot, it's safer to be consistent with the not-gigantic 1488 * hugepages and clear the PG_reserved bit from all tail pages 1489 * too. Otherwse drivers using get_user_pages() to access tail 1490 * pages may get the reference counting wrong if they see 1491 * PG_reserved set on a tail page (despite the head page not 1492 * having PG_reserved set). Enforcing this consistency between 1493 * head and tail pages allows drivers to optimize away a check 1494 * on the head page when they need know if put_page() is needed 1495 * after get_user_pages(). 1496 */ 1497 __ClearPageReserved(p); 1498 set_page_count(p, 0); 1499 set_compound_head(p, page); 1500 } 1501 atomic_set(compound_mapcount_ptr(page), -1); 1502 1503 if (hpage_pincount_available(page)) 1504 atomic_set(compound_pincount_ptr(page), 0); 1505 } 1506 1507 /* 1508 * PageHuge() only returns true for hugetlbfs pages, but not for normal or 1509 * transparent huge pages. See the PageTransHuge() documentation for more 1510 * details. 1511 */ 1512 int PageHuge(struct page *page) 1513 { 1514 if (!PageCompound(page)) 1515 return 0; 1516 1517 page = compound_head(page); 1518 return page[1].compound_dtor == HUGETLB_PAGE_DTOR; 1519 } 1520 EXPORT_SYMBOL_GPL(PageHuge); 1521 1522 /* 1523 * PageHeadHuge() only returns true for hugetlbfs head page, but not for 1524 * normal or transparent huge pages. 1525 */ 1526 int PageHeadHuge(struct page *page_head) 1527 { 1528 if (!PageHead(page_head)) 1529 return 0; 1530 1531 return page_head[1].compound_dtor == HUGETLB_PAGE_DTOR; 1532 } 1533 1534 /* 1535 * Find address_space associated with hugetlbfs page. 1536 * Upon entry page is locked and page 'was' mapped although mapped state 1537 * could change. If necessary, use anon_vma to find vma and associated 1538 * address space. The returned mapping may be stale, but it can not be 1539 * invalid as page lock (which is held) is required to destroy mapping. 1540 */ 1541 static struct address_space *_get_hugetlb_page_mapping(struct page *hpage) 1542 { 1543 struct anon_vma *anon_vma; 1544 pgoff_t pgoff_start, pgoff_end; 1545 struct anon_vma_chain *avc; 1546 struct address_space *mapping = page_mapping(hpage); 1547 1548 /* Simple file based mapping */ 1549 if (mapping) 1550 return mapping; 1551 1552 /* 1553 * Even anonymous hugetlbfs mappings are associated with an 1554 * underlying hugetlbfs file (see hugetlb_file_setup in mmap 1555 * code). Find a vma associated with the anonymous vma, and 1556 * use the file pointer to get address_space. 1557 */ 1558 anon_vma = page_lock_anon_vma_read(hpage); 1559 if (!anon_vma) 1560 return mapping; /* NULL */ 1561 1562 /* Use first found vma */ 1563 pgoff_start = page_to_pgoff(hpage); 1564 pgoff_end = pgoff_start + hpage_nr_pages(hpage) - 1; 1565 anon_vma_interval_tree_foreach(avc, &anon_vma->rb_root, 1566 pgoff_start, pgoff_end) { 1567 struct vm_area_struct *vma = avc->vma; 1568 1569 mapping = vma->vm_file->f_mapping; 1570 break; 1571 } 1572 1573 anon_vma_unlock_read(anon_vma); 1574 return mapping; 1575 } 1576 1577 /* 1578 * Find and lock address space (mapping) in write mode. 1579 * 1580 * Upon entry, the page is locked which allows us to find the mapping 1581 * even in the case of an anon page. However, locking order dictates 1582 * the i_mmap_rwsem be acquired BEFORE the page lock. This is hugetlbfs 1583 * specific. So, we first try to lock the sema while still holding the 1584 * page lock. If this works, great! If not, then we need to drop the 1585 * page lock and then acquire i_mmap_rwsem and reacquire page lock. Of 1586 * course, need to revalidate state along the way. 1587 */ 1588 struct address_space *hugetlb_page_mapping_lock_write(struct page *hpage) 1589 { 1590 struct address_space *mapping, *mapping2; 1591 1592 mapping = _get_hugetlb_page_mapping(hpage); 1593 retry: 1594 if (!mapping) 1595 return mapping; 1596 1597 /* 1598 * If no contention, take lock and return 1599 */ 1600 if (i_mmap_trylock_write(mapping)) 1601 return mapping; 1602 1603 /* 1604 * Must drop page lock and wait on mapping sema. 1605 * Note: Once page lock is dropped, mapping could become invalid. 1606 * As a hack, increase map count until we lock page again. 1607 */ 1608 atomic_inc(&hpage->_mapcount); 1609 unlock_page(hpage); 1610 i_mmap_lock_write(mapping); 1611 lock_page(hpage); 1612 atomic_add_negative(-1, &hpage->_mapcount); 1613 1614 /* verify page is still mapped */ 1615 if (!page_mapped(hpage)) { 1616 i_mmap_unlock_write(mapping); 1617 return NULL; 1618 } 1619 1620 /* 1621 * Get address space again and verify it is the same one 1622 * we locked. If not, drop lock and retry. 1623 */ 1624 mapping2 = _get_hugetlb_page_mapping(hpage); 1625 if (mapping2 != mapping) { 1626 i_mmap_unlock_write(mapping); 1627 mapping = mapping2; 1628 goto retry; 1629 } 1630 1631 return mapping; 1632 } 1633 1634 pgoff_t __basepage_index(struct page *page) 1635 { 1636 struct page *page_head = compound_head(page); 1637 pgoff_t index = page_index(page_head); 1638 unsigned long compound_idx; 1639 1640 if (!PageHuge(page_head)) 1641 return page_index(page); 1642 1643 if (compound_order(page_head) >= MAX_ORDER) 1644 compound_idx = page_to_pfn(page) - page_to_pfn(page_head); 1645 else 1646 compound_idx = page - page_head; 1647 1648 return (index << compound_order(page_head)) + compound_idx; 1649 } 1650 1651 static struct page *alloc_buddy_huge_page(struct hstate *h, 1652 gfp_t gfp_mask, int nid, nodemask_t *nmask, 1653 nodemask_t *node_alloc_noretry) 1654 { 1655 int order = huge_page_order(h); 1656 struct page *page; 1657 bool alloc_try_hard = true; 1658 1659 /* 1660 * By default we always try hard to allocate the page with 1661 * __GFP_RETRY_MAYFAIL flag. However, if we are allocating pages in 1662 * a loop (to adjust global huge page counts) and previous allocation 1663 * failed, do not continue to try hard on the same node. Use the 1664 * node_alloc_noretry bitmap to manage this state information. 1665 */ 1666 if (node_alloc_noretry && node_isset(nid, *node_alloc_noretry)) 1667 alloc_try_hard = false; 1668 gfp_mask |= __GFP_COMP|__GFP_NOWARN; 1669 if (alloc_try_hard) 1670 gfp_mask |= __GFP_RETRY_MAYFAIL; 1671 if (nid == NUMA_NO_NODE) 1672 nid = numa_mem_id(); 1673 page = __alloc_pages_nodemask(gfp_mask, order, nid, nmask); 1674 if (page) 1675 __count_vm_event(HTLB_BUDDY_PGALLOC); 1676 else 1677 __count_vm_event(HTLB_BUDDY_PGALLOC_FAIL); 1678 1679 /* 1680 * If we did not specify __GFP_RETRY_MAYFAIL, but still got a page this 1681 * indicates an overall state change. Clear bit so that we resume 1682 * normal 'try hard' allocations. 1683 */ 1684 if (node_alloc_noretry && page && !alloc_try_hard) 1685 node_clear(nid, *node_alloc_noretry); 1686 1687 /* 1688 * If we tried hard to get a page but failed, set bit so that 1689 * subsequent attempts will not try as hard until there is an 1690 * overall state change. 1691 */ 1692 if (node_alloc_noretry && !page && alloc_try_hard) 1693 node_set(nid, *node_alloc_noretry); 1694 1695 return page; 1696 } 1697 1698 /* 1699 * Common helper to allocate a fresh hugetlb page. All specific allocators 1700 * should use this function to get new hugetlb pages 1701 */ 1702 static struct page *alloc_fresh_huge_page(struct hstate *h, 1703 gfp_t gfp_mask, int nid, nodemask_t *nmask, 1704 nodemask_t *node_alloc_noretry) 1705 { 1706 struct page *page; 1707 1708 if (hstate_is_gigantic(h)) 1709 page = alloc_gigantic_page(h, gfp_mask, nid, nmask); 1710 else 1711 page = alloc_buddy_huge_page(h, gfp_mask, 1712 nid, nmask, node_alloc_noretry); 1713 if (!page) 1714 return NULL; 1715 1716 if (hstate_is_gigantic(h)) 1717 prep_compound_gigantic_page(page, huge_page_order(h)); 1718 prep_new_huge_page(h, page, page_to_nid(page)); 1719 1720 return page; 1721 } 1722 1723 /* 1724 * Allocates a fresh page to the hugetlb allocator pool in the node interleaved 1725 * manner. 1726 */ 1727 static int alloc_pool_huge_page(struct hstate *h, nodemask_t *nodes_allowed, 1728 nodemask_t *node_alloc_noretry) 1729 { 1730 struct page *page; 1731 int nr_nodes, node; 1732 gfp_t gfp_mask = htlb_alloc_mask(h) | __GFP_THISNODE; 1733 1734 for_each_node_mask_to_alloc(h, nr_nodes, node, nodes_allowed) { 1735 page = alloc_fresh_huge_page(h, gfp_mask, node, nodes_allowed, 1736 node_alloc_noretry); 1737 if (page) 1738 break; 1739 } 1740 1741 if (!page) 1742 return 0; 1743 1744 put_page(page); /* free it into the hugepage allocator */ 1745 1746 return 1; 1747 } 1748 1749 /* 1750 * Free huge page from pool from next node to free. 1751 * Attempt to keep persistent huge pages more or less 1752 * balanced over allowed nodes. 1753 * Called with hugetlb_lock locked. 1754 */ 1755 static int free_pool_huge_page(struct hstate *h, nodemask_t *nodes_allowed, 1756 bool acct_surplus) 1757 { 1758 int nr_nodes, node; 1759 int ret = 0; 1760 1761 for_each_node_mask_to_free(h, nr_nodes, node, nodes_allowed) { 1762 /* 1763 * If we're returning unused surplus pages, only examine 1764 * nodes with surplus pages. 1765 */ 1766 if ((!acct_surplus || h->surplus_huge_pages_node[node]) && 1767 !list_empty(&h->hugepage_freelists[node])) { 1768 struct page *page = 1769 list_entry(h->hugepage_freelists[node].next, 1770 struct page, lru); 1771 list_del(&page->lru); 1772 h->free_huge_pages--; 1773 h->free_huge_pages_node[node]--; 1774 if (acct_surplus) { 1775 h->surplus_huge_pages--; 1776 h->surplus_huge_pages_node[node]--; 1777 } 1778 update_and_free_page(h, page); 1779 ret = 1; 1780 break; 1781 } 1782 } 1783 1784 return ret; 1785 } 1786 1787 /* 1788 * Dissolve a given free hugepage into free buddy pages. This function does 1789 * nothing for in-use hugepages and non-hugepages. 1790 * This function returns values like below: 1791 * 1792 * -EBUSY: failed to dissolved free hugepages or the hugepage is in-use 1793 * (allocated or reserved.) 1794 * 0: successfully dissolved free hugepages or the page is not a 1795 * hugepage (considered as already dissolved) 1796 */ 1797 int dissolve_free_huge_page(struct page *page) 1798 { 1799 int rc = -EBUSY; 1800 1801 /* Not to disrupt normal path by vainly holding hugetlb_lock */ 1802 if (!PageHuge(page)) 1803 return 0; 1804 1805 spin_lock(&hugetlb_lock); 1806 if (!PageHuge(page)) { 1807 rc = 0; 1808 goto out; 1809 } 1810 1811 if (!page_count(page)) { 1812 struct page *head = compound_head(page); 1813 struct hstate *h = page_hstate(head); 1814 int nid = page_to_nid(head); 1815 if (h->free_huge_pages - h->resv_huge_pages == 0) 1816 goto out; 1817 /* 1818 * Move PageHWPoison flag from head page to the raw error page, 1819 * which makes any subpages rather than the error page reusable. 1820 */ 1821 if (PageHWPoison(head) && page != head) { 1822 SetPageHWPoison(page); 1823 ClearPageHWPoison(head); 1824 } 1825 list_del(&head->lru); 1826 h->free_huge_pages--; 1827 h->free_huge_pages_node[nid]--; 1828 h->max_huge_pages--; 1829 update_and_free_page(h, head); 1830 rc = 0; 1831 } 1832 out: 1833 spin_unlock(&hugetlb_lock); 1834 return rc; 1835 } 1836 1837 /* 1838 * Dissolve free hugepages in a given pfn range. Used by memory hotplug to 1839 * make specified memory blocks removable from the system. 1840 * Note that this will dissolve a free gigantic hugepage completely, if any 1841 * part of it lies within the given range. 1842 * Also note that if dissolve_free_huge_page() returns with an error, all 1843 * free hugepages that were dissolved before that error are lost. 1844 */ 1845 int dissolve_free_huge_pages(unsigned long start_pfn, unsigned long end_pfn) 1846 { 1847 unsigned long pfn; 1848 struct page *page; 1849 int rc = 0; 1850 1851 if (!hugepages_supported()) 1852 return rc; 1853 1854 for (pfn = start_pfn; pfn < end_pfn; pfn += 1 << minimum_order) { 1855 page = pfn_to_page(pfn); 1856 rc = dissolve_free_huge_page(page); 1857 if (rc) 1858 break; 1859 } 1860 1861 return rc; 1862 } 1863 1864 /* 1865 * Allocates a fresh surplus page from the page allocator. 1866 */ 1867 static struct page *alloc_surplus_huge_page(struct hstate *h, gfp_t gfp_mask, 1868 int nid, nodemask_t *nmask) 1869 { 1870 struct page *page = NULL; 1871 1872 if (hstate_is_gigantic(h)) 1873 return NULL; 1874 1875 spin_lock(&hugetlb_lock); 1876 if (h->surplus_huge_pages >= h->nr_overcommit_huge_pages) 1877 goto out_unlock; 1878 spin_unlock(&hugetlb_lock); 1879 1880 page = alloc_fresh_huge_page(h, gfp_mask, nid, nmask, NULL); 1881 if (!page) 1882 return NULL; 1883 1884 spin_lock(&hugetlb_lock); 1885 /* 1886 * We could have raced with the pool size change. 1887 * Double check that and simply deallocate the new page 1888 * if we would end up overcommiting the surpluses. Abuse 1889 * temporary page to workaround the nasty free_huge_page 1890 * codeflow 1891 */ 1892 if (h->surplus_huge_pages >= h->nr_overcommit_huge_pages) { 1893 SetPageHugeTemporary(page); 1894 spin_unlock(&hugetlb_lock); 1895 put_page(page); 1896 return NULL; 1897 } else { 1898 h->surplus_huge_pages++; 1899 h->surplus_huge_pages_node[page_to_nid(page)]++; 1900 } 1901 1902 out_unlock: 1903 spin_unlock(&hugetlb_lock); 1904 1905 return page; 1906 } 1907 1908 struct page *alloc_migrate_huge_page(struct hstate *h, gfp_t gfp_mask, 1909 int nid, nodemask_t *nmask) 1910 { 1911 struct page *page; 1912 1913 if (hstate_is_gigantic(h)) 1914 return NULL; 1915 1916 page = alloc_fresh_huge_page(h, gfp_mask, nid, nmask, NULL); 1917 if (!page) 1918 return NULL; 1919 1920 /* 1921 * We do not account these pages as surplus because they are only 1922 * temporary and will be released properly on the last reference 1923 */ 1924 SetPageHugeTemporary(page); 1925 1926 return page; 1927 } 1928 1929 /* 1930 * Use the VMA's mpolicy to allocate a huge page from the buddy. 1931 */ 1932 static 1933 struct page *alloc_buddy_huge_page_with_mpol(struct hstate *h, 1934 struct vm_area_struct *vma, unsigned long addr) 1935 { 1936 struct page *page; 1937 struct mempolicy *mpol; 1938 gfp_t gfp_mask = htlb_alloc_mask(h); 1939 int nid; 1940 nodemask_t *nodemask; 1941 1942 nid = huge_node(vma, addr, gfp_mask, &mpol, &nodemask); 1943 page = alloc_surplus_huge_page(h, gfp_mask, nid, nodemask); 1944 mpol_cond_put(mpol); 1945 1946 return page; 1947 } 1948 1949 /* page migration callback function */ 1950 struct page *alloc_huge_page_node(struct hstate *h, int nid) 1951 { 1952 gfp_t gfp_mask = htlb_alloc_mask(h); 1953 struct page *page = NULL; 1954 1955 if (nid != NUMA_NO_NODE) 1956 gfp_mask |= __GFP_THISNODE; 1957 1958 spin_lock(&hugetlb_lock); 1959 if (h->free_huge_pages - h->resv_huge_pages > 0) 1960 page = dequeue_huge_page_nodemask(h, gfp_mask, nid, NULL); 1961 spin_unlock(&hugetlb_lock); 1962 1963 if (!page) 1964 page = alloc_migrate_huge_page(h, gfp_mask, nid, NULL); 1965 1966 return page; 1967 } 1968 1969 /* page migration callback function */ 1970 struct page *alloc_huge_page_nodemask(struct hstate *h, int preferred_nid, 1971 nodemask_t *nmask) 1972 { 1973 gfp_t gfp_mask = htlb_alloc_mask(h); 1974 1975 spin_lock(&hugetlb_lock); 1976 if (h->free_huge_pages - h->resv_huge_pages > 0) { 1977 struct page *page; 1978 1979 page = dequeue_huge_page_nodemask(h, gfp_mask, preferred_nid, nmask); 1980 if (page) { 1981 spin_unlock(&hugetlb_lock); 1982 return page; 1983 } 1984 } 1985 spin_unlock(&hugetlb_lock); 1986 1987 return alloc_migrate_huge_page(h, gfp_mask, preferred_nid, nmask); 1988 } 1989 1990 /* mempolicy aware migration callback */ 1991 struct page *alloc_huge_page_vma(struct hstate *h, struct vm_area_struct *vma, 1992 unsigned long address) 1993 { 1994 struct mempolicy *mpol; 1995 nodemask_t *nodemask; 1996 struct page *page; 1997 gfp_t gfp_mask; 1998 int node; 1999 2000 gfp_mask = htlb_alloc_mask(h); 2001 node = huge_node(vma, address, gfp_mask, &mpol, &nodemask); 2002 page = alloc_huge_page_nodemask(h, node, nodemask); 2003 mpol_cond_put(mpol); 2004 2005 return page; 2006 } 2007 2008 /* 2009 * Increase the hugetlb pool such that it can accommodate a reservation 2010 * of size 'delta'. 2011 */ 2012 static int gather_surplus_pages(struct hstate *h, int delta) 2013 __must_hold(&hugetlb_lock) 2014 { 2015 struct list_head surplus_list; 2016 struct page *page, *tmp; 2017 int ret, i; 2018 int needed, allocated; 2019 bool alloc_ok = true; 2020 2021 needed = (h->resv_huge_pages + delta) - h->free_huge_pages; 2022 if (needed <= 0) { 2023 h->resv_huge_pages += delta; 2024 return 0; 2025 } 2026 2027 allocated = 0; 2028 INIT_LIST_HEAD(&surplus_list); 2029 2030 ret = -ENOMEM; 2031 retry: 2032 spin_unlock(&hugetlb_lock); 2033 for (i = 0; i < needed; i++) { 2034 page = alloc_surplus_huge_page(h, htlb_alloc_mask(h), 2035 NUMA_NO_NODE, NULL); 2036 if (!page) { 2037 alloc_ok = false; 2038 break; 2039 } 2040 list_add(&page->lru, &surplus_list); 2041 cond_resched(); 2042 } 2043 allocated += i; 2044 2045 /* 2046 * After retaking hugetlb_lock, we need to recalculate 'needed' 2047 * because either resv_huge_pages or free_huge_pages may have changed. 2048 */ 2049 spin_lock(&hugetlb_lock); 2050 needed = (h->resv_huge_pages + delta) - 2051 (h->free_huge_pages + allocated); 2052 if (needed > 0) { 2053 if (alloc_ok) 2054 goto retry; 2055 /* 2056 * We were not able to allocate enough pages to 2057 * satisfy the entire reservation so we free what 2058 * we've allocated so far. 2059 */ 2060 goto free; 2061 } 2062 /* 2063 * The surplus_list now contains _at_least_ the number of extra pages 2064 * needed to accommodate the reservation. Add the appropriate number 2065 * of pages to the hugetlb pool and free the extras back to the buddy 2066 * allocator. Commit the entire reservation here to prevent another 2067 * process from stealing the pages as they are added to the pool but 2068 * before they are reserved. 2069 */ 2070 needed += allocated; 2071 h->resv_huge_pages += delta; 2072 ret = 0; 2073 2074 /* Free the needed pages to the hugetlb pool */ 2075 list_for_each_entry_safe(page, tmp, &surplus_list, lru) { 2076 if ((--needed) < 0) 2077 break; 2078 /* 2079 * This page is now managed by the hugetlb allocator and has 2080 * no users -- drop the buddy allocator's reference. 2081 */ 2082 put_page_testzero(page); 2083 VM_BUG_ON_PAGE(page_count(page), page); 2084 enqueue_huge_page(h, page); 2085 } 2086 free: 2087 spin_unlock(&hugetlb_lock); 2088 2089 /* Free unnecessary surplus pages to the buddy allocator */ 2090 list_for_each_entry_safe(page, tmp, &surplus_list, lru) 2091 put_page(page); 2092 spin_lock(&hugetlb_lock); 2093 2094 return ret; 2095 } 2096 2097 /* 2098 * This routine has two main purposes: 2099 * 1) Decrement the reservation count (resv_huge_pages) by the value passed 2100 * in unused_resv_pages. This corresponds to the prior adjustments made 2101 * to the associated reservation map. 2102 * 2) Free any unused surplus pages that may have been allocated to satisfy 2103 * the reservation. As many as unused_resv_pages may be freed. 2104 * 2105 * Called with hugetlb_lock held. However, the lock could be dropped (and 2106 * reacquired) during calls to cond_resched_lock. Whenever dropping the lock, 2107 * we must make sure nobody else can claim pages we are in the process of 2108 * freeing. Do this by ensuring resv_huge_page always is greater than the 2109 * number of huge pages we plan to free when dropping the lock. 2110 */ 2111 static void return_unused_surplus_pages(struct hstate *h, 2112 unsigned long unused_resv_pages) 2113 { 2114 unsigned long nr_pages; 2115 2116 /* Cannot return gigantic pages currently */ 2117 if (hstate_is_gigantic(h)) 2118 goto out; 2119 2120 /* 2121 * Part (or even all) of the reservation could have been backed 2122 * by pre-allocated pages. Only free surplus pages. 2123 */ 2124 nr_pages = min(unused_resv_pages, h->surplus_huge_pages); 2125 2126 /* 2127 * We want to release as many surplus pages as possible, spread 2128 * evenly across all nodes with memory. Iterate across these nodes 2129 * until we can no longer free unreserved surplus pages. This occurs 2130 * when the nodes with surplus pages have no free pages. 2131 * free_pool_huge_page() will balance the the freed pages across the 2132 * on-line nodes with memory and will handle the hstate accounting. 2133 * 2134 * Note that we decrement resv_huge_pages as we free the pages. If 2135 * we drop the lock, resv_huge_pages will still be sufficiently large 2136 * to cover subsequent pages we may free. 2137 */ 2138 while (nr_pages--) { 2139 h->resv_huge_pages--; 2140 unused_resv_pages--; 2141 if (!free_pool_huge_page(h, &node_states[N_MEMORY], 1)) 2142 goto out; 2143 cond_resched_lock(&hugetlb_lock); 2144 } 2145 2146 out: 2147 /* Fully uncommit the reservation */ 2148 h->resv_huge_pages -= unused_resv_pages; 2149 } 2150 2151 2152 /* 2153 * vma_needs_reservation, vma_commit_reservation and vma_end_reservation 2154 * are used by the huge page allocation routines to manage reservations. 2155 * 2156 * vma_needs_reservation is called to determine if the huge page at addr 2157 * within the vma has an associated reservation. If a reservation is 2158 * needed, the value 1 is returned. The caller is then responsible for 2159 * managing the global reservation and subpool usage counts. After 2160 * the huge page has been allocated, vma_commit_reservation is called 2161 * to add the page to the reservation map. If the page allocation fails, 2162 * the reservation must be ended instead of committed. vma_end_reservation 2163 * is called in such cases. 2164 * 2165 * In the normal case, vma_commit_reservation returns the same value 2166 * as the preceding vma_needs_reservation call. The only time this 2167 * is not the case is if a reserve map was changed between calls. It 2168 * is the responsibility of the caller to notice the difference and 2169 * take appropriate action. 2170 * 2171 * vma_add_reservation is used in error paths where a reservation must 2172 * be restored when a newly allocated huge page must be freed. It is 2173 * to be called after calling vma_needs_reservation to determine if a 2174 * reservation exists. 2175 */ 2176 enum vma_resv_mode { 2177 VMA_NEEDS_RESV, 2178 VMA_COMMIT_RESV, 2179 VMA_END_RESV, 2180 VMA_ADD_RESV, 2181 }; 2182 static long __vma_reservation_common(struct hstate *h, 2183 struct vm_area_struct *vma, unsigned long addr, 2184 enum vma_resv_mode mode) 2185 { 2186 struct resv_map *resv; 2187 pgoff_t idx; 2188 long ret; 2189 long dummy_out_regions_needed; 2190 2191 resv = vma_resv_map(vma); 2192 if (!resv) 2193 return 1; 2194 2195 idx = vma_hugecache_offset(h, vma, addr); 2196 switch (mode) { 2197 case VMA_NEEDS_RESV: 2198 ret = region_chg(resv, idx, idx + 1, &dummy_out_regions_needed); 2199 /* We assume that vma_reservation_* routines always operate on 2200 * 1 page, and that adding to resv map a 1 page entry can only 2201 * ever require 1 region. 2202 */ 2203 VM_BUG_ON(dummy_out_regions_needed != 1); 2204 break; 2205 case VMA_COMMIT_RESV: 2206 ret = region_add(resv, idx, idx + 1, 1, NULL, NULL); 2207 /* region_add calls of range 1 should never fail. */ 2208 VM_BUG_ON(ret < 0); 2209 break; 2210 case VMA_END_RESV: 2211 region_abort(resv, idx, idx + 1, 1); 2212 ret = 0; 2213 break; 2214 case VMA_ADD_RESV: 2215 if (vma->vm_flags & VM_MAYSHARE) { 2216 ret = region_add(resv, idx, idx + 1, 1, NULL, NULL); 2217 /* region_add calls of range 1 should never fail. */ 2218 VM_BUG_ON(ret < 0); 2219 } else { 2220 region_abort(resv, idx, idx + 1, 1); 2221 ret = region_del(resv, idx, idx + 1); 2222 } 2223 break; 2224 default: 2225 BUG(); 2226 } 2227 2228 if (vma->vm_flags & VM_MAYSHARE) 2229 return ret; 2230 else if (is_vma_resv_set(vma, HPAGE_RESV_OWNER) && ret >= 0) { 2231 /* 2232 * In most cases, reserves always exist for private mappings. 2233 * However, a file associated with mapping could have been 2234 * hole punched or truncated after reserves were consumed. 2235 * As subsequent fault on such a range will not use reserves. 2236 * Subtle - The reserve map for private mappings has the 2237 * opposite meaning than that of shared mappings. If NO 2238 * entry is in the reserve map, it means a reservation exists. 2239 * If an entry exists in the reserve map, it means the 2240 * reservation has already been consumed. As a result, the 2241 * return value of this routine is the opposite of the 2242 * value returned from reserve map manipulation routines above. 2243 */ 2244 if (ret) 2245 return 0; 2246 else 2247 return 1; 2248 } 2249 else 2250 return ret < 0 ? ret : 0; 2251 } 2252 2253 static long vma_needs_reservation(struct hstate *h, 2254 struct vm_area_struct *vma, unsigned long addr) 2255 { 2256 return __vma_reservation_common(h, vma, addr, VMA_NEEDS_RESV); 2257 } 2258 2259 static long vma_commit_reservation(struct hstate *h, 2260 struct vm_area_struct *vma, unsigned long addr) 2261 { 2262 return __vma_reservation_common(h, vma, addr, VMA_COMMIT_RESV); 2263 } 2264 2265 static void vma_end_reservation(struct hstate *h, 2266 struct vm_area_struct *vma, unsigned long addr) 2267 { 2268 (void)__vma_reservation_common(h, vma, addr, VMA_END_RESV); 2269 } 2270 2271 static long vma_add_reservation(struct hstate *h, 2272 struct vm_area_struct *vma, unsigned long addr) 2273 { 2274 return __vma_reservation_common(h, vma, addr, VMA_ADD_RESV); 2275 } 2276 2277 /* 2278 * This routine is called to restore a reservation on error paths. In the 2279 * specific error paths, a huge page was allocated (via alloc_huge_page) 2280 * and is about to be freed. If a reservation for the page existed, 2281 * alloc_huge_page would have consumed the reservation and set PagePrivate 2282 * in the newly allocated page. When the page is freed via free_huge_page, 2283 * the global reservation count will be incremented if PagePrivate is set. 2284 * However, free_huge_page can not adjust the reserve map. Adjust the 2285 * reserve map here to be consistent with global reserve count adjustments 2286 * to be made by free_huge_page. 2287 */ 2288 static void restore_reserve_on_error(struct hstate *h, 2289 struct vm_area_struct *vma, unsigned long address, 2290 struct page *page) 2291 { 2292 if (unlikely(PagePrivate(page))) { 2293 long rc = vma_needs_reservation(h, vma, address); 2294 2295 if (unlikely(rc < 0)) { 2296 /* 2297 * Rare out of memory condition in reserve map 2298 * manipulation. Clear PagePrivate so that 2299 * global reserve count will not be incremented 2300 * by free_huge_page. This will make it appear 2301 * as though the reservation for this page was 2302 * consumed. This may prevent the task from 2303 * faulting in the page at a later time. This 2304 * is better than inconsistent global huge page 2305 * accounting of reserve counts. 2306 */ 2307 ClearPagePrivate(page); 2308 } else if (rc) { 2309 rc = vma_add_reservation(h, vma, address); 2310 if (unlikely(rc < 0)) 2311 /* 2312 * See above comment about rare out of 2313 * memory condition. 2314 */ 2315 ClearPagePrivate(page); 2316 } else 2317 vma_end_reservation(h, vma, address); 2318 } 2319 } 2320 2321 struct page *alloc_huge_page(struct vm_area_struct *vma, 2322 unsigned long addr, int avoid_reserve) 2323 { 2324 struct hugepage_subpool *spool = subpool_vma(vma); 2325 struct hstate *h = hstate_vma(vma); 2326 struct page *page; 2327 long map_chg, map_commit; 2328 long gbl_chg; 2329 int ret, idx; 2330 struct hugetlb_cgroup *h_cg; 2331 bool deferred_reserve; 2332 2333 idx = hstate_index(h); 2334 /* 2335 * Examine the region/reserve map to determine if the process 2336 * has a reservation for the page to be allocated. A return 2337 * code of zero indicates a reservation exists (no change). 2338 */ 2339 map_chg = gbl_chg = vma_needs_reservation(h, vma, addr); 2340 if (map_chg < 0) 2341 return ERR_PTR(-ENOMEM); 2342 2343 /* 2344 * Processes that did not create the mapping will have no 2345 * reserves as indicated by the region/reserve map. Check 2346 * that the allocation will not exceed the subpool limit. 2347 * Allocations for MAP_NORESERVE mappings also need to be 2348 * checked against any subpool limit. 2349 */ 2350 if (map_chg || avoid_reserve) { 2351 gbl_chg = hugepage_subpool_get_pages(spool, 1); 2352 if (gbl_chg < 0) { 2353 vma_end_reservation(h, vma, addr); 2354 return ERR_PTR(-ENOSPC); 2355 } 2356 2357 /* 2358 * Even though there was no reservation in the region/reserve 2359 * map, there could be reservations associated with the 2360 * subpool that can be used. This would be indicated if the 2361 * return value of hugepage_subpool_get_pages() is zero. 2362 * However, if avoid_reserve is specified we still avoid even 2363 * the subpool reservations. 2364 */ 2365 if (avoid_reserve) 2366 gbl_chg = 1; 2367 } 2368 2369 /* If this allocation is not consuming a reservation, charge it now. 2370 */ 2371 deferred_reserve = map_chg || avoid_reserve || !vma_resv_map(vma); 2372 if (deferred_reserve) { 2373 ret = hugetlb_cgroup_charge_cgroup_rsvd( 2374 idx, pages_per_huge_page(h), &h_cg); 2375 if (ret) 2376 goto out_subpool_put; 2377 } 2378 2379 ret = hugetlb_cgroup_charge_cgroup(idx, pages_per_huge_page(h), &h_cg); 2380 if (ret) 2381 goto out_uncharge_cgroup_reservation; 2382 2383 spin_lock(&hugetlb_lock); 2384 /* 2385 * glb_chg is passed to indicate whether or not a page must be taken 2386 * from the global free pool (global change). gbl_chg == 0 indicates 2387 * a reservation exists for the allocation. 2388 */ 2389 page = dequeue_huge_page_vma(h, vma, addr, avoid_reserve, gbl_chg); 2390 if (!page) { 2391 spin_unlock(&hugetlb_lock); 2392 page = alloc_buddy_huge_page_with_mpol(h, vma, addr); 2393 if (!page) 2394 goto out_uncharge_cgroup; 2395 if (!avoid_reserve && vma_has_reserves(vma, gbl_chg)) { 2396 SetPagePrivate(page); 2397 h->resv_huge_pages--; 2398 } 2399 spin_lock(&hugetlb_lock); 2400 list_move(&page->lru, &h->hugepage_activelist); 2401 /* Fall through */ 2402 } 2403 hugetlb_cgroup_commit_charge(idx, pages_per_huge_page(h), h_cg, page); 2404 /* If allocation is not consuming a reservation, also store the 2405 * hugetlb_cgroup pointer on the page. 2406 */ 2407 if (deferred_reserve) { 2408 hugetlb_cgroup_commit_charge_rsvd(idx, pages_per_huge_page(h), 2409 h_cg, page); 2410 } 2411 2412 spin_unlock(&hugetlb_lock); 2413 2414 set_page_private(page, (unsigned long)spool); 2415 2416 map_commit = vma_commit_reservation(h, vma, addr); 2417 if (unlikely(map_chg > map_commit)) { 2418 /* 2419 * The page was added to the reservation map between 2420 * vma_needs_reservation and vma_commit_reservation. 2421 * This indicates a race with hugetlb_reserve_pages. 2422 * Adjust for the subpool count incremented above AND 2423 * in hugetlb_reserve_pages for the same page. Also, 2424 * the reservation count added in hugetlb_reserve_pages 2425 * no longer applies. 2426 */ 2427 long rsv_adjust; 2428 2429 rsv_adjust = hugepage_subpool_put_pages(spool, 1); 2430 hugetlb_acct_memory(h, -rsv_adjust); 2431 } 2432 return page; 2433 2434 out_uncharge_cgroup: 2435 hugetlb_cgroup_uncharge_cgroup(idx, pages_per_huge_page(h), h_cg); 2436 out_uncharge_cgroup_reservation: 2437 if (deferred_reserve) 2438 hugetlb_cgroup_uncharge_cgroup_rsvd(idx, pages_per_huge_page(h), 2439 h_cg); 2440 out_subpool_put: 2441 if (map_chg || avoid_reserve) 2442 hugepage_subpool_put_pages(spool, 1); 2443 vma_end_reservation(h, vma, addr); 2444 return ERR_PTR(-ENOSPC); 2445 } 2446 2447 int alloc_bootmem_huge_page(struct hstate *h) 2448 __attribute__ ((weak, alias("__alloc_bootmem_huge_page"))); 2449 int __alloc_bootmem_huge_page(struct hstate *h) 2450 { 2451 struct huge_bootmem_page *m; 2452 int nr_nodes, node; 2453 2454 for_each_node_mask_to_alloc(h, nr_nodes, node, &node_states[N_MEMORY]) { 2455 void *addr; 2456 2457 addr = memblock_alloc_try_nid_raw( 2458 huge_page_size(h), huge_page_size(h), 2459 0, MEMBLOCK_ALLOC_ACCESSIBLE, node); 2460 if (addr) { 2461 /* 2462 * Use the beginning of the huge page to store the 2463 * huge_bootmem_page struct (until gather_bootmem 2464 * puts them into the mem_map). 2465 */ 2466 m = addr; 2467 goto found; 2468 } 2469 } 2470 return 0; 2471 2472 found: 2473 BUG_ON(!IS_ALIGNED(virt_to_phys(m), huge_page_size(h))); 2474 /* Put them into a private list first because mem_map is not up yet */ 2475 INIT_LIST_HEAD(&m->list); 2476 list_add(&m->list, &huge_boot_pages); 2477 m->hstate = h; 2478 return 1; 2479 } 2480 2481 static void __init prep_compound_huge_page(struct page *page, 2482 unsigned int order) 2483 { 2484 if (unlikely(order > (MAX_ORDER - 1))) 2485 prep_compound_gigantic_page(page, order); 2486 else 2487 prep_compound_page(page, order); 2488 } 2489 2490 /* Put bootmem huge pages into the standard lists after mem_map is up */ 2491 static void __init gather_bootmem_prealloc(void) 2492 { 2493 struct huge_bootmem_page *m; 2494 2495 list_for_each_entry(m, &huge_boot_pages, list) { 2496 struct page *page = virt_to_page(m); 2497 struct hstate *h = m->hstate; 2498 2499 WARN_ON(page_count(page) != 1); 2500 prep_compound_huge_page(page, h->order); 2501 WARN_ON(PageReserved(page)); 2502 prep_new_huge_page(h, page, page_to_nid(page)); 2503 put_page(page); /* free it into the hugepage allocator */ 2504 2505 /* 2506 * If we had gigantic hugepages allocated at boot time, we need 2507 * to restore the 'stolen' pages to totalram_pages in order to 2508 * fix confusing memory reports from free(1) and another 2509 * side-effects, like CommitLimit going negative. 2510 */ 2511 if (hstate_is_gigantic(h)) 2512 adjust_managed_page_count(page, 1 << h->order); 2513 cond_resched(); 2514 } 2515 } 2516 2517 static void __init hugetlb_hstate_alloc_pages(struct hstate *h) 2518 { 2519 unsigned long i; 2520 nodemask_t *node_alloc_noretry; 2521 2522 if (!hstate_is_gigantic(h)) { 2523 /* 2524 * Bit mask controlling how hard we retry per-node allocations. 2525 * Ignore errors as lower level routines can deal with 2526 * node_alloc_noretry == NULL. If this kmalloc fails at boot 2527 * time, we are likely in bigger trouble. 2528 */ 2529 node_alloc_noretry = kmalloc(sizeof(*node_alloc_noretry), 2530 GFP_KERNEL); 2531 } else { 2532 /* allocations done at boot time */ 2533 node_alloc_noretry = NULL; 2534 } 2535 2536 /* bit mask controlling how hard we retry per-node allocations */ 2537 if (node_alloc_noretry) 2538 nodes_clear(*node_alloc_noretry); 2539 2540 for (i = 0; i < h->max_huge_pages; ++i) { 2541 if (hstate_is_gigantic(h)) { 2542 if (!alloc_bootmem_huge_page(h)) 2543 break; 2544 } else if (!alloc_pool_huge_page(h, 2545 &node_states[N_MEMORY], 2546 node_alloc_noretry)) 2547 break; 2548 cond_resched(); 2549 } 2550 if (i < h->max_huge_pages) { 2551 char buf[32]; 2552 2553 string_get_size(huge_page_size(h), 1, STRING_UNITS_2, buf, 32); 2554 pr_warn("HugeTLB: allocating %lu of page size %s failed. Only allocated %lu hugepages.\n", 2555 h->max_huge_pages, buf, i); 2556 h->max_huge_pages = i; 2557 } 2558 2559 kfree(node_alloc_noretry); 2560 } 2561 2562 static void __init hugetlb_init_hstates(void) 2563 { 2564 struct hstate *h; 2565 2566 for_each_hstate(h) { 2567 if (minimum_order > huge_page_order(h)) 2568 minimum_order = huge_page_order(h); 2569 2570 /* oversize hugepages were init'ed in early boot */ 2571 if (!hstate_is_gigantic(h)) 2572 hugetlb_hstate_alloc_pages(h); 2573 } 2574 VM_BUG_ON(minimum_order == UINT_MAX); 2575 } 2576 2577 static void __init report_hugepages(void) 2578 { 2579 struct hstate *h; 2580 2581 for_each_hstate(h) { 2582 char buf[32]; 2583 2584 string_get_size(huge_page_size(h), 1, STRING_UNITS_2, buf, 32); 2585 pr_info("HugeTLB registered %s page size, pre-allocated %ld pages\n", 2586 buf, h->free_huge_pages); 2587 } 2588 } 2589 2590 #ifdef CONFIG_HIGHMEM 2591 static void try_to_free_low(struct hstate *h, unsigned long count, 2592 nodemask_t *nodes_allowed) 2593 { 2594 int i; 2595 2596 if (hstate_is_gigantic(h)) 2597 return; 2598 2599 for_each_node_mask(i, *nodes_allowed) { 2600 struct page *page, *next; 2601 struct list_head *freel = &h->hugepage_freelists[i]; 2602 list_for_each_entry_safe(page, next, freel, lru) { 2603 if (count >= h->nr_huge_pages) 2604 return; 2605 if (PageHighMem(page)) 2606 continue; 2607 list_del(&page->lru); 2608 update_and_free_page(h, page); 2609 h->free_huge_pages--; 2610 h->free_huge_pages_node[page_to_nid(page)]--; 2611 } 2612 } 2613 } 2614 #else 2615 static inline void try_to_free_low(struct hstate *h, unsigned long count, 2616 nodemask_t *nodes_allowed) 2617 { 2618 } 2619 #endif 2620 2621 /* 2622 * Increment or decrement surplus_huge_pages. Keep node-specific counters 2623 * balanced by operating on them in a round-robin fashion. 2624 * Returns 1 if an adjustment was made. 2625 */ 2626 static int adjust_pool_surplus(struct hstate *h, nodemask_t *nodes_allowed, 2627 int delta) 2628 { 2629 int nr_nodes, node; 2630 2631 VM_BUG_ON(delta != -1 && delta != 1); 2632 2633 if (delta < 0) { 2634 for_each_node_mask_to_alloc(h, nr_nodes, node, nodes_allowed) { 2635 if (h->surplus_huge_pages_node[node]) 2636 goto found; 2637 } 2638 } else { 2639 for_each_node_mask_to_free(h, nr_nodes, node, nodes_allowed) { 2640 if (h->surplus_huge_pages_node[node] < 2641 h->nr_huge_pages_node[node]) 2642 goto found; 2643 } 2644 } 2645 return 0; 2646 2647 found: 2648 h->surplus_huge_pages += delta; 2649 h->surplus_huge_pages_node[node] += delta; 2650 return 1; 2651 } 2652 2653 #define persistent_huge_pages(h) (h->nr_huge_pages - h->surplus_huge_pages) 2654 static int set_max_huge_pages(struct hstate *h, unsigned long count, int nid, 2655 nodemask_t *nodes_allowed) 2656 { 2657 unsigned long min_count, ret; 2658 NODEMASK_ALLOC(nodemask_t, node_alloc_noretry, GFP_KERNEL); 2659 2660 /* 2661 * Bit mask controlling how hard we retry per-node allocations. 2662 * If we can not allocate the bit mask, do not attempt to allocate 2663 * the requested huge pages. 2664 */ 2665 if (node_alloc_noretry) 2666 nodes_clear(*node_alloc_noretry); 2667 else 2668 return -ENOMEM; 2669 2670 spin_lock(&hugetlb_lock); 2671 2672 /* 2673 * Check for a node specific request. 2674 * Changing node specific huge page count may require a corresponding 2675 * change to the global count. In any case, the passed node mask 2676 * (nodes_allowed) will restrict alloc/free to the specified node. 2677 */ 2678 if (nid != NUMA_NO_NODE) { 2679 unsigned long old_count = count; 2680 2681 count += h->nr_huge_pages - h->nr_huge_pages_node[nid]; 2682 /* 2683 * User may have specified a large count value which caused the 2684 * above calculation to overflow. In this case, they wanted 2685 * to allocate as many huge pages as possible. Set count to 2686 * largest possible value to align with their intention. 2687 */ 2688 if (count < old_count) 2689 count = ULONG_MAX; 2690 } 2691 2692 /* 2693 * Gigantic pages runtime allocation depend on the capability for large 2694 * page range allocation. 2695 * If the system does not provide this feature, return an error when 2696 * the user tries to allocate gigantic pages but let the user free the 2697 * boottime allocated gigantic pages. 2698 */ 2699 if (hstate_is_gigantic(h) && !IS_ENABLED(CONFIG_CONTIG_ALLOC)) { 2700 if (count > persistent_huge_pages(h)) { 2701 spin_unlock(&hugetlb_lock); 2702 NODEMASK_FREE(node_alloc_noretry); 2703 return -EINVAL; 2704 } 2705 /* Fall through to decrease pool */ 2706 } 2707 2708 /* 2709 * Increase the pool size 2710 * First take pages out of surplus state. Then make up the 2711 * remaining difference by allocating fresh huge pages. 2712 * 2713 * We might race with alloc_surplus_huge_page() here and be unable 2714 * to convert a surplus huge page to a normal huge page. That is 2715 * not critical, though, it just means the overall size of the 2716 * pool might be one hugepage larger than it needs to be, but 2717 * within all the constraints specified by the sysctls. 2718 */ 2719 while (h->surplus_huge_pages && count > persistent_huge_pages(h)) { 2720 if (!adjust_pool_surplus(h, nodes_allowed, -1)) 2721 break; 2722 } 2723 2724 while (count > persistent_huge_pages(h)) { 2725 /* 2726 * If this allocation races such that we no longer need the 2727 * page, free_huge_page will handle it by freeing the page 2728 * and reducing the surplus. 2729 */ 2730 spin_unlock(&hugetlb_lock); 2731 2732 /* yield cpu to avoid soft lockup */ 2733 cond_resched(); 2734 2735 ret = alloc_pool_huge_page(h, nodes_allowed, 2736 node_alloc_noretry); 2737 spin_lock(&hugetlb_lock); 2738 if (!ret) 2739 goto out; 2740 2741 /* Bail for signals. Probably ctrl-c from user */ 2742 if (signal_pending(current)) 2743 goto out; 2744 } 2745 2746 /* 2747 * Decrease the pool size 2748 * First return free pages to the buddy allocator (being careful 2749 * to keep enough around to satisfy reservations). Then place 2750 * pages into surplus state as needed so the pool will shrink 2751 * to the desired size as pages become free. 2752 * 2753 * By placing pages into the surplus state independent of the 2754 * overcommit value, we are allowing the surplus pool size to 2755 * exceed overcommit. There are few sane options here. Since 2756 * alloc_surplus_huge_page() is checking the global counter, 2757 * though, we'll note that we're not allowed to exceed surplus 2758 * and won't grow the pool anywhere else. Not until one of the 2759 * sysctls are changed, or the surplus pages go out of use. 2760 */ 2761 min_count = h->resv_huge_pages + h->nr_huge_pages - h->free_huge_pages; 2762 min_count = max(count, min_count); 2763 try_to_free_low(h, min_count, nodes_allowed); 2764 while (min_count < persistent_huge_pages(h)) { 2765 if (!free_pool_huge_page(h, nodes_allowed, 0)) 2766 break; 2767 cond_resched_lock(&hugetlb_lock); 2768 } 2769 while (count < persistent_huge_pages(h)) { 2770 if (!adjust_pool_surplus(h, nodes_allowed, 1)) 2771 break; 2772 } 2773 out: 2774 h->max_huge_pages = persistent_huge_pages(h); 2775 spin_unlock(&hugetlb_lock); 2776 2777 NODEMASK_FREE(node_alloc_noretry); 2778 2779 return 0; 2780 } 2781 2782 #define HSTATE_ATTR_RO(_name) \ 2783 static struct kobj_attribute _name##_attr = __ATTR_RO(_name) 2784 2785 #define HSTATE_ATTR(_name) \ 2786 static struct kobj_attribute _name##_attr = \ 2787 __ATTR(_name, 0644, _name##_show, _name##_store) 2788 2789 static struct kobject *hugepages_kobj; 2790 static struct kobject *hstate_kobjs[HUGE_MAX_HSTATE]; 2791 2792 static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp); 2793 2794 static struct hstate *kobj_to_hstate(struct kobject *kobj, int *nidp) 2795 { 2796 int i; 2797 2798 for (i = 0; i < HUGE_MAX_HSTATE; i++) 2799 if (hstate_kobjs[i] == kobj) { 2800 if (nidp) 2801 *nidp = NUMA_NO_NODE; 2802 return &hstates[i]; 2803 } 2804 2805 return kobj_to_node_hstate(kobj, nidp); 2806 } 2807 2808 static ssize_t nr_hugepages_show_common(struct kobject *kobj, 2809 struct kobj_attribute *attr, char *buf) 2810 { 2811 struct hstate *h; 2812 unsigned long nr_huge_pages; 2813 int nid; 2814 2815 h = kobj_to_hstate(kobj, &nid); 2816 if (nid == NUMA_NO_NODE) 2817 nr_huge_pages = h->nr_huge_pages; 2818 else 2819 nr_huge_pages = h->nr_huge_pages_node[nid]; 2820 2821 return sprintf(buf, "%lu\n", nr_huge_pages); 2822 } 2823 2824 static ssize_t __nr_hugepages_store_common(bool obey_mempolicy, 2825 struct hstate *h, int nid, 2826 unsigned long count, size_t len) 2827 { 2828 int err; 2829 nodemask_t nodes_allowed, *n_mask; 2830 2831 if (hstate_is_gigantic(h) && !gigantic_page_runtime_supported()) 2832 return -EINVAL; 2833 2834 if (nid == NUMA_NO_NODE) { 2835 /* 2836 * global hstate attribute 2837 */ 2838 if (!(obey_mempolicy && 2839 init_nodemask_of_mempolicy(&nodes_allowed))) 2840 n_mask = &node_states[N_MEMORY]; 2841 else 2842 n_mask = &nodes_allowed; 2843 } else { 2844 /* 2845 * Node specific request. count adjustment happens in 2846 * set_max_huge_pages() after acquiring hugetlb_lock. 2847 */ 2848 init_nodemask_of_node(&nodes_allowed, nid); 2849 n_mask = &nodes_allowed; 2850 } 2851 2852 err = set_max_huge_pages(h, count, nid, n_mask); 2853 2854 return err ? err : len; 2855 } 2856 2857 static ssize_t nr_hugepages_store_common(bool obey_mempolicy, 2858 struct kobject *kobj, const char *buf, 2859 size_t len) 2860 { 2861 struct hstate *h; 2862 unsigned long count; 2863 int nid; 2864 int err; 2865 2866 err = kstrtoul(buf, 10, &count); 2867 if (err) 2868 return err; 2869 2870 h = kobj_to_hstate(kobj, &nid); 2871 return __nr_hugepages_store_common(obey_mempolicy, h, nid, count, len); 2872 } 2873 2874 static ssize_t nr_hugepages_show(struct kobject *kobj, 2875 struct kobj_attribute *attr, char *buf) 2876 { 2877 return nr_hugepages_show_common(kobj, attr, buf); 2878 } 2879 2880 static ssize_t nr_hugepages_store(struct kobject *kobj, 2881 struct kobj_attribute *attr, const char *buf, size_t len) 2882 { 2883 return nr_hugepages_store_common(false, kobj, buf, len); 2884 } 2885 HSTATE_ATTR(nr_hugepages); 2886 2887 #ifdef CONFIG_NUMA 2888 2889 /* 2890 * hstate attribute for optionally mempolicy-based constraint on persistent 2891 * huge page alloc/free. 2892 */ 2893 static ssize_t nr_hugepages_mempolicy_show(struct kobject *kobj, 2894 struct kobj_attribute *attr, char *buf) 2895 { 2896 return nr_hugepages_show_common(kobj, attr, buf); 2897 } 2898 2899 static ssize_t nr_hugepages_mempolicy_store(struct kobject *kobj, 2900 struct kobj_attribute *attr, const char *buf, size_t len) 2901 { 2902 return nr_hugepages_store_common(true, kobj, buf, len); 2903 } 2904 HSTATE_ATTR(nr_hugepages_mempolicy); 2905 #endif 2906 2907 2908 static ssize_t nr_overcommit_hugepages_show(struct kobject *kobj, 2909 struct kobj_attribute *attr, char *buf) 2910 { 2911 struct hstate *h = kobj_to_hstate(kobj, NULL); 2912 return sprintf(buf, "%lu\n", h->nr_overcommit_huge_pages); 2913 } 2914 2915 static ssize_t nr_overcommit_hugepages_store(struct kobject *kobj, 2916 struct kobj_attribute *attr, const char *buf, size_t count) 2917 { 2918 int err; 2919 unsigned long input; 2920 struct hstate *h = kobj_to_hstate(kobj, NULL); 2921 2922 if (hstate_is_gigantic(h)) 2923 return -EINVAL; 2924 2925 err = kstrtoul(buf, 10, &input); 2926 if (err) 2927 return err; 2928 2929 spin_lock(&hugetlb_lock); 2930 h->nr_overcommit_huge_pages = input; 2931 spin_unlock(&hugetlb_lock); 2932 2933 return count; 2934 } 2935 HSTATE_ATTR(nr_overcommit_hugepages); 2936 2937 static ssize_t free_hugepages_show(struct kobject *kobj, 2938 struct kobj_attribute *attr, char *buf) 2939 { 2940 struct hstate *h; 2941 unsigned long free_huge_pages; 2942 int nid; 2943 2944 h = kobj_to_hstate(kobj, &nid); 2945 if (nid == NUMA_NO_NODE) 2946 free_huge_pages = h->free_huge_pages; 2947 else 2948 free_huge_pages = h->free_huge_pages_node[nid]; 2949 2950 return sprintf(buf, "%lu\n", free_huge_pages); 2951 } 2952 HSTATE_ATTR_RO(free_hugepages); 2953 2954 static ssize_t resv_hugepages_show(struct kobject *kobj, 2955 struct kobj_attribute *attr, char *buf) 2956 { 2957 struct hstate *h = kobj_to_hstate(kobj, NULL); 2958 return sprintf(buf, "%lu\n", h->resv_huge_pages); 2959 } 2960 HSTATE_ATTR_RO(resv_hugepages); 2961 2962 static ssize_t surplus_hugepages_show(struct kobject *kobj, 2963 struct kobj_attribute *attr, char *buf) 2964 { 2965 struct hstate *h; 2966 unsigned long surplus_huge_pages; 2967 int nid; 2968 2969 h = kobj_to_hstate(kobj, &nid); 2970 if (nid == NUMA_NO_NODE) 2971 surplus_huge_pages = h->surplus_huge_pages; 2972 else 2973 surplus_huge_pages = h->surplus_huge_pages_node[nid]; 2974 2975 return sprintf(buf, "%lu\n", surplus_huge_pages); 2976 } 2977 HSTATE_ATTR_RO(surplus_hugepages); 2978 2979 static struct attribute *hstate_attrs[] = { 2980 &nr_hugepages_attr.attr, 2981 &nr_overcommit_hugepages_attr.attr, 2982 &free_hugepages_attr.attr, 2983 &resv_hugepages_attr.attr, 2984 &surplus_hugepages_attr.attr, 2985 #ifdef CONFIG_NUMA 2986 &nr_hugepages_mempolicy_attr.attr, 2987 #endif 2988 NULL, 2989 }; 2990 2991 static const struct attribute_group hstate_attr_group = { 2992 .attrs = hstate_attrs, 2993 }; 2994 2995 static int hugetlb_sysfs_add_hstate(struct hstate *h, struct kobject *parent, 2996 struct kobject **hstate_kobjs, 2997 const struct attribute_group *hstate_attr_group) 2998 { 2999 int retval; 3000 int hi = hstate_index(h); 3001 3002 hstate_kobjs[hi] = kobject_create_and_add(h->name, parent); 3003 if (!hstate_kobjs[hi]) 3004 return -ENOMEM; 3005 3006 retval = sysfs_create_group(hstate_kobjs[hi], hstate_attr_group); 3007 if (retval) 3008 kobject_put(hstate_kobjs[hi]); 3009 3010 return retval; 3011 } 3012 3013 static void __init hugetlb_sysfs_init(void) 3014 { 3015 struct hstate *h; 3016 int err; 3017 3018 hugepages_kobj = kobject_create_and_add("hugepages", mm_kobj); 3019 if (!hugepages_kobj) 3020 return; 3021 3022 for_each_hstate(h) { 3023 err = hugetlb_sysfs_add_hstate(h, hugepages_kobj, 3024 hstate_kobjs, &hstate_attr_group); 3025 if (err) 3026 pr_err("Hugetlb: Unable to add hstate %s", h->name); 3027 } 3028 } 3029 3030 #ifdef CONFIG_NUMA 3031 3032 /* 3033 * node_hstate/s - associate per node hstate attributes, via their kobjects, 3034 * with node devices in node_devices[] using a parallel array. The array 3035 * index of a node device or _hstate == node id. 3036 * This is here to avoid any static dependency of the node device driver, in 3037 * the base kernel, on the hugetlb module. 3038 */ 3039 struct node_hstate { 3040 struct kobject *hugepages_kobj; 3041 struct kobject *hstate_kobjs[HUGE_MAX_HSTATE]; 3042 }; 3043 static struct node_hstate node_hstates[MAX_NUMNODES]; 3044 3045 /* 3046 * A subset of global hstate attributes for node devices 3047 */ 3048 static struct attribute *per_node_hstate_attrs[] = { 3049 &nr_hugepages_attr.attr, 3050 &free_hugepages_attr.attr, 3051 &surplus_hugepages_attr.attr, 3052 NULL, 3053 }; 3054 3055 static const struct attribute_group per_node_hstate_attr_group = { 3056 .attrs = per_node_hstate_attrs, 3057 }; 3058 3059 /* 3060 * kobj_to_node_hstate - lookup global hstate for node device hstate attr kobj. 3061 * Returns node id via non-NULL nidp. 3062 */ 3063 static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp) 3064 { 3065 int nid; 3066 3067 for (nid = 0; nid < nr_node_ids; nid++) { 3068 struct node_hstate *nhs = &node_hstates[nid]; 3069 int i; 3070 for (i = 0; i < HUGE_MAX_HSTATE; i++) 3071 if (nhs->hstate_kobjs[i] == kobj) { 3072 if (nidp) 3073 *nidp = nid; 3074 return &hstates[i]; 3075 } 3076 } 3077 3078 BUG(); 3079 return NULL; 3080 } 3081 3082 /* 3083 * Unregister hstate attributes from a single node device. 3084 * No-op if no hstate attributes attached. 3085 */ 3086 static void hugetlb_unregister_node(struct node *node) 3087 { 3088 struct hstate *h; 3089 struct node_hstate *nhs = &node_hstates[node->dev.id]; 3090 3091 if (!nhs->hugepages_kobj) 3092 return; /* no hstate attributes */ 3093 3094 for_each_hstate(h) { 3095 int idx = hstate_index(h); 3096 if (nhs->hstate_kobjs[idx]) { 3097 kobject_put(nhs->hstate_kobjs[idx]); 3098 nhs->hstate_kobjs[idx] = NULL; 3099 } 3100 } 3101 3102 kobject_put(nhs->hugepages_kobj); 3103 nhs->hugepages_kobj = NULL; 3104 } 3105 3106 3107 /* 3108 * Register hstate attributes for a single node device. 3109 * No-op if attributes already registered. 3110 */ 3111 static void hugetlb_register_node(struct node *node) 3112 { 3113 struct hstate *h; 3114 struct node_hstate *nhs = &node_hstates[node->dev.id]; 3115 int err; 3116 3117 if (nhs->hugepages_kobj) 3118 return; /* already allocated */ 3119 3120 nhs->hugepages_kobj = kobject_create_and_add("hugepages", 3121 &node->dev.kobj); 3122 if (!nhs->hugepages_kobj) 3123 return; 3124 3125 for_each_hstate(h) { 3126 err = hugetlb_sysfs_add_hstate(h, nhs->hugepages_kobj, 3127 nhs->hstate_kobjs, 3128 &per_node_hstate_attr_group); 3129 if (err) { 3130 pr_err("Hugetlb: Unable to add hstate %s for node %d\n", 3131 h->name, node->dev.id); 3132 hugetlb_unregister_node(node); 3133 break; 3134 } 3135 } 3136 } 3137 3138 /* 3139 * hugetlb init time: register hstate attributes for all registered node 3140 * devices of nodes that have memory. All on-line nodes should have 3141 * registered their associated device by this time. 3142 */ 3143 static void __init hugetlb_register_all_nodes(void) 3144 { 3145 int nid; 3146 3147 for_each_node_state(nid, N_MEMORY) { 3148 struct node *node = node_devices[nid]; 3149 if (node->dev.id == nid) 3150 hugetlb_register_node(node); 3151 } 3152 3153 /* 3154 * Let the node device driver know we're here so it can 3155 * [un]register hstate attributes on node hotplug. 3156 */ 3157 register_hugetlbfs_with_node(hugetlb_register_node, 3158 hugetlb_unregister_node); 3159 } 3160 #else /* !CONFIG_NUMA */ 3161 3162 static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp) 3163 { 3164 BUG(); 3165 if (nidp) 3166 *nidp = -1; 3167 return NULL; 3168 } 3169 3170 static void hugetlb_register_all_nodes(void) { } 3171 3172 #endif 3173 3174 static int __init hugetlb_init(void) 3175 { 3176 int i; 3177 3178 if (!hugepages_supported()) 3179 return 0; 3180 3181 if (!size_to_hstate(default_hstate_size)) { 3182 if (default_hstate_size != 0) { 3183 pr_err("HugeTLB: unsupported default_hugepagesz %lu. Reverting to %lu\n", 3184 default_hstate_size, HPAGE_SIZE); 3185 } 3186 3187 default_hstate_size = HPAGE_SIZE; 3188 if (!size_to_hstate(default_hstate_size)) 3189 hugetlb_add_hstate(HUGETLB_PAGE_ORDER); 3190 } 3191 default_hstate_idx = hstate_index(size_to_hstate(default_hstate_size)); 3192 if (default_hstate_max_huge_pages) { 3193 if (!default_hstate.max_huge_pages) 3194 default_hstate.max_huge_pages = default_hstate_max_huge_pages; 3195 } 3196 3197 hugetlb_init_hstates(); 3198 gather_bootmem_prealloc(); 3199 report_hugepages(); 3200 3201 hugetlb_sysfs_init(); 3202 hugetlb_register_all_nodes(); 3203 hugetlb_cgroup_file_init(); 3204 3205 #ifdef CONFIG_SMP 3206 num_fault_mutexes = roundup_pow_of_two(8 * num_possible_cpus()); 3207 #else 3208 num_fault_mutexes = 1; 3209 #endif 3210 hugetlb_fault_mutex_table = 3211 kmalloc_array(num_fault_mutexes, sizeof(struct mutex), 3212 GFP_KERNEL); 3213 BUG_ON(!hugetlb_fault_mutex_table); 3214 3215 for (i = 0; i < num_fault_mutexes; i++) 3216 mutex_init(&hugetlb_fault_mutex_table[i]); 3217 return 0; 3218 } 3219 subsys_initcall(hugetlb_init); 3220 3221 /* Should be called on processing a hugepagesz=... option */ 3222 void __init hugetlb_bad_size(void) 3223 { 3224 parsed_valid_hugepagesz = false; 3225 } 3226 3227 void __init hugetlb_add_hstate(unsigned int order) 3228 { 3229 struct hstate *h; 3230 unsigned long i; 3231 3232 if (size_to_hstate(PAGE_SIZE << order)) { 3233 pr_warn("hugepagesz= specified twice, ignoring\n"); 3234 return; 3235 } 3236 BUG_ON(hugetlb_max_hstate >= HUGE_MAX_HSTATE); 3237 BUG_ON(order == 0); 3238 h = &hstates[hugetlb_max_hstate++]; 3239 h->order = order; 3240 h->mask = ~((1ULL << (order + PAGE_SHIFT)) - 1); 3241 h->nr_huge_pages = 0; 3242 h->free_huge_pages = 0; 3243 for (i = 0; i < MAX_NUMNODES; ++i) 3244 INIT_LIST_HEAD(&h->hugepage_freelists[i]); 3245 INIT_LIST_HEAD(&h->hugepage_activelist); 3246 h->next_nid_to_alloc = first_memory_node; 3247 h->next_nid_to_free = first_memory_node; 3248 snprintf(h->name, HSTATE_NAME_LEN, "hugepages-%lukB", 3249 huge_page_size(h)/1024); 3250 3251 parsed_hstate = h; 3252 } 3253 3254 static int __init hugetlb_nrpages_setup(char *s) 3255 { 3256 unsigned long *mhp; 3257 static unsigned long *last_mhp; 3258 3259 if (!parsed_valid_hugepagesz) { 3260 pr_warn("hugepages = %s preceded by " 3261 "an unsupported hugepagesz, ignoring\n", s); 3262 parsed_valid_hugepagesz = true; 3263 return 1; 3264 } 3265 /* 3266 * !hugetlb_max_hstate means we haven't parsed a hugepagesz= parameter yet, 3267 * so this hugepages= parameter goes to the "default hstate". 3268 */ 3269 else if (!hugetlb_max_hstate) 3270 mhp = &default_hstate_max_huge_pages; 3271 else 3272 mhp = &parsed_hstate->max_huge_pages; 3273 3274 if (mhp == last_mhp) { 3275 pr_warn("hugepages= specified twice without interleaving hugepagesz=, ignoring\n"); 3276 return 1; 3277 } 3278 3279 if (sscanf(s, "%lu", mhp) <= 0) 3280 *mhp = 0; 3281 3282 /* 3283 * Global state is always initialized later in hugetlb_init. 3284 * But we need to allocate >= MAX_ORDER hstates here early to still 3285 * use the bootmem allocator. 3286 */ 3287 if (hugetlb_max_hstate && parsed_hstate->order >= MAX_ORDER) 3288 hugetlb_hstate_alloc_pages(parsed_hstate); 3289 3290 last_mhp = mhp; 3291 3292 return 1; 3293 } 3294 __setup("hugepages=", hugetlb_nrpages_setup); 3295 3296 static int __init hugetlb_default_setup(char *s) 3297 { 3298 default_hstate_size = memparse(s, &s); 3299 return 1; 3300 } 3301 __setup("default_hugepagesz=", hugetlb_default_setup); 3302 3303 static unsigned int cpuset_mems_nr(unsigned int *array) 3304 { 3305 int node; 3306 unsigned int nr = 0; 3307 3308 for_each_node_mask(node, cpuset_current_mems_allowed) 3309 nr += array[node]; 3310 3311 return nr; 3312 } 3313 3314 #ifdef CONFIG_SYSCTL 3315 static int hugetlb_sysctl_handler_common(bool obey_mempolicy, 3316 struct ctl_table *table, int write, 3317 void __user *buffer, size_t *length, loff_t *ppos) 3318 { 3319 struct hstate *h = &default_hstate; 3320 unsigned long tmp = h->max_huge_pages; 3321 int ret; 3322 3323 if (!hugepages_supported()) 3324 return -EOPNOTSUPP; 3325 3326 table->data = &tmp; 3327 table->maxlen = sizeof(unsigned long); 3328 ret = proc_doulongvec_minmax(table, write, buffer, length, ppos); 3329 if (ret) 3330 goto out; 3331 3332 if (write) 3333 ret = __nr_hugepages_store_common(obey_mempolicy, h, 3334 NUMA_NO_NODE, tmp, *length); 3335 out: 3336 return ret; 3337 } 3338 3339 int hugetlb_sysctl_handler(struct ctl_table *table, int write, 3340 void __user *buffer, size_t *length, loff_t *ppos) 3341 { 3342 3343 return hugetlb_sysctl_handler_common(false, table, write, 3344 buffer, length, ppos); 3345 } 3346 3347 #ifdef CONFIG_NUMA 3348 int hugetlb_mempolicy_sysctl_handler(struct ctl_table *table, int write, 3349 void __user *buffer, size_t *length, loff_t *ppos) 3350 { 3351 return hugetlb_sysctl_handler_common(true, table, write, 3352 buffer, length, ppos); 3353 } 3354 #endif /* CONFIG_NUMA */ 3355 3356 int hugetlb_overcommit_handler(struct ctl_table *table, int write, 3357 void __user *buffer, 3358 size_t *length, loff_t *ppos) 3359 { 3360 struct hstate *h = &default_hstate; 3361 unsigned long tmp; 3362 int ret; 3363 3364 if (!hugepages_supported()) 3365 return -EOPNOTSUPP; 3366 3367 tmp = h->nr_overcommit_huge_pages; 3368 3369 if (write && hstate_is_gigantic(h)) 3370 return -EINVAL; 3371 3372 table->data = &tmp; 3373 table->maxlen = sizeof(unsigned long); 3374 ret = proc_doulongvec_minmax(table, write, buffer, length, ppos); 3375 if (ret) 3376 goto out; 3377 3378 if (write) { 3379 spin_lock(&hugetlb_lock); 3380 h->nr_overcommit_huge_pages = tmp; 3381 spin_unlock(&hugetlb_lock); 3382 } 3383 out: 3384 return ret; 3385 } 3386 3387 #endif /* CONFIG_SYSCTL */ 3388 3389 void hugetlb_report_meminfo(struct seq_file *m) 3390 { 3391 struct hstate *h; 3392 unsigned long total = 0; 3393 3394 if (!hugepages_supported()) 3395 return; 3396 3397 for_each_hstate(h) { 3398 unsigned long count = h->nr_huge_pages; 3399 3400 total += (PAGE_SIZE << huge_page_order(h)) * count; 3401 3402 if (h == &default_hstate) 3403 seq_printf(m, 3404 "HugePages_Total: %5lu\n" 3405 "HugePages_Free: %5lu\n" 3406 "HugePages_Rsvd: %5lu\n" 3407 "HugePages_Surp: %5lu\n" 3408 "Hugepagesize: %8lu kB\n", 3409 count, 3410 h->free_huge_pages, 3411 h->resv_huge_pages, 3412 h->surplus_huge_pages, 3413 (PAGE_SIZE << huge_page_order(h)) / 1024); 3414 } 3415 3416 seq_printf(m, "Hugetlb: %8lu kB\n", total / 1024); 3417 } 3418 3419 int hugetlb_report_node_meminfo(int nid, char *buf) 3420 { 3421 struct hstate *h = &default_hstate; 3422 if (!hugepages_supported()) 3423 return 0; 3424 return sprintf(buf, 3425 "Node %d HugePages_Total: %5u\n" 3426 "Node %d HugePages_Free: %5u\n" 3427 "Node %d HugePages_Surp: %5u\n", 3428 nid, h->nr_huge_pages_node[nid], 3429 nid, h->free_huge_pages_node[nid], 3430 nid, h->surplus_huge_pages_node[nid]); 3431 } 3432 3433 void hugetlb_show_meminfo(void) 3434 { 3435 struct hstate *h; 3436 int nid; 3437 3438 if (!hugepages_supported()) 3439 return; 3440 3441 for_each_node_state(nid, N_MEMORY) 3442 for_each_hstate(h) 3443 pr_info("Node %d hugepages_total=%u hugepages_free=%u hugepages_surp=%u hugepages_size=%lukB\n", 3444 nid, 3445 h->nr_huge_pages_node[nid], 3446 h->free_huge_pages_node[nid], 3447 h->surplus_huge_pages_node[nid], 3448 1UL << (huge_page_order(h) + PAGE_SHIFT - 10)); 3449 } 3450 3451 void hugetlb_report_usage(struct seq_file *m, struct mm_struct *mm) 3452 { 3453 seq_printf(m, "HugetlbPages:\t%8lu kB\n", 3454 atomic_long_read(&mm->hugetlb_usage) << (PAGE_SHIFT - 10)); 3455 } 3456 3457 /* Return the number pages of memory we physically have, in PAGE_SIZE units. */ 3458 unsigned long hugetlb_total_pages(void) 3459 { 3460 struct hstate *h; 3461 unsigned long nr_total_pages = 0; 3462 3463 for_each_hstate(h) 3464 nr_total_pages += h->nr_huge_pages * pages_per_huge_page(h); 3465 return nr_total_pages; 3466 } 3467 3468 static int hugetlb_acct_memory(struct hstate *h, long delta) 3469 { 3470 int ret = -ENOMEM; 3471 3472 spin_lock(&hugetlb_lock); 3473 /* 3474 * When cpuset is configured, it breaks the strict hugetlb page 3475 * reservation as the accounting is done on a global variable. Such 3476 * reservation is completely rubbish in the presence of cpuset because 3477 * the reservation is not checked against page availability for the 3478 * current cpuset. Application can still potentially OOM'ed by kernel 3479 * with lack of free htlb page in cpuset that the task is in. 3480 * Attempt to enforce strict accounting with cpuset is almost 3481 * impossible (or too ugly) because cpuset is too fluid that 3482 * task or memory node can be dynamically moved between cpusets. 3483 * 3484 * The change of semantics for shared hugetlb mapping with cpuset is 3485 * undesirable. However, in order to preserve some of the semantics, 3486 * we fall back to check against current free page availability as 3487 * a best attempt and hopefully to minimize the impact of changing 3488 * semantics that cpuset has. 3489 */ 3490 if (delta > 0) { 3491 if (gather_surplus_pages(h, delta) < 0) 3492 goto out; 3493 3494 if (delta > cpuset_mems_nr(h->free_huge_pages_node)) { 3495 return_unused_surplus_pages(h, delta); 3496 goto out; 3497 } 3498 } 3499 3500 ret = 0; 3501 if (delta < 0) 3502 return_unused_surplus_pages(h, (unsigned long) -delta); 3503 3504 out: 3505 spin_unlock(&hugetlb_lock); 3506 return ret; 3507 } 3508 3509 static void hugetlb_vm_op_open(struct vm_area_struct *vma) 3510 { 3511 struct resv_map *resv = vma_resv_map(vma); 3512 3513 /* 3514 * This new VMA should share its siblings reservation map if present. 3515 * The VMA will only ever have a valid reservation map pointer where 3516 * it is being copied for another still existing VMA. As that VMA 3517 * has a reference to the reservation map it cannot disappear until 3518 * after this open call completes. It is therefore safe to take a 3519 * new reference here without additional locking. 3520 */ 3521 if (resv && is_vma_resv_set(vma, HPAGE_RESV_OWNER)) 3522 kref_get(&resv->refs); 3523 } 3524 3525 static void hugetlb_vm_op_close(struct vm_area_struct *vma) 3526 { 3527 struct hstate *h = hstate_vma(vma); 3528 struct resv_map *resv = vma_resv_map(vma); 3529 struct hugepage_subpool *spool = subpool_vma(vma); 3530 unsigned long reserve, start, end; 3531 long gbl_reserve; 3532 3533 if (!resv || !is_vma_resv_set(vma, HPAGE_RESV_OWNER)) 3534 return; 3535 3536 start = vma_hugecache_offset(h, vma, vma->vm_start); 3537 end = vma_hugecache_offset(h, vma, vma->vm_end); 3538 3539 reserve = (end - start) - region_count(resv, start, end); 3540 hugetlb_cgroup_uncharge_counter(resv, start, end); 3541 if (reserve) { 3542 /* 3543 * Decrement reserve counts. The global reserve count may be 3544 * adjusted if the subpool has a minimum size. 3545 */ 3546 gbl_reserve = hugepage_subpool_put_pages(spool, reserve); 3547 hugetlb_acct_memory(h, -gbl_reserve); 3548 } 3549 3550 kref_put(&resv->refs, resv_map_release); 3551 } 3552 3553 static int hugetlb_vm_op_split(struct vm_area_struct *vma, unsigned long addr) 3554 { 3555 if (addr & ~(huge_page_mask(hstate_vma(vma)))) 3556 return -EINVAL; 3557 return 0; 3558 } 3559 3560 static unsigned long hugetlb_vm_op_pagesize(struct vm_area_struct *vma) 3561 { 3562 struct hstate *hstate = hstate_vma(vma); 3563 3564 return 1UL << huge_page_shift(hstate); 3565 } 3566 3567 /* 3568 * We cannot handle pagefaults against hugetlb pages at all. They cause 3569 * handle_mm_fault() to try to instantiate regular-sized pages in the 3570 * hugegpage VMA. do_page_fault() is supposed to trap this, so BUG is we get 3571 * this far. 3572 */ 3573 static vm_fault_t hugetlb_vm_op_fault(struct vm_fault *vmf) 3574 { 3575 BUG(); 3576 return 0; 3577 } 3578 3579 /* 3580 * When a new function is introduced to vm_operations_struct and added 3581 * to hugetlb_vm_ops, please consider adding the function to shm_vm_ops. 3582 * This is because under System V memory model, mappings created via 3583 * shmget/shmat with "huge page" specified are backed by hugetlbfs files, 3584 * their original vm_ops are overwritten with shm_vm_ops. 3585 */ 3586 const struct vm_operations_struct hugetlb_vm_ops = { 3587 .fault = hugetlb_vm_op_fault, 3588 .open = hugetlb_vm_op_open, 3589 .close = hugetlb_vm_op_close, 3590 .split = hugetlb_vm_op_split, 3591 .pagesize = hugetlb_vm_op_pagesize, 3592 }; 3593 3594 static pte_t make_huge_pte(struct vm_area_struct *vma, struct page *page, 3595 int writable) 3596 { 3597 pte_t entry; 3598 3599 if (writable) { 3600 entry = huge_pte_mkwrite(huge_pte_mkdirty(mk_huge_pte(page, 3601 vma->vm_page_prot))); 3602 } else { 3603 entry = huge_pte_wrprotect(mk_huge_pte(page, 3604 vma->vm_page_prot)); 3605 } 3606 entry = pte_mkyoung(entry); 3607 entry = pte_mkhuge(entry); 3608 entry = arch_make_huge_pte(entry, vma, page, writable); 3609 3610 return entry; 3611 } 3612 3613 static void set_huge_ptep_writable(struct vm_area_struct *vma, 3614 unsigned long address, pte_t *ptep) 3615 { 3616 pte_t entry; 3617 3618 entry = huge_pte_mkwrite(huge_pte_mkdirty(huge_ptep_get(ptep))); 3619 if (huge_ptep_set_access_flags(vma, address, ptep, entry, 1)) 3620 update_mmu_cache(vma, address, ptep); 3621 } 3622 3623 bool is_hugetlb_entry_migration(pte_t pte) 3624 { 3625 swp_entry_t swp; 3626 3627 if (huge_pte_none(pte) || pte_present(pte)) 3628 return false; 3629 swp = pte_to_swp_entry(pte); 3630 if (non_swap_entry(swp) && is_migration_entry(swp)) 3631 return true; 3632 else 3633 return false; 3634 } 3635 3636 static int is_hugetlb_entry_hwpoisoned(pte_t pte) 3637 { 3638 swp_entry_t swp; 3639 3640 if (huge_pte_none(pte) || pte_present(pte)) 3641 return 0; 3642 swp = pte_to_swp_entry(pte); 3643 if (non_swap_entry(swp) && is_hwpoison_entry(swp)) 3644 return 1; 3645 else 3646 return 0; 3647 } 3648 3649 int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src, 3650 struct vm_area_struct *vma) 3651 { 3652 pte_t *src_pte, *dst_pte, entry, dst_entry; 3653 struct page *ptepage; 3654 unsigned long addr; 3655 int cow; 3656 struct hstate *h = hstate_vma(vma); 3657 unsigned long sz = huge_page_size(h); 3658 struct address_space *mapping = vma->vm_file->f_mapping; 3659 struct mmu_notifier_range range; 3660 int ret = 0; 3661 3662 cow = (vma->vm_flags & (VM_SHARED | VM_MAYWRITE)) == VM_MAYWRITE; 3663 3664 if (cow) { 3665 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, src, 3666 vma->vm_start, 3667 vma->vm_end); 3668 mmu_notifier_invalidate_range_start(&range); 3669 } else { 3670 /* 3671 * For shared mappings i_mmap_rwsem must be held to call 3672 * huge_pte_alloc, otherwise the returned ptep could go 3673 * away if part of a shared pmd and another thread calls 3674 * huge_pmd_unshare. 3675 */ 3676 i_mmap_lock_read(mapping); 3677 } 3678 3679 for (addr = vma->vm_start; addr < vma->vm_end; addr += sz) { 3680 spinlock_t *src_ptl, *dst_ptl; 3681 src_pte = huge_pte_offset(src, addr, sz); 3682 if (!src_pte) 3683 continue; 3684 dst_pte = huge_pte_alloc(dst, addr, sz); 3685 if (!dst_pte) { 3686 ret = -ENOMEM; 3687 break; 3688 } 3689 3690 /* 3691 * If the pagetables are shared don't copy or take references. 3692 * dst_pte == src_pte is the common case of src/dest sharing. 3693 * 3694 * However, src could have 'unshared' and dst shares with 3695 * another vma. If dst_pte !none, this implies sharing. 3696 * Check here before taking page table lock, and once again 3697 * after taking the lock below. 3698 */ 3699 dst_entry = huge_ptep_get(dst_pte); 3700 if ((dst_pte == src_pte) || !huge_pte_none(dst_entry)) 3701 continue; 3702 3703 dst_ptl = huge_pte_lock(h, dst, dst_pte); 3704 src_ptl = huge_pte_lockptr(h, src, src_pte); 3705 spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING); 3706 entry = huge_ptep_get(src_pte); 3707 dst_entry = huge_ptep_get(dst_pte); 3708 if (huge_pte_none(entry) || !huge_pte_none(dst_entry)) { 3709 /* 3710 * Skip if src entry none. Also, skip in the 3711 * unlikely case dst entry !none as this implies 3712 * sharing with another vma. 3713 */ 3714 ; 3715 } else if (unlikely(is_hugetlb_entry_migration(entry) || 3716 is_hugetlb_entry_hwpoisoned(entry))) { 3717 swp_entry_t swp_entry = pte_to_swp_entry(entry); 3718 3719 if (is_write_migration_entry(swp_entry) && cow) { 3720 /* 3721 * COW mappings require pages in both 3722 * parent and child to be set to read. 3723 */ 3724 make_migration_entry_read(&swp_entry); 3725 entry = swp_entry_to_pte(swp_entry); 3726 set_huge_swap_pte_at(src, addr, src_pte, 3727 entry, sz); 3728 } 3729 set_huge_swap_pte_at(dst, addr, dst_pte, entry, sz); 3730 } else { 3731 if (cow) { 3732 /* 3733 * No need to notify as we are downgrading page 3734 * table protection not changing it to point 3735 * to a new page. 3736 * 3737 * See Documentation/vm/mmu_notifier.rst 3738 */ 3739 huge_ptep_set_wrprotect(src, addr, src_pte); 3740 } 3741 entry = huge_ptep_get(src_pte); 3742 ptepage = pte_page(entry); 3743 get_page(ptepage); 3744 page_dup_rmap(ptepage, true); 3745 set_huge_pte_at(dst, addr, dst_pte, entry); 3746 hugetlb_count_add(pages_per_huge_page(h), dst); 3747 } 3748 spin_unlock(src_ptl); 3749 spin_unlock(dst_ptl); 3750 } 3751 3752 if (cow) 3753 mmu_notifier_invalidate_range_end(&range); 3754 else 3755 i_mmap_unlock_read(mapping); 3756 3757 return ret; 3758 } 3759 3760 void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct *vma, 3761 unsigned long start, unsigned long end, 3762 struct page *ref_page) 3763 { 3764 struct mm_struct *mm = vma->vm_mm; 3765 unsigned long address; 3766 pte_t *ptep; 3767 pte_t pte; 3768 spinlock_t *ptl; 3769 struct page *page; 3770 struct hstate *h = hstate_vma(vma); 3771 unsigned long sz = huge_page_size(h); 3772 struct mmu_notifier_range range; 3773 3774 WARN_ON(!is_vm_hugetlb_page(vma)); 3775 BUG_ON(start & ~huge_page_mask(h)); 3776 BUG_ON(end & ~huge_page_mask(h)); 3777 3778 /* 3779 * This is a hugetlb vma, all the pte entries should point 3780 * to huge page. 3781 */ 3782 tlb_change_page_size(tlb, sz); 3783 tlb_start_vma(tlb, vma); 3784 3785 /* 3786 * If sharing possible, alert mmu notifiers of worst case. 3787 */ 3788 mmu_notifier_range_init(&range, MMU_NOTIFY_UNMAP, 0, vma, mm, start, 3789 end); 3790 adjust_range_if_pmd_sharing_possible(vma, &range.start, &range.end); 3791 mmu_notifier_invalidate_range_start(&range); 3792 address = start; 3793 for (; address < end; address += sz) { 3794 ptep = huge_pte_offset(mm, address, sz); 3795 if (!ptep) 3796 continue; 3797 3798 ptl = huge_pte_lock(h, mm, ptep); 3799 if (huge_pmd_unshare(mm, &address, ptep)) { 3800 spin_unlock(ptl); 3801 /* 3802 * We just unmapped a page of PMDs by clearing a PUD. 3803 * The caller's TLB flush range should cover this area. 3804 */ 3805 continue; 3806 } 3807 3808 pte = huge_ptep_get(ptep); 3809 if (huge_pte_none(pte)) { 3810 spin_unlock(ptl); 3811 continue; 3812 } 3813 3814 /* 3815 * Migrating hugepage or HWPoisoned hugepage is already 3816 * unmapped and its refcount is dropped, so just clear pte here. 3817 */ 3818 if (unlikely(!pte_present(pte))) { 3819 huge_pte_clear(mm, address, ptep, sz); 3820 spin_unlock(ptl); 3821 continue; 3822 } 3823 3824 page = pte_page(pte); 3825 /* 3826 * If a reference page is supplied, it is because a specific 3827 * page is being unmapped, not a range. Ensure the page we 3828 * are about to unmap is the actual page of interest. 3829 */ 3830 if (ref_page) { 3831 if (page != ref_page) { 3832 spin_unlock(ptl); 3833 continue; 3834 } 3835 /* 3836 * Mark the VMA as having unmapped its page so that 3837 * future faults in this VMA will fail rather than 3838 * looking like data was lost 3839 */ 3840 set_vma_resv_flags(vma, HPAGE_RESV_UNMAPPED); 3841 } 3842 3843 pte = huge_ptep_get_and_clear(mm, address, ptep); 3844 tlb_remove_huge_tlb_entry(h, tlb, ptep, address); 3845 if (huge_pte_dirty(pte)) 3846 set_page_dirty(page); 3847 3848 hugetlb_count_sub(pages_per_huge_page(h), mm); 3849 page_remove_rmap(page, true); 3850 3851 spin_unlock(ptl); 3852 tlb_remove_page_size(tlb, page, huge_page_size(h)); 3853 /* 3854 * Bail out after unmapping reference page if supplied 3855 */ 3856 if (ref_page) 3857 break; 3858 } 3859 mmu_notifier_invalidate_range_end(&range); 3860 tlb_end_vma(tlb, vma); 3861 } 3862 3863 void __unmap_hugepage_range_final(struct mmu_gather *tlb, 3864 struct vm_area_struct *vma, unsigned long start, 3865 unsigned long end, struct page *ref_page) 3866 { 3867 __unmap_hugepage_range(tlb, vma, start, end, ref_page); 3868 3869 /* 3870 * Clear this flag so that x86's huge_pmd_share page_table_shareable 3871 * test will fail on a vma being torn down, and not grab a page table 3872 * on its way out. We're lucky that the flag has such an appropriate 3873 * name, and can in fact be safely cleared here. We could clear it 3874 * before the __unmap_hugepage_range above, but all that's necessary 3875 * is to clear it before releasing the i_mmap_rwsem. This works 3876 * because in the context this is called, the VMA is about to be 3877 * destroyed and the i_mmap_rwsem is held. 3878 */ 3879 vma->vm_flags &= ~VM_MAYSHARE; 3880 } 3881 3882 void unmap_hugepage_range(struct vm_area_struct *vma, unsigned long start, 3883 unsigned long end, struct page *ref_page) 3884 { 3885 struct mm_struct *mm; 3886 struct mmu_gather tlb; 3887 unsigned long tlb_start = start; 3888 unsigned long tlb_end = end; 3889 3890 /* 3891 * If shared PMDs were possibly used within this vma range, adjust 3892 * start/end for worst case tlb flushing. 3893 * Note that we can not be sure if PMDs are shared until we try to 3894 * unmap pages. However, we want to make sure TLB flushing covers 3895 * the largest possible range. 3896 */ 3897 adjust_range_if_pmd_sharing_possible(vma, &tlb_start, &tlb_end); 3898 3899 mm = vma->vm_mm; 3900 3901 tlb_gather_mmu(&tlb, mm, tlb_start, tlb_end); 3902 __unmap_hugepage_range(&tlb, vma, start, end, ref_page); 3903 tlb_finish_mmu(&tlb, tlb_start, tlb_end); 3904 } 3905 3906 /* 3907 * This is called when the original mapper is failing to COW a MAP_PRIVATE 3908 * mappping it owns the reserve page for. The intention is to unmap the page 3909 * from other VMAs and let the children be SIGKILLed if they are faulting the 3910 * same region. 3911 */ 3912 static void unmap_ref_private(struct mm_struct *mm, struct vm_area_struct *vma, 3913 struct page *page, unsigned long address) 3914 { 3915 struct hstate *h = hstate_vma(vma); 3916 struct vm_area_struct *iter_vma; 3917 struct address_space *mapping; 3918 pgoff_t pgoff; 3919 3920 /* 3921 * vm_pgoff is in PAGE_SIZE units, hence the different calculation 3922 * from page cache lookup which is in HPAGE_SIZE units. 3923 */ 3924 address = address & huge_page_mask(h); 3925 pgoff = ((address - vma->vm_start) >> PAGE_SHIFT) + 3926 vma->vm_pgoff; 3927 mapping = vma->vm_file->f_mapping; 3928 3929 /* 3930 * Take the mapping lock for the duration of the table walk. As 3931 * this mapping should be shared between all the VMAs, 3932 * __unmap_hugepage_range() is called as the lock is already held 3933 */ 3934 i_mmap_lock_write(mapping); 3935 vma_interval_tree_foreach(iter_vma, &mapping->i_mmap, pgoff, pgoff) { 3936 /* Do not unmap the current VMA */ 3937 if (iter_vma == vma) 3938 continue; 3939 3940 /* 3941 * Shared VMAs have their own reserves and do not affect 3942 * MAP_PRIVATE accounting but it is possible that a shared 3943 * VMA is using the same page so check and skip such VMAs. 3944 */ 3945 if (iter_vma->vm_flags & VM_MAYSHARE) 3946 continue; 3947 3948 /* 3949 * Unmap the page from other VMAs without their own reserves. 3950 * They get marked to be SIGKILLed if they fault in these 3951 * areas. This is because a future no-page fault on this VMA 3952 * could insert a zeroed page instead of the data existing 3953 * from the time of fork. This would look like data corruption 3954 */ 3955 if (!is_vma_resv_set(iter_vma, HPAGE_RESV_OWNER)) 3956 unmap_hugepage_range(iter_vma, address, 3957 address + huge_page_size(h), page); 3958 } 3959 i_mmap_unlock_write(mapping); 3960 } 3961 3962 /* 3963 * Hugetlb_cow() should be called with page lock of the original hugepage held. 3964 * Called with hugetlb_instantiation_mutex held and pte_page locked so we 3965 * cannot race with other handlers or page migration. 3966 * Keep the pte_same checks anyway to make transition from the mutex easier. 3967 */ 3968 static vm_fault_t hugetlb_cow(struct mm_struct *mm, struct vm_area_struct *vma, 3969 unsigned long address, pte_t *ptep, 3970 struct page *pagecache_page, spinlock_t *ptl) 3971 { 3972 pte_t pte; 3973 struct hstate *h = hstate_vma(vma); 3974 struct page *old_page, *new_page; 3975 int outside_reserve = 0; 3976 vm_fault_t ret = 0; 3977 unsigned long haddr = address & huge_page_mask(h); 3978 struct mmu_notifier_range range; 3979 3980 pte = huge_ptep_get(ptep); 3981 old_page = pte_page(pte); 3982 3983 retry_avoidcopy: 3984 /* If no-one else is actually using this page, avoid the copy 3985 * and just make the page writable */ 3986 if (page_mapcount(old_page) == 1 && PageAnon(old_page)) { 3987 page_move_anon_rmap(old_page, vma); 3988 set_huge_ptep_writable(vma, haddr, ptep); 3989 return 0; 3990 } 3991 3992 /* 3993 * If the process that created a MAP_PRIVATE mapping is about to 3994 * perform a COW due to a shared page count, attempt to satisfy 3995 * the allocation without using the existing reserves. The pagecache 3996 * page is used to determine if the reserve at this address was 3997 * consumed or not. If reserves were used, a partial faulted mapping 3998 * at the time of fork() could consume its reserves on COW instead 3999 * of the full address range. 4000 */ 4001 if (is_vma_resv_set(vma, HPAGE_RESV_OWNER) && 4002 old_page != pagecache_page) 4003 outside_reserve = 1; 4004 4005 get_page(old_page); 4006 4007 /* 4008 * Drop page table lock as buddy allocator may be called. It will 4009 * be acquired again before returning to the caller, as expected. 4010 */ 4011 spin_unlock(ptl); 4012 new_page = alloc_huge_page(vma, haddr, outside_reserve); 4013 4014 if (IS_ERR(new_page)) { 4015 /* 4016 * If a process owning a MAP_PRIVATE mapping fails to COW, 4017 * it is due to references held by a child and an insufficient 4018 * huge page pool. To guarantee the original mappers 4019 * reliability, unmap the page from child processes. The child 4020 * may get SIGKILLed if it later faults. 4021 */ 4022 if (outside_reserve) { 4023 put_page(old_page); 4024 BUG_ON(huge_pte_none(pte)); 4025 unmap_ref_private(mm, vma, old_page, haddr); 4026 BUG_ON(huge_pte_none(pte)); 4027 spin_lock(ptl); 4028 ptep = huge_pte_offset(mm, haddr, huge_page_size(h)); 4029 if (likely(ptep && 4030 pte_same(huge_ptep_get(ptep), pte))) 4031 goto retry_avoidcopy; 4032 /* 4033 * race occurs while re-acquiring page table 4034 * lock, and our job is done. 4035 */ 4036 return 0; 4037 } 4038 4039 ret = vmf_error(PTR_ERR(new_page)); 4040 goto out_release_old; 4041 } 4042 4043 /* 4044 * When the original hugepage is shared one, it does not have 4045 * anon_vma prepared. 4046 */ 4047 if (unlikely(anon_vma_prepare(vma))) { 4048 ret = VM_FAULT_OOM; 4049 goto out_release_all; 4050 } 4051 4052 copy_user_huge_page(new_page, old_page, address, vma, 4053 pages_per_huge_page(h)); 4054 __SetPageUptodate(new_page); 4055 4056 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, mm, haddr, 4057 haddr + huge_page_size(h)); 4058 mmu_notifier_invalidate_range_start(&range); 4059 4060 /* 4061 * Retake the page table lock to check for racing updates 4062 * before the page tables are altered 4063 */ 4064 spin_lock(ptl); 4065 ptep = huge_pte_offset(mm, haddr, huge_page_size(h)); 4066 if (likely(ptep && pte_same(huge_ptep_get(ptep), pte))) { 4067 ClearPagePrivate(new_page); 4068 4069 /* Break COW */ 4070 huge_ptep_clear_flush(vma, haddr, ptep); 4071 mmu_notifier_invalidate_range(mm, range.start, range.end); 4072 set_huge_pte_at(mm, haddr, ptep, 4073 make_huge_pte(vma, new_page, 1)); 4074 page_remove_rmap(old_page, true); 4075 hugepage_add_new_anon_rmap(new_page, vma, haddr); 4076 set_page_huge_active(new_page); 4077 /* Make the old page be freed below */ 4078 new_page = old_page; 4079 } 4080 spin_unlock(ptl); 4081 mmu_notifier_invalidate_range_end(&range); 4082 out_release_all: 4083 restore_reserve_on_error(h, vma, haddr, new_page); 4084 put_page(new_page); 4085 out_release_old: 4086 put_page(old_page); 4087 4088 spin_lock(ptl); /* Caller expects lock to be held */ 4089 return ret; 4090 } 4091 4092 /* Return the pagecache page at a given address within a VMA */ 4093 static struct page *hugetlbfs_pagecache_page(struct hstate *h, 4094 struct vm_area_struct *vma, unsigned long address) 4095 { 4096 struct address_space *mapping; 4097 pgoff_t idx; 4098 4099 mapping = vma->vm_file->f_mapping; 4100 idx = vma_hugecache_offset(h, vma, address); 4101 4102 return find_lock_page(mapping, idx); 4103 } 4104 4105 /* 4106 * Return whether there is a pagecache page to back given address within VMA. 4107 * Caller follow_hugetlb_page() holds page_table_lock so we cannot lock_page. 4108 */ 4109 static bool hugetlbfs_pagecache_present(struct hstate *h, 4110 struct vm_area_struct *vma, unsigned long address) 4111 { 4112 struct address_space *mapping; 4113 pgoff_t idx; 4114 struct page *page; 4115 4116 mapping = vma->vm_file->f_mapping; 4117 idx = vma_hugecache_offset(h, vma, address); 4118 4119 page = find_get_page(mapping, idx); 4120 if (page) 4121 put_page(page); 4122 return page != NULL; 4123 } 4124 4125 int huge_add_to_page_cache(struct page *page, struct address_space *mapping, 4126 pgoff_t idx) 4127 { 4128 struct inode *inode = mapping->host; 4129 struct hstate *h = hstate_inode(inode); 4130 int err = add_to_page_cache(page, mapping, idx, GFP_KERNEL); 4131 4132 if (err) 4133 return err; 4134 ClearPagePrivate(page); 4135 4136 /* 4137 * set page dirty so that it will not be removed from cache/file 4138 * by non-hugetlbfs specific code paths. 4139 */ 4140 set_page_dirty(page); 4141 4142 spin_lock(&inode->i_lock); 4143 inode->i_blocks += blocks_per_huge_page(h); 4144 spin_unlock(&inode->i_lock); 4145 return 0; 4146 } 4147 4148 static vm_fault_t hugetlb_no_page(struct mm_struct *mm, 4149 struct vm_area_struct *vma, 4150 struct address_space *mapping, pgoff_t idx, 4151 unsigned long address, pte_t *ptep, unsigned int flags) 4152 { 4153 struct hstate *h = hstate_vma(vma); 4154 vm_fault_t ret = VM_FAULT_SIGBUS; 4155 int anon_rmap = 0; 4156 unsigned long size; 4157 struct page *page; 4158 pte_t new_pte; 4159 spinlock_t *ptl; 4160 unsigned long haddr = address & huge_page_mask(h); 4161 bool new_page = false; 4162 4163 /* 4164 * Currently, we are forced to kill the process in the event the 4165 * original mapper has unmapped pages from the child due to a failed 4166 * COW. Warn that such a situation has occurred as it may not be obvious 4167 */ 4168 if (is_vma_resv_set(vma, HPAGE_RESV_UNMAPPED)) { 4169 pr_warn_ratelimited("PID %d killed due to inadequate hugepage pool\n", 4170 current->pid); 4171 return ret; 4172 } 4173 4174 /* 4175 * We can not race with truncation due to holding i_mmap_rwsem. 4176 * i_size is modified when holding i_mmap_rwsem, so check here 4177 * once for faults beyond end of file. 4178 */ 4179 size = i_size_read(mapping->host) >> huge_page_shift(h); 4180 if (idx >= size) 4181 goto out; 4182 4183 retry: 4184 page = find_lock_page(mapping, idx); 4185 if (!page) { 4186 /* 4187 * Check for page in userfault range 4188 */ 4189 if (userfaultfd_missing(vma)) { 4190 u32 hash; 4191 struct vm_fault vmf = { 4192 .vma = vma, 4193 .address = haddr, 4194 .flags = flags, 4195 /* 4196 * Hard to debug if it ends up being 4197 * used by a callee that assumes 4198 * something about the other 4199 * uninitialized fields... same as in 4200 * memory.c 4201 */ 4202 }; 4203 4204 /* 4205 * hugetlb_fault_mutex and i_mmap_rwsem must be 4206 * dropped before handling userfault. Reacquire 4207 * after handling fault to make calling code simpler. 4208 */ 4209 hash = hugetlb_fault_mutex_hash(mapping, idx); 4210 mutex_unlock(&hugetlb_fault_mutex_table[hash]); 4211 i_mmap_unlock_read(mapping); 4212 ret = handle_userfault(&vmf, VM_UFFD_MISSING); 4213 i_mmap_lock_read(mapping); 4214 mutex_lock(&hugetlb_fault_mutex_table[hash]); 4215 goto out; 4216 } 4217 4218 page = alloc_huge_page(vma, haddr, 0); 4219 if (IS_ERR(page)) { 4220 /* 4221 * Returning error will result in faulting task being 4222 * sent SIGBUS. The hugetlb fault mutex prevents two 4223 * tasks from racing to fault in the same page which 4224 * could result in false unable to allocate errors. 4225 * Page migration does not take the fault mutex, but 4226 * does a clear then write of pte's under page table 4227 * lock. Page fault code could race with migration, 4228 * notice the clear pte and try to allocate a page 4229 * here. Before returning error, get ptl and make 4230 * sure there really is no pte entry. 4231 */ 4232 ptl = huge_pte_lock(h, mm, ptep); 4233 if (!huge_pte_none(huge_ptep_get(ptep))) { 4234 ret = 0; 4235 spin_unlock(ptl); 4236 goto out; 4237 } 4238 spin_unlock(ptl); 4239 ret = vmf_error(PTR_ERR(page)); 4240 goto out; 4241 } 4242 clear_huge_page(page, address, pages_per_huge_page(h)); 4243 __SetPageUptodate(page); 4244 new_page = true; 4245 4246 if (vma->vm_flags & VM_MAYSHARE) { 4247 int err = huge_add_to_page_cache(page, mapping, idx); 4248 if (err) { 4249 put_page(page); 4250 if (err == -EEXIST) 4251 goto retry; 4252 goto out; 4253 } 4254 } else { 4255 lock_page(page); 4256 if (unlikely(anon_vma_prepare(vma))) { 4257 ret = VM_FAULT_OOM; 4258 goto backout_unlocked; 4259 } 4260 anon_rmap = 1; 4261 } 4262 } else { 4263 /* 4264 * If memory error occurs between mmap() and fault, some process 4265 * don't have hwpoisoned swap entry for errored virtual address. 4266 * So we need to block hugepage fault by PG_hwpoison bit check. 4267 */ 4268 if (unlikely(PageHWPoison(page))) { 4269 ret = VM_FAULT_HWPOISON | 4270 VM_FAULT_SET_HINDEX(hstate_index(h)); 4271 goto backout_unlocked; 4272 } 4273 } 4274 4275 /* 4276 * If we are going to COW a private mapping later, we examine the 4277 * pending reservations for this page now. This will ensure that 4278 * any allocations necessary to record that reservation occur outside 4279 * the spinlock. 4280 */ 4281 if ((flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED)) { 4282 if (vma_needs_reservation(h, vma, haddr) < 0) { 4283 ret = VM_FAULT_OOM; 4284 goto backout_unlocked; 4285 } 4286 /* Just decrements count, does not deallocate */ 4287 vma_end_reservation(h, vma, haddr); 4288 } 4289 4290 ptl = huge_pte_lock(h, mm, ptep); 4291 ret = 0; 4292 if (!huge_pte_none(huge_ptep_get(ptep))) 4293 goto backout; 4294 4295 if (anon_rmap) { 4296 ClearPagePrivate(page); 4297 hugepage_add_new_anon_rmap(page, vma, haddr); 4298 } else 4299 page_dup_rmap(page, true); 4300 new_pte = make_huge_pte(vma, page, ((vma->vm_flags & VM_WRITE) 4301 && (vma->vm_flags & VM_SHARED))); 4302 set_huge_pte_at(mm, haddr, ptep, new_pte); 4303 4304 hugetlb_count_add(pages_per_huge_page(h), mm); 4305 if ((flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED)) { 4306 /* Optimization, do the COW without a second fault */ 4307 ret = hugetlb_cow(mm, vma, address, ptep, page, ptl); 4308 } 4309 4310 spin_unlock(ptl); 4311 4312 /* 4313 * Only make newly allocated pages active. Existing pages found 4314 * in the pagecache could be !page_huge_active() if they have been 4315 * isolated for migration. 4316 */ 4317 if (new_page) 4318 set_page_huge_active(page); 4319 4320 unlock_page(page); 4321 out: 4322 return ret; 4323 4324 backout: 4325 spin_unlock(ptl); 4326 backout_unlocked: 4327 unlock_page(page); 4328 restore_reserve_on_error(h, vma, haddr, page); 4329 put_page(page); 4330 goto out; 4331 } 4332 4333 #ifdef CONFIG_SMP 4334 u32 hugetlb_fault_mutex_hash(struct address_space *mapping, pgoff_t idx) 4335 { 4336 unsigned long key[2]; 4337 u32 hash; 4338 4339 key[0] = (unsigned long) mapping; 4340 key[1] = idx; 4341 4342 hash = jhash2((u32 *)&key, sizeof(key)/(sizeof(u32)), 0); 4343 4344 return hash & (num_fault_mutexes - 1); 4345 } 4346 #else 4347 /* 4348 * For uniprocesor systems we always use a single mutex, so just 4349 * return 0 and avoid the hashing overhead. 4350 */ 4351 u32 hugetlb_fault_mutex_hash(struct address_space *mapping, pgoff_t idx) 4352 { 4353 return 0; 4354 } 4355 #endif 4356 4357 vm_fault_t hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma, 4358 unsigned long address, unsigned int flags) 4359 { 4360 pte_t *ptep, entry; 4361 spinlock_t *ptl; 4362 vm_fault_t ret; 4363 u32 hash; 4364 pgoff_t idx; 4365 struct page *page = NULL; 4366 struct page *pagecache_page = NULL; 4367 struct hstate *h = hstate_vma(vma); 4368 struct address_space *mapping; 4369 int need_wait_lock = 0; 4370 unsigned long haddr = address & huge_page_mask(h); 4371 4372 ptep = huge_pte_offset(mm, haddr, huge_page_size(h)); 4373 if (ptep) { 4374 /* 4375 * Since we hold no locks, ptep could be stale. That is 4376 * OK as we are only making decisions based on content and 4377 * not actually modifying content here. 4378 */ 4379 entry = huge_ptep_get(ptep); 4380 if (unlikely(is_hugetlb_entry_migration(entry))) { 4381 migration_entry_wait_huge(vma, mm, ptep); 4382 return 0; 4383 } else if (unlikely(is_hugetlb_entry_hwpoisoned(entry))) 4384 return VM_FAULT_HWPOISON_LARGE | 4385 VM_FAULT_SET_HINDEX(hstate_index(h)); 4386 } else { 4387 ptep = huge_pte_alloc(mm, haddr, huge_page_size(h)); 4388 if (!ptep) 4389 return VM_FAULT_OOM; 4390 } 4391 4392 /* 4393 * Acquire i_mmap_rwsem before calling huge_pte_alloc and hold 4394 * until finished with ptep. This serves two purposes: 4395 * 1) It prevents huge_pmd_unshare from being called elsewhere 4396 * and making the ptep no longer valid. 4397 * 2) It synchronizes us with i_size modifications during truncation. 4398 * 4399 * ptep could have already be assigned via huge_pte_offset. That 4400 * is OK, as huge_pte_alloc will return the same value unless 4401 * something has changed. 4402 */ 4403 mapping = vma->vm_file->f_mapping; 4404 i_mmap_lock_read(mapping); 4405 ptep = huge_pte_alloc(mm, haddr, huge_page_size(h)); 4406 if (!ptep) { 4407 i_mmap_unlock_read(mapping); 4408 return VM_FAULT_OOM; 4409 } 4410 4411 /* 4412 * Serialize hugepage allocation and instantiation, so that we don't 4413 * get spurious allocation failures if two CPUs race to instantiate 4414 * the same page in the page cache. 4415 */ 4416 idx = vma_hugecache_offset(h, vma, haddr); 4417 hash = hugetlb_fault_mutex_hash(mapping, idx); 4418 mutex_lock(&hugetlb_fault_mutex_table[hash]); 4419 4420 entry = huge_ptep_get(ptep); 4421 if (huge_pte_none(entry)) { 4422 ret = hugetlb_no_page(mm, vma, mapping, idx, address, ptep, flags); 4423 goto out_mutex; 4424 } 4425 4426 ret = 0; 4427 4428 /* 4429 * entry could be a migration/hwpoison entry at this point, so this 4430 * check prevents the kernel from going below assuming that we have 4431 * a active hugepage in pagecache. This goto expects the 2nd page fault, 4432 * and is_hugetlb_entry_(migration|hwpoisoned) check will properly 4433 * handle it. 4434 */ 4435 if (!pte_present(entry)) 4436 goto out_mutex; 4437 4438 /* 4439 * If we are going to COW the mapping later, we examine the pending 4440 * reservations for this page now. This will ensure that any 4441 * allocations necessary to record that reservation occur outside the 4442 * spinlock. For private mappings, we also lookup the pagecache 4443 * page now as it is used to determine if a reservation has been 4444 * consumed. 4445 */ 4446 if ((flags & FAULT_FLAG_WRITE) && !huge_pte_write(entry)) { 4447 if (vma_needs_reservation(h, vma, haddr) < 0) { 4448 ret = VM_FAULT_OOM; 4449 goto out_mutex; 4450 } 4451 /* Just decrements count, does not deallocate */ 4452 vma_end_reservation(h, vma, haddr); 4453 4454 if (!(vma->vm_flags & VM_MAYSHARE)) 4455 pagecache_page = hugetlbfs_pagecache_page(h, 4456 vma, haddr); 4457 } 4458 4459 ptl = huge_pte_lock(h, mm, ptep); 4460 4461 /* Check for a racing update before calling hugetlb_cow */ 4462 if (unlikely(!pte_same(entry, huge_ptep_get(ptep)))) 4463 goto out_ptl; 4464 4465 /* 4466 * hugetlb_cow() requires page locks of pte_page(entry) and 4467 * pagecache_page, so here we need take the former one 4468 * when page != pagecache_page or !pagecache_page. 4469 */ 4470 page = pte_page(entry); 4471 if (page != pagecache_page) 4472 if (!trylock_page(page)) { 4473 need_wait_lock = 1; 4474 goto out_ptl; 4475 } 4476 4477 get_page(page); 4478 4479 if (flags & FAULT_FLAG_WRITE) { 4480 if (!huge_pte_write(entry)) { 4481 ret = hugetlb_cow(mm, vma, address, ptep, 4482 pagecache_page, ptl); 4483 goto out_put_page; 4484 } 4485 entry = huge_pte_mkdirty(entry); 4486 } 4487 entry = pte_mkyoung(entry); 4488 if (huge_ptep_set_access_flags(vma, haddr, ptep, entry, 4489 flags & FAULT_FLAG_WRITE)) 4490 update_mmu_cache(vma, haddr, ptep); 4491 out_put_page: 4492 if (page != pagecache_page) 4493 unlock_page(page); 4494 put_page(page); 4495 out_ptl: 4496 spin_unlock(ptl); 4497 4498 if (pagecache_page) { 4499 unlock_page(pagecache_page); 4500 put_page(pagecache_page); 4501 } 4502 out_mutex: 4503 mutex_unlock(&hugetlb_fault_mutex_table[hash]); 4504 i_mmap_unlock_read(mapping); 4505 /* 4506 * Generally it's safe to hold refcount during waiting page lock. But 4507 * here we just wait to defer the next page fault to avoid busy loop and 4508 * the page is not used after unlocked before returning from the current 4509 * page fault. So we are safe from accessing freed page, even if we wait 4510 * here without taking refcount. 4511 */ 4512 if (need_wait_lock) 4513 wait_on_page_locked(page); 4514 return ret; 4515 } 4516 4517 /* 4518 * Used by userfaultfd UFFDIO_COPY. Based on mcopy_atomic_pte with 4519 * modifications for huge pages. 4520 */ 4521 int hugetlb_mcopy_atomic_pte(struct mm_struct *dst_mm, 4522 pte_t *dst_pte, 4523 struct vm_area_struct *dst_vma, 4524 unsigned long dst_addr, 4525 unsigned long src_addr, 4526 struct page **pagep) 4527 { 4528 struct address_space *mapping; 4529 pgoff_t idx; 4530 unsigned long size; 4531 int vm_shared = dst_vma->vm_flags & VM_SHARED; 4532 struct hstate *h = hstate_vma(dst_vma); 4533 pte_t _dst_pte; 4534 spinlock_t *ptl; 4535 int ret; 4536 struct page *page; 4537 4538 if (!*pagep) { 4539 ret = -ENOMEM; 4540 page = alloc_huge_page(dst_vma, dst_addr, 0); 4541 if (IS_ERR(page)) 4542 goto out; 4543 4544 ret = copy_huge_page_from_user(page, 4545 (const void __user *) src_addr, 4546 pages_per_huge_page(h), false); 4547 4548 /* fallback to copy_from_user outside mmap_sem */ 4549 if (unlikely(ret)) { 4550 ret = -ENOENT; 4551 *pagep = page; 4552 /* don't free the page */ 4553 goto out; 4554 } 4555 } else { 4556 page = *pagep; 4557 *pagep = NULL; 4558 } 4559 4560 /* 4561 * The memory barrier inside __SetPageUptodate makes sure that 4562 * preceding stores to the page contents become visible before 4563 * the set_pte_at() write. 4564 */ 4565 __SetPageUptodate(page); 4566 4567 mapping = dst_vma->vm_file->f_mapping; 4568 idx = vma_hugecache_offset(h, dst_vma, dst_addr); 4569 4570 /* 4571 * If shared, add to page cache 4572 */ 4573 if (vm_shared) { 4574 size = i_size_read(mapping->host) >> huge_page_shift(h); 4575 ret = -EFAULT; 4576 if (idx >= size) 4577 goto out_release_nounlock; 4578 4579 /* 4580 * Serialization between remove_inode_hugepages() and 4581 * huge_add_to_page_cache() below happens through the 4582 * hugetlb_fault_mutex_table that here must be hold by 4583 * the caller. 4584 */ 4585 ret = huge_add_to_page_cache(page, mapping, idx); 4586 if (ret) 4587 goto out_release_nounlock; 4588 } 4589 4590 ptl = huge_pte_lockptr(h, dst_mm, dst_pte); 4591 spin_lock(ptl); 4592 4593 /* 4594 * Recheck the i_size after holding PT lock to make sure not 4595 * to leave any page mapped (as page_mapped()) beyond the end 4596 * of the i_size (remove_inode_hugepages() is strict about 4597 * enforcing that). If we bail out here, we'll also leave a 4598 * page in the radix tree in the vm_shared case beyond the end 4599 * of the i_size, but remove_inode_hugepages() will take care 4600 * of it as soon as we drop the hugetlb_fault_mutex_table. 4601 */ 4602 size = i_size_read(mapping->host) >> huge_page_shift(h); 4603 ret = -EFAULT; 4604 if (idx >= size) 4605 goto out_release_unlock; 4606 4607 ret = -EEXIST; 4608 if (!huge_pte_none(huge_ptep_get(dst_pte))) 4609 goto out_release_unlock; 4610 4611 if (vm_shared) { 4612 page_dup_rmap(page, true); 4613 } else { 4614 ClearPagePrivate(page); 4615 hugepage_add_new_anon_rmap(page, dst_vma, dst_addr); 4616 } 4617 4618 _dst_pte = make_huge_pte(dst_vma, page, dst_vma->vm_flags & VM_WRITE); 4619 if (dst_vma->vm_flags & VM_WRITE) 4620 _dst_pte = huge_pte_mkdirty(_dst_pte); 4621 _dst_pte = pte_mkyoung(_dst_pte); 4622 4623 set_huge_pte_at(dst_mm, dst_addr, dst_pte, _dst_pte); 4624 4625 (void)huge_ptep_set_access_flags(dst_vma, dst_addr, dst_pte, _dst_pte, 4626 dst_vma->vm_flags & VM_WRITE); 4627 hugetlb_count_add(pages_per_huge_page(h), dst_mm); 4628 4629 /* No need to invalidate - it was non-present before */ 4630 update_mmu_cache(dst_vma, dst_addr, dst_pte); 4631 4632 spin_unlock(ptl); 4633 set_page_huge_active(page); 4634 if (vm_shared) 4635 unlock_page(page); 4636 ret = 0; 4637 out: 4638 return ret; 4639 out_release_unlock: 4640 spin_unlock(ptl); 4641 if (vm_shared) 4642 unlock_page(page); 4643 out_release_nounlock: 4644 put_page(page); 4645 goto out; 4646 } 4647 4648 long follow_hugetlb_page(struct mm_struct *mm, struct vm_area_struct *vma, 4649 struct page **pages, struct vm_area_struct **vmas, 4650 unsigned long *position, unsigned long *nr_pages, 4651 long i, unsigned int flags, int *locked) 4652 { 4653 unsigned long pfn_offset; 4654 unsigned long vaddr = *position; 4655 unsigned long remainder = *nr_pages; 4656 struct hstate *h = hstate_vma(vma); 4657 int err = -EFAULT; 4658 4659 while (vaddr < vma->vm_end && remainder) { 4660 pte_t *pte; 4661 spinlock_t *ptl = NULL; 4662 int absent; 4663 struct page *page; 4664 4665 /* 4666 * If we have a pending SIGKILL, don't keep faulting pages and 4667 * potentially allocating memory. 4668 */ 4669 if (fatal_signal_pending(current)) { 4670 remainder = 0; 4671 break; 4672 } 4673 4674 /* 4675 * Some archs (sparc64, sh*) have multiple pte_ts to 4676 * each hugepage. We have to make sure we get the 4677 * first, for the page indexing below to work. 4678 * 4679 * Note that page table lock is not held when pte is null. 4680 */ 4681 pte = huge_pte_offset(mm, vaddr & huge_page_mask(h), 4682 huge_page_size(h)); 4683 if (pte) 4684 ptl = huge_pte_lock(h, mm, pte); 4685 absent = !pte || huge_pte_none(huge_ptep_get(pte)); 4686 4687 /* 4688 * When coredumping, it suits get_dump_page if we just return 4689 * an error where there's an empty slot with no huge pagecache 4690 * to back it. This way, we avoid allocating a hugepage, and 4691 * the sparse dumpfile avoids allocating disk blocks, but its 4692 * huge holes still show up with zeroes where they need to be. 4693 */ 4694 if (absent && (flags & FOLL_DUMP) && 4695 !hugetlbfs_pagecache_present(h, vma, vaddr)) { 4696 if (pte) 4697 spin_unlock(ptl); 4698 remainder = 0; 4699 break; 4700 } 4701 4702 /* 4703 * We need call hugetlb_fault for both hugepages under migration 4704 * (in which case hugetlb_fault waits for the migration,) and 4705 * hwpoisoned hugepages (in which case we need to prevent the 4706 * caller from accessing to them.) In order to do this, we use 4707 * here is_swap_pte instead of is_hugetlb_entry_migration and 4708 * is_hugetlb_entry_hwpoisoned. This is because it simply covers 4709 * both cases, and because we can't follow correct pages 4710 * directly from any kind of swap entries. 4711 */ 4712 if (absent || is_swap_pte(huge_ptep_get(pte)) || 4713 ((flags & FOLL_WRITE) && 4714 !huge_pte_write(huge_ptep_get(pte)))) { 4715 vm_fault_t ret; 4716 unsigned int fault_flags = 0; 4717 4718 if (pte) 4719 spin_unlock(ptl); 4720 if (flags & FOLL_WRITE) 4721 fault_flags |= FAULT_FLAG_WRITE; 4722 if (locked) 4723 fault_flags |= FAULT_FLAG_ALLOW_RETRY | 4724 FAULT_FLAG_KILLABLE; 4725 if (flags & FOLL_NOWAIT) 4726 fault_flags |= FAULT_FLAG_ALLOW_RETRY | 4727 FAULT_FLAG_RETRY_NOWAIT; 4728 if (flags & FOLL_TRIED) { 4729 /* 4730 * Note: FAULT_FLAG_ALLOW_RETRY and 4731 * FAULT_FLAG_TRIED can co-exist 4732 */ 4733 fault_flags |= FAULT_FLAG_TRIED; 4734 } 4735 ret = hugetlb_fault(mm, vma, vaddr, fault_flags); 4736 if (ret & VM_FAULT_ERROR) { 4737 err = vm_fault_to_errno(ret, flags); 4738 remainder = 0; 4739 break; 4740 } 4741 if (ret & VM_FAULT_RETRY) { 4742 if (locked && 4743 !(fault_flags & FAULT_FLAG_RETRY_NOWAIT)) 4744 *locked = 0; 4745 *nr_pages = 0; 4746 /* 4747 * VM_FAULT_RETRY must not return an 4748 * error, it will return zero 4749 * instead. 4750 * 4751 * No need to update "position" as the 4752 * caller will not check it after 4753 * *nr_pages is set to 0. 4754 */ 4755 return i; 4756 } 4757 continue; 4758 } 4759 4760 pfn_offset = (vaddr & ~huge_page_mask(h)) >> PAGE_SHIFT; 4761 page = pte_page(huge_ptep_get(pte)); 4762 4763 /* 4764 * If subpage information not requested, update counters 4765 * and skip the same_page loop below. 4766 */ 4767 if (!pages && !vmas && !pfn_offset && 4768 (vaddr + huge_page_size(h) < vma->vm_end) && 4769 (remainder >= pages_per_huge_page(h))) { 4770 vaddr += huge_page_size(h); 4771 remainder -= pages_per_huge_page(h); 4772 i += pages_per_huge_page(h); 4773 spin_unlock(ptl); 4774 continue; 4775 } 4776 4777 same_page: 4778 if (pages) { 4779 pages[i] = mem_map_offset(page, pfn_offset); 4780 /* 4781 * try_grab_page() should always succeed here, because: 4782 * a) we hold the ptl lock, and b) we've just checked 4783 * that the huge page is present in the page tables. If 4784 * the huge page is present, then the tail pages must 4785 * also be present. The ptl prevents the head page and 4786 * tail pages from being rearranged in any way. So this 4787 * page must be available at this point, unless the page 4788 * refcount overflowed: 4789 */ 4790 if (WARN_ON_ONCE(!try_grab_page(pages[i], flags))) { 4791 spin_unlock(ptl); 4792 remainder = 0; 4793 err = -ENOMEM; 4794 break; 4795 } 4796 } 4797 4798 if (vmas) 4799 vmas[i] = vma; 4800 4801 vaddr += PAGE_SIZE; 4802 ++pfn_offset; 4803 --remainder; 4804 ++i; 4805 if (vaddr < vma->vm_end && remainder && 4806 pfn_offset < pages_per_huge_page(h)) { 4807 /* 4808 * We use pfn_offset to avoid touching the pageframes 4809 * of this compound page. 4810 */ 4811 goto same_page; 4812 } 4813 spin_unlock(ptl); 4814 } 4815 *nr_pages = remainder; 4816 /* 4817 * setting position is actually required only if remainder is 4818 * not zero but it's faster not to add a "if (remainder)" 4819 * branch. 4820 */ 4821 *position = vaddr; 4822 4823 return i ? i : err; 4824 } 4825 4826 #ifndef __HAVE_ARCH_FLUSH_HUGETLB_TLB_RANGE 4827 /* 4828 * ARCHes with special requirements for evicting HUGETLB backing TLB entries can 4829 * implement this. 4830 */ 4831 #define flush_hugetlb_tlb_range(vma, addr, end) flush_tlb_range(vma, addr, end) 4832 #endif 4833 4834 unsigned long hugetlb_change_protection(struct vm_area_struct *vma, 4835 unsigned long address, unsigned long end, pgprot_t newprot) 4836 { 4837 struct mm_struct *mm = vma->vm_mm; 4838 unsigned long start = address; 4839 pte_t *ptep; 4840 pte_t pte; 4841 struct hstate *h = hstate_vma(vma); 4842 unsigned long pages = 0; 4843 bool shared_pmd = false; 4844 struct mmu_notifier_range range; 4845 4846 /* 4847 * In the case of shared PMDs, the area to flush could be beyond 4848 * start/end. Set range.start/range.end to cover the maximum possible 4849 * range if PMD sharing is possible. 4850 */ 4851 mmu_notifier_range_init(&range, MMU_NOTIFY_PROTECTION_VMA, 4852 0, vma, mm, start, end); 4853 adjust_range_if_pmd_sharing_possible(vma, &range.start, &range.end); 4854 4855 BUG_ON(address >= end); 4856 flush_cache_range(vma, range.start, range.end); 4857 4858 mmu_notifier_invalidate_range_start(&range); 4859 i_mmap_lock_write(vma->vm_file->f_mapping); 4860 for (; address < end; address += huge_page_size(h)) { 4861 spinlock_t *ptl; 4862 ptep = huge_pte_offset(mm, address, huge_page_size(h)); 4863 if (!ptep) 4864 continue; 4865 ptl = huge_pte_lock(h, mm, ptep); 4866 if (huge_pmd_unshare(mm, &address, ptep)) { 4867 pages++; 4868 spin_unlock(ptl); 4869 shared_pmd = true; 4870 continue; 4871 } 4872 pte = huge_ptep_get(ptep); 4873 if (unlikely(is_hugetlb_entry_hwpoisoned(pte))) { 4874 spin_unlock(ptl); 4875 continue; 4876 } 4877 if (unlikely(is_hugetlb_entry_migration(pte))) { 4878 swp_entry_t entry = pte_to_swp_entry(pte); 4879 4880 if (is_write_migration_entry(entry)) { 4881 pte_t newpte; 4882 4883 make_migration_entry_read(&entry); 4884 newpte = swp_entry_to_pte(entry); 4885 set_huge_swap_pte_at(mm, address, ptep, 4886 newpte, huge_page_size(h)); 4887 pages++; 4888 } 4889 spin_unlock(ptl); 4890 continue; 4891 } 4892 if (!huge_pte_none(pte)) { 4893 pte_t old_pte; 4894 4895 old_pte = huge_ptep_modify_prot_start(vma, address, ptep); 4896 pte = pte_mkhuge(huge_pte_modify(old_pte, newprot)); 4897 pte = arch_make_huge_pte(pte, vma, NULL, 0); 4898 huge_ptep_modify_prot_commit(vma, address, ptep, old_pte, pte); 4899 pages++; 4900 } 4901 spin_unlock(ptl); 4902 } 4903 /* 4904 * Must flush TLB before releasing i_mmap_rwsem: x86's huge_pmd_unshare 4905 * may have cleared our pud entry and done put_page on the page table: 4906 * once we release i_mmap_rwsem, another task can do the final put_page 4907 * and that page table be reused and filled with junk. If we actually 4908 * did unshare a page of pmds, flush the range corresponding to the pud. 4909 */ 4910 if (shared_pmd) 4911 flush_hugetlb_tlb_range(vma, range.start, range.end); 4912 else 4913 flush_hugetlb_tlb_range(vma, start, end); 4914 /* 4915 * No need to call mmu_notifier_invalidate_range() we are downgrading 4916 * page table protection not changing it to point to a new page. 4917 * 4918 * See Documentation/vm/mmu_notifier.rst 4919 */ 4920 i_mmap_unlock_write(vma->vm_file->f_mapping); 4921 mmu_notifier_invalidate_range_end(&range); 4922 4923 return pages << h->order; 4924 } 4925 4926 int hugetlb_reserve_pages(struct inode *inode, 4927 long from, long to, 4928 struct vm_area_struct *vma, 4929 vm_flags_t vm_flags) 4930 { 4931 long ret, chg, add = -1; 4932 struct hstate *h = hstate_inode(inode); 4933 struct hugepage_subpool *spool = subpool_inode(inode); 4934 struct resv_map *resv_map; 4935 struct hugetlb_cgroup *h_cg = NULL; 4936 long gbl_reserve, regions_needed = 0; 4937 4938 /* This should never happen */ 4939 if (from > to) { 4940 VM_WARN(1, "%s called with a negative range\n", __func__); 4941 return -EINVAL; 4942 } 4943 4944 /* 4945 * Only apply hugepage reservation if asked. At fault time, an 4946 * attempt will be made for VM_NORESERVE to allocate a page 4947 * without using reserves 4948 */ 4949 if (vm_flags & VM_NORESERVE) 4950 return 0; 4951 4952 /* 4953 * Shared mappings base their reservation on the number of pages that 4954 * are already allocated on behalf of the file. Private mappings need 4955 * to reserve the full area even if read-only as mprotect() may be 4956 * called to make the mapping read-write. Assume !vma is a shm mapping 4957 */ 4958 if (!vma || vma->vm_flags & VM_MAYSHARE) { 4959 /* 4960 * resv_map can not be NULL as hugetlb_reserve_pages is only 4961 * called for inodes for which resv_maps were created (see 4962 * hugetlbfs_get_inode). 4963 */ 4964 resv_map = inode_resv_map(inode); 4965 4966 chg = region_chg(resv_map, from, to, ®ions_needed); 4967 4968 } else { 4969 /* Private mapping. */ 4970 resv_map = resv_map_alloc(); 4971 if (!resv_map) 4972 return -ENOMEM; 4973 4974 chg = to - from; 4975 4976 set_vma_resv_map(vma, resv_map); 4977 set_vma_resv_flags(vma, HPAGE_RESV_OWNER); 4978 } 4979 4980 if (chg < 0) { 4981 ret = chg; 4982 goto out_err; 4983 } 4984 4985 ret = hugetlb_cgroup_charge_cgroup_rsvd( 4986 hstate_index(h), chg * pages_per_huge_page(h), &h_cg); 4987 4988 if (ret < 0) { 4989 ret = -ENOMEM; 4990 goto out_err; 4991 } 4992 4993 if (vma && !(vma->vm_flags & VM_MAYSHARE) && h_cg) { 4994 /* For private mappings, the hugetlb_cgroup uncharge info hangs 4995 * of the resv_map. 4996 */ 4997 resv_map_set_hugetlb_cgroup_uncharge_info(resv_map, h_cg, h); 4998 } 4999 5000 /* 5001 * There must be enough pages in the subpool for the mapping. If 5002 * the subpool has a minimum size, there may be some global 5003 * reservations already in place (gbl_reserve). 5004 */ 5005 gbl_reserve = hugepage_subpool_get_pages(spool, chg); 5006 if (gbl_reserve < 0) { 5007 ret = -ENOSPC; 5008 goto out_uncharge_cgroup; 5009 } 5010 5011 /* 5012 * Check enough hugepages are available for the reservation. 5013 * Hand the pages back to the subpool if there are not 5014 */ 5015 ret = hugetlb_acct_memory(h, gbl_reserve); 5016 if (ret < 0) { 5017 goto out_put_pages; 5018 } 5019 5020 /* 5021 * Account for the reservations made. Shared mappings record regions 5022 * that have reservations as they are shared by multiple VMAs. 5023 * When the last VMA disappears, the region map says how much 5024 * the reservation was and the page cache tells how much of 5025 * the reservation was consumed. Private mappings are per-VMA and 5026 * only the consumed reservations are tracked. When the VMA 5027 * disappears, the original reservation is the VMA size and the 5028 * consumed reservations are stored in the map. Hence, nothing 5029 * else has to be done for private mappings here 5030 */ 5031 if (!vma || vma->vm_flags & VM_MAYSHARE) { 5032 add = region_add(resv_map, from, to, regions_needed, h, h_cg); 5033 5034 if (unlikely(add < 0)) { 5035 hugetlb_acct_memory(h, -gbl_reserve); 5036 goto out_put_pages; 5037 } else if (unlikely(chg > add)) { 5038 /* 5039 * pages in this range were added to the reserve 5040 * map between region_chg and region_add. This 5041 * indicates a race with alloc_huge_page. Adjust 5042 * the subpool and reserve counts modified above 5043 * based on the difference. 5044 */ 5045 long rsv_adjust; 5046 5047 hugetlb_cgroup_uncharge_cgroup_rsvd( 5048 hstate_index(h), 5049 (chg - add) * pages_per_huge_page(h), h_cg); 5050 5051 rsv_adjust = hugepage_subpool_put_pages(spool, 5052 chg - add); 5053 hugetlb_acct_memory(h, -rsv_adjust); 5054 } 5055 } 5056 return 0; 5057 out_put_pages: 5058 /* put back original number of pages, chg */ 5059 (void)hugepage_subpool_put_pages(spool, chg); 5060 out_uncharge_cgroup: 5061 hugetlb_cgroup_uncharge_cgroup_rsvd(hstate_index(h), 5062 chg * pages_per_huge_page(h), h_cg); 5063 out_err: 5064 if (!vma || vma->vm_flags & VM_MAYSHARE) 5065 /* Only call region_abort if the region_chg succeeded but the 5066 * region_add failed or didn't run. 5067 */ 5068 if (chg >= 0 && add < 0) 5069 region_abort(resv_map, from, to, regions_needed); 5070 if (vma && is_vma_resv_set(vma, HPAGE_RESV_OWNER)) 5071 kref_put(&resv_map->refs, resv_map_release); 5072 return ret; 5073 } 5074 5075 long hugetlb_unreserve_pages(struct inode *inode, long start, long end, 5076 long freed) 5077 { 5078 struct hstate *h = hstate_inode(inode); 5079 struct resv_map *resv_map = inode_resv_map(inode); 5080 long chg = 0; 5081 struct hugepage_subpool *spool = subpool_inode(inode); 5082 long gbl_reserve; 5083 5084 /* 5085 * Since this routine can be called in the evict inode path for all 5086 * hugetlbfs inodes, resv_map could be NULL. 5087 */ 5088 if (resv_map) { 5089 chg = region_del(resv_map, start, end); 5090 /* 5091 * region_del() can fail in the rare case where a region 5092 * must be split and another region descriptor can not be 5093 * allocated. If end == LONG_MAX, it will not fail. 5094 */ 5095 if (chg < 0) 5096 return chg; 5097 } 5098 5099 spin_lock(&inode->i_lock); 5100 inode->i_blocks -= (blocks_per_huge_page(h) * freed); 5101 spin_unlock(&inode->i_lock); 5102 5103 /* 5104 * If the subpool has a minimum size, the number of global 5105 * reservations to be released may be adjusted. 5106 */ 5107 gbl_reserve = hugepage_subpool_put_pages(spool, (chg - freed)); 5108 hugetlb_acct_memory(h, -gbl_reserve); 5109 5110 return 0; 5111 } 5112 5113 #ifdef CONFIG_ARCH_WANT_HUGE_PMD_SHARE 5114 static unsigned long page_table_shareable(struct vm_area_struct *svma, 5115 struct vm_area_struct *vma, 5116 unsigned long addr, pgoff_t idx) 5117 { 5118 unsigned long saddr = ((idx - svma->vm_pgoff) << PAGE_SHIFT) + 5119 svma->vm_start; 5120 unsigned long sbase = saddr & PUD_MASK; 5121 unsigned long s_end = sbase + PUD_SIZE; 5122 5123 /* Allow segments to share if only one is marked locked */ 5124 unsigned long vm_flags = vma->vm_flags & VM_LOCKED_CLEAR_MASK; 5125 unsigned long svm_flags = svma->vm_flags & VM_LOCKED_CLEAR_MASK; 5126 5127 /* 5128 * match the virtual addresses, permission and the alignment of the 5129 * page table page. 5130 */ 5131 if (pmd_index(addr) != pmd_index(saddr) || 5132 vm_flags != svm_flags || 5133 sbase < svma->vm_start || svma->vm_end < s_end) 5134 return 0; 5135 5136 return saddr; 5137 } 5138 5139 static bool vma_shareable(struct vm_area_struct *vma, unsigned long addr) 5140 { 5141 unsigned long base = addr & PUD_MASK; 5142 unsigned long end = base + PUD_SIZE; 5143 5144 /* 5145 * check on proper vm_flags and page table alignment 5146 */ 5147 if (vma->vm_flags & VM_MAYSHARE && range_in_vma(vma, base, end)) 5148 return true; 5149 return false; 5150 } 5151 5152 /* 5153 * Determine if start,end range within vma could be mapped by shared pmd. 5154 * If yes, adjust start and end to cover range associated with possible 5155 * shared pmd mappings. 5156 */ 5157 void adjust_range_if_pmd_sharing_possible(struct vm_area_struct *vma, 5158 unsigned long *start, unsigned long *end) 5159 { 5160 unsigned long check_addr; 5161 5162 if (!(vma->vm_flags & VM_MAYSHARE)) 5163 return; 5164 5165 for (check_addr = *start; check_addr < *end; check_addr += PUD_SIZE) { 5166 unsigned long a_start = check_addr & PUD_MASK; 5167 unsigned long a_end = a_start + PUD_SIZE; 5168 5169 /* 5170 * If sharing is possible, adjust start/end if necessary. 5171 */ 5172 if (range_in_vma(vma, a_start, a_end)) { 5173 if (a_start < *start) 5174 *start = a_start; 5175 if (a_end > *end) 5176 *end = a_end; 5177 } 5178 } 5179 } 5180 5181 /* 5182 * Search for a shareable pmd page for hugetlb. In any case calls pmd_alloc() 5183 * and returns the corresponding pte. While this is not necessary for the 5184 * !shared pmd case because we can allocate the pmd later as well, it makes the 5185 * code much cleaner. 5186 * 5187 * This routine must be called with i_mmap_rwsem held in at least read mode. 5188 * For hugetlbfs, this prevents removal of any page table entries associated 5189 * with the address space. This is important as we are setting up sharing 5190 * based on existing page table entries (mappings). 5191 */ 5192 pte_t *huge_pmd_share(struct mm_struct *mm, unsigned long addr, pud_t *pud) 5193 { 5194 struct vm_area_struct *vma = find_vma(mm, addr); 5195 struct address_space *mapping = vma->vm_file->f_mapping; 5196 pgoff_t idx = ((addr - vma->vm_start) >> PAGE_SHIFT) + 5197 vma->vm_pgoff; 5198 struct vm_area_struct *svma; 5199 unsigned long saddr; 5200 pte_t *spte = NULL; 5201 pte_t *pte; 5202 spinlock_t *ptl; 5203 5204 if (!vma_shareable(vma, addr)) 5205 return (pte_t *)pmd_alloc(mm, pud, addr); 5206 5207 vma_interval_tree_foreach(svma, &mapping->i_mmap, idx, idx) { 5208 if (svma == vma) 5209 continue; 5210 5211 saddr = page_table_shareable(svma, vma, addr, idx); 5212 if (saddr) { 5213 spte = huge_pte_offset(svma->vm_mm, saddr, 5214 vma_mmu_pagesize(svma)); 5215 if (spte) { 5216 get_page(virt_to_page(spte)); 5217 break; 5218 } 5219 } 5220 } 5221 5222 if (!spte) 5223 goto out; 5224 5225 ptl = huge_pte_lock(hstate_vma(vma), mm, spte); 5226 if (pud_none(*pud)) { 5227 pud_populate(mm, pud, 5228 (pmd_t *)((unsigned long)spte & PAGE_MASK)); 5229 mm_inc_nr_pmds(mm); 5230 } else { 5231 put_page(virt_to_page(spte)); 5232 } 5233 spin_unlock(ptl); 5234 out: 5235 pte = (pte_t *)pmd_alloc(mm, pud, addr); 5236 return pte; 5237 } 5238 5239 /* 5240 * unmap huge page backed by shared pte. 5241 * 5242 * Hugetlb pte page is ref counted at the time of mapping. If pte is shared 5243 * indicated by page_count > 1, unmap is achieved by clearing pud and 5244 * decrementing the ref count. If count == 1, the pte page is not shared. 5245 * 5246 * Called with page table lock held and i_mmap_rwsem held in write mode. 5247 * 5248 * returns: 1 successfully unmapped a shared pte page 5249 * 0 the underlying pte page is not shared, or it is the last user 5250 */ 5251 int huge_pmd_unshare(struct mm_struct *mm, unsigned long *addr, pte_t *ptep) 5252 { 5253 pgd_t *pgd = pgd_offset(mm, *addr); 5254 p4d_t *p4d = p4d_offset(pgd, *addr); 5255 pud_t *pud = pud_offset(p4d, *addr); 5256 5257 BUG_ON(page_count(virt_to_page(ptep)) == 0); 5258 if (page_count(virt_to_page(ptep)) == 1) 5259 return 0; 5260 5261 pud_clear(pud); 5262 put_page(virt_to_page(ptep)); 5263 mm_dec_nr_pmds(mm); 5264 *addr = ALIGN(*addr, HPAGE_SIZE * PTRS_PER_PTE) - HPAGE_SIZE; 5265 return 1; 5266 } 5267 #define want_pmd_share() (1) 5268 #else /* !CONFIG_ARCH_WANT_HUGE_PMD_SHARE */ 5269 pte_t *huge_pmd_share(struct mm_struct *mm, unsigned long addr, pud_t *pud) 5270 { 5271 return NULL; 5272 } 5273 5274 int huge_pmd_unshare(struct mm_struct *mm, unsigned long *addr, pte_t *ptep) 5275 { 5276 return 0; 5277 } 5278 5279 void adjust_range_if_pmd_sharing_possible(struct vm_area_struct *vma, 5280 unsigned long *start, unsigned long *end) 5281 { 5282 } 5283 #define want_pmd_share() (0) 5284 #endif /* CONFIG_ARCH_WANT_HUGE_PMD_SHARE */ 5285 5286 #ifdef CONFIG_ARCH_WANT_GENERAL_HUGETLB 5287 pte_t *huge_pte_alloc(struct mm_struct *mm, 5288 unsigned long addr, unsigned long sz) 5289 { 5290 pgd_t *pgd; 5291 p4d_t *p4d; 5292 pud_t *pud; 5293 pte_t *pte = NULL; 5294 5295 pgd = pgd_offset(mm, addr); 5296 p4d = p4d_alloc(mm, pgd, addr); 5297 if (!p4d) 5298 return NULL; 5299 pud = pud_alloc(mm, p4d, addr); 5300 if (pud) { 5301 if (sz == PUD_SIZE) { 5302 pte = (pte_t *)pud; 5303 } else { 5304 BUG_ON(sz != PMD_SIZE); 5305 if (want_pmd_share() && pud_none(*pud)) 5306 pte = huge_pmd_share(mm, addr, pud); 5307 else 5308 pte = (pte_t *)pmd_alloc(mm, pud, addr); 5309 } 5310 } 5311 BUG_ON(pte && pte_present(*pte) && !pte_huge(*pte)); 5312 5313 return pte; 5314 } 5315 5316 /* 5317 * huge_pte_offset() - Walk the page table to resolve the hugepage 5318 * entry at address @addr 5319 * 5320 * Return: Pointer to page table or swap entry (PUD or PMD) for 5321 * address @addr, or NULL if a p*d_none() entry is encountered and the 5322 * size @sz doesn't match the hugepage size at this level of the page 5323 * table. 5324 */ 5325 pte_t *huge_pte_offset(struct mm_struct *mm, 5326 unsigned long addr, unsigned long sz) 5327 { 5328 pgd_t *pgd; 5329 p4d_t *p4d; 5330 pud_t *pud; 5331 pmd_t *pmd; 5332 5333 pgd = pgd_offset(mm, addr); 5334 if (!pgd_present(*pgd)) 5335 return NULL; 5336 p4d = p4d_offset(pgd, addr); 5337 if (!p4d_present(*p4d)) 5338 return NULL; 5339 5340 pud = pud_offset(p4d, addr); 5341 if (sz != PUD_SIZE && pud_none(*pud)) 5342 return NULL; 5343 /* hugepage or swap? */ 5344 if (pud_huge(*pud) || !pud_present(*pud)) 5345 return (pte_t *)pud; 5346 5347 pmd = pmd_offset(pud, addr); 5348 if (sz != PMD_SIZE && pmd_none(*pmd)) 5349 return NULL; 5350 /* hugepage or swap? */ 5351 if (pmd_huge(*pmd) || !pmd_present(*pmd)) 5352 return (pte_t *)pmd; 5353 5354 return NULL; 5355 } 5356 5357 #endif /* CONFIG_ARCH_WANT_GENERAL_HUGETLB */ 5358 5359 /* 5360 * These functions are overwritable if your architecture needs its own 5361 * behavior. 5362 */ 5363 struct page * __weak 5364 follow_huge_addr(struct mm_struct *mm, unsigned long address, 5365 int write) 5366 { 5367 return ERR_PTR(-EINVAL); 5368 } 5369 5370 struct page * __weak 5371 follow_huge_pd(struct vm_area_struct *vma, 5372 unsigned long address, hugepd_t hpd, int flags, int pdshift) 5373 { 5374 WARN(1, "hugepd follow called with no support for hugepage directory format\n"); 5375 return NULL; 5376 } 5377 5378 struct page * __weak 5379 follow_huge_pmd(struct mm_struct *mm, unsigned long address, 5380 pmd_t *pmd, int flags) 5381 { 5382 struct page *page = NULL; 5383 spinlock_t *ptl; 5384 pte_t pte; 5385 5386 /* FOLL_GET and FOLL_PIN are mutually exclusive. */ 5387 if (WARN_ON_ONCE((flags & (FOLL_PIN | FOLL_GET)) == 5388 (FOLL_PIN | FOLL_GET))) 5389 return NULL; 5390 5391 retry: 5392 ptl = pmd_lockptr(mm, pmd); 5393 spin_lock(ptl); 5394 /* 5395 * make sure that the address range covered by this pmd is not 5396 * unmapped from other threads. 5397 */ 5398 if (!pmd_huge(*pmd)) 5399 goto out; 5400 pte = huge_ptep_get((pte_t *)pmd); 5401 if (pte_present(pte)) { 5402 page = pmd_page(*pmd) + ((address & ~PMD_MASK) >> PAGE_SHIFT); 5403 /* 5404 * try_grab_page() should always succeed here, because: a) we 5405 * hold the pmd (ptl) lock, and b) we've just checked that the 5406 * huge pmd (head) page is present in the page tables. The ptl 5407 * prevents the head page and tail pages from being rearranged 5408 * in any way. So this page must be available at this point, 5409 * unless the page refcount overflowed: 5410 */ 5411 if (WARN_ON_ONCE(!try_grab_page(page, flags))) { 5412 page = NULL; 5413 goto out; 5414 } 5415 } else { 5416 if (is_hugetlb_entry_migration(pte)) { 5417 spin_unlock(ptl); 5418 __migration_entry_wait(mm, (pte_t *)pmd, ptl); 5419 goto retry; 5420 } 5421 /* 5422 * hwpoisoned entry is treated as no_page_table in 5423 * follow_page_mask(). 5424 */ 5425 } 5426 out: 5427 spin_unlock(ptl); 5428 return page; 5429 } 5430 5431 struct page * __weak 5432 follow_huge_pud(struct mm_struct *mm, unsigned long address, 5433 pud_t *pud, int flags) 5434 { 5435 if (flags & (FOLL_GET | FOLL_PIN)) 5436 return NULL; 5437 5438 return pte_page(*(pte_t *)pud) + ((address & ~PUD_MASK) >> PAGE_SHIFT); 5439 } 5440 5441 struct page * __weak 5442 follow_huge_pgd(struct mm_struct *mm, unsigned long address, pgd_t *pgd, int flags) 5443 { 5444 if (flags & (FOLL_GET | FOLL_PIN)) 5445 return NULL; 5446 5447 return pte_page(*(pte_t *)pgd) + ((address & ~PGDIR_MASK) >> PAGE_SHIFT); 5448 } 5449 5450 bool isolate_huge_page(struct page *page, struct list_head *list) 5451 { 5452 bool ret = true; 5453 5454 VM_BUG_ON_PAGE(!PageHead(page), page); 5455 spin_lock(&hugetlb_lock); 5456 if (!page_huge_active(page) || !get_page_unless_zero(page)) { 5457 ret = false; 5458 goto unlock; 5459 } 5460 clear_page_huge_active(page); 5461 list_move_tail(&page->lru, list); 5462 unlock: 5463 spin_unlock(&hugetlb_lock); 5464 return ret; 5465 } 5466 5467 void putback_active_hugepage(struct page *page) 5468 { 5469 VM_BUG_ON_PAGE(!PageHead(page), page); 5470 spin_lock(&hugetlb_lock); 5471 set_page_huge_active(page); 5472 list_move_tail(&page->lru, &(page_hstate(page))->hugepage_activelist); 5473 spin_unlock(&hugetlb_lock); 5474 put_page(page); 5475 } 5476 5477 void move_hugetlb_state(struct page *oldpage, struct page *newpage, int reason) 5478 { 5479 struct hstate *h = page_hstate(oldpage); 5480 5481 hugetlb_cgroup_migrate(oldpage, newpage); 5482 set_page_owner_migrate_reason(newpage, reason); 5483 5484 /* 5485 * transfer temporary state of the new huge page. This is 5486 * reverse to other transitions because the newpage is going to 5487 * be final while the old one will be freed so it takes over 5488 * the temporary status. 5489 * 5490 * Also note that we have to transfer the per-node surplus state 5491 * here as well otherwise the global surplus count will not match 5492 * the per-node's. 5493 */ 5494 if (PageHugeTemporary(newpage)) { 5495 int old_nid = page_to_nid(oldpage); 5496 int new_nid = page_to_nid(newpage); 5497 5498 SetPageHugeTemporary(oldpage); 5499 ClearPageHugeTemporary(newpage); 5500 5501 spin_lock(&hugetlb_lock); 5502 if (h->surplus_huge_pages_node[old_nid]) { 5503 h->surplus_huge_pages_node[old_nid]--; 5504 h->surplus_huge_pages_node[new_nid]++; 5505 } 5506 spin_unlock(&hugetlb_lock); 5507 } 5508 } 5509