1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Generic hugetlb support. 4 * (C) Nadia Yvette Chambers, April 2004 5 */ 6 #include <linux/list.h> 7 #include <linux/init.h> 8 #include <linux/mm.h> 9 #include <linux/seq_file.h> 10 #include <linux/sysctl.h> 11 #include <linux/highmem.h> 12 #include <linux/mmu_notifier.h> 13 #include <linux/nodemask.h> 14 #include <linux/pagemap.h> 15 #include <linux/mempolicy.h> 16 #include <linux/compiler.h> 17 #include <linux/cpuset.h> 18 #include <linux/mutex.h> 19 #include <linux/memblock.h> 20 #include <linux/sysfs.h> 21 #include <linux/slab.h> 22 #include <linux/sched/mm.h> 23 #include <linux/mmdebug.h> 24 #include <linux/sched/signal.h> 25 #include <linux/rmap.h> 26 #include <linux/string_helpers.h> 27 #include <linux/swap.h> 28 #include <linux/swapops.h> 29 #include <linux/jhash.h> 30 #include <linux/numa.h> 31 #include <linux/llist.h> 32 #include <linux/cma.h> 33 34 #include <asm/page.h> 35 #include <asm/pgalloc.h> 36 #include <asm/tlb.h> 37 38 #include <linux/io.h> 39 #include <linux/hugetlb.h> 40 #include <linux/hugetlb_cgroup.h> 41 #include <linux/node.h> 42 #include <linux/userfaultfd_k.h> 43 #include <linux/page_owner.h> 44 #include "internal.h" 45 46 int hugetlb_max_hstate __read_mostly; 47 unsigned int default_hstate_idx; 48 struct hstate hstates[HUGE_MAX_HSTATE]; 49 50 #ifdef CONFIG_CMA 51 static struct cma *hugetlb_cma[MAX_NUMNODES]; 52 #endif 53 static unsigned long hugetlb_cma_size __initdata; 54 55 /* 56 * Minimum page order among possible hugepage sizes, set to a proper value 57 * at boot time. 58 */ 59 static unsigned int minimum_order __read_mostly = UINT_MAX; 60 61 __initdata LIST_HEAD(huge_boot_pages); 62 63 /* for command line parsing */ 64 static struct hstate * __initdata parsed_hstate; 65 static unsigned long __initdata default_hstate_max_huge_pages; 66 static bool __initdata parsed_valid_hugepagesz = true; 67 static bool __initdata parsed_default_hugepagesz; 68 69 /* 70 * Protects updates to hugepage_freelists, hugepage_activelist, nr_huge_pages, 71 * free_huge_pages, and surplus_huge_pages. 72 */ 73 DEFINE_SPINLOCK(hugetlb_lock); 74 75 /* 76 * Serializes faults on the same logical page. This is used to 77 * prevent spurious OOMs when the hugepage pool is fully utilized. 78 */ 79 static int num_fault_mutexes; 80 struct mutex *hugetlb_fault_mutex_table ____cacheline_aligned_in_smp; 81 82 /* Forward declaration */ 83 static int hugetlb_acct_memory(struct hstate *h, long delta); 84 85 static inline void unlock_or_release_subpool(struct hugepage_subpool *spool) 86 { 87 bool free = (spool->count == 0) && (spool->used_hpages == 0); 88 89 spin_unlock(&spool->lock); 90 91 /* If no pages are used, and no other handles to the subpool 92 * remain, give up any reservations based on minimum size and 93 * free the subpool */ 94 if (free) { 95 if (spool->min_hpages != -1) 96 hugetlb_acct_memory(spool->hstate, 97 -spool->min_hpages); 98 kfree(spool); 99 } 100 } 101 102 struct hugepage_subpool *hugepage_new_subpool(struct hstate *h, long max_hpages, 103 long min_hpages) 104 { 105 struct hugepage_subpool *spool; 106 107 spool = kzalloc(sizeof(*spool), GFP_KERNEL); 108 if (!spool) 109 return NULL; 110 111 spin_lock_init(&spool->lock); 112 spool->count = 1; 113 spool->max_hpages = max_hpages; 114 spool->hstate = h; 115 spool->min_hpages = min_hpages; 116 117 if (min_hpages != -1 && hugetlb_acct_memory(h, min_hpages)) { 118 kfree(spool); 119 return NULL; 120 } 121 spool->rsv_hpages = min_hpages; 122 123 return spool; 124 } 125 126 void hugepage_put_subpool(struct hugepage_subpool *spool) 127 { 128 spin_lock(&spool->lock); 129 BUG_ON(!spool->count); 130 spool->count--; 131 unlock_or_release_subpool(spool); 132 } 133 134 /* 135 * Subpool accounting for allocating and reserving pages. 136 * Return -ENOMEM if there are not enough resources to satisfy the 137 * request. Otherwise, return the number of pages by which the 138 * global pools must be adjusted (upward). The returned value may 139 * only be different than the passed value (delta) in the case where 140 * a subpool minimum size must be maintained. 141 */ 142 static long hugepage_subpool_get_pages(struct hugepage_subpool *spool, 143 long delta) 144 { 145 long ret = delta; 146 147 if (!spool) 148 return ret; 149 150 spin_lock(&spool->lock); 151 152 if (spool->max_hpages != -1) { /* maximum size accounting */ 153 if ((spool->used_hpages + delta) <= spool->max_hpages) 154 spool->used_hpages += delta; 155 else { 156 ret = -ENOMEM; 157 goto unlock_ret; 158 } 159 } 160 161 /* minimum size accounting */ 162 if (spool->min_hpages != -1 && spool->rsv_hpages) { 163 if (delta > spool->rsv_hpages) { 164 /* 165 * Asking for more reserves than those already taken on 166 * behalf of subpool. Return difference. 167 */ 168 ret = delta - spool->rsv_hpages; 169 spool->rsv_hpages = 0; 170 } else { 171 ret = 0; /* reserves already accounted for */ 172 spool->rsv_hpages -= delta; 173 } 174 } 175 176 unlock_ret: 177 spin_unlock(&spool->lock); 178 return ret; 179 } 180 181 /* 182 * Subpool accounting for freeing and unreserving pages. 183 * Return the number of global page reservations that must be dropped. 184 * The return value may only be different than the passed value (delta) 185 * in the case where a subpool minimum size must be maintained. 186 */ 187 static long hugepage_subpool_put_pages(struct hugepage_subpool *spool, 188 long delta) 189 { 190 long ret = delta; 191 192 if (!spool) 193 return delta; 194 195 spin_lock(&spool->lock); 196 197 if (spool->max_hpages != -1) /* maximum size accounting */ 198 spool->used_hpages -= delta; 199 200 /* minimum size accounting */ 201 if (spool->min_hpages != -1 && spool->used_hpages < spool->min_hpages) { 202 if (spool->rsv_hpages + delta <= spool->min_hpages) 203 ret = 0; 204 else 205 ret = spool->rsv_hpages + delta - spool->min_hpages; 206 207 spool->rsv_hpages += delta; 208 if (spool->rsv_hpages > spool->min_hpages) 209 spool->rsv_hpages = spool->min_hpages; 210 } 211 212 /* 213 * If hugetlbfs_put_super couldn't free spool due to an outstanding 214 * quota reference, free it now. 215 */ 216 unlock_or_release_subpool(spool); 217 218 return ret; 219 } 220 221 static inline struct hugepage_subpool *subpool_inode(struct inode *inode) 222 { 223 return HUGETLBFS_SB(inode->i_sb)->spool; 224 } 225 226 static inline struct hugepage_subpool *subpool_vma(struct vm_area_struct *vma) 227 { 228 return subpool_inode(file_inode(vma->vm_file)); 229 } 230 231 /* Helper that removes a struct file_region from the resv_map cache and returns 232 * it for use. 233 */ 234 static struct file_region * 235 get_file_region_entry_from_cache(struct resv_map *resv, long from, long to) 236 { 237 struct file_region *nrg = NULL; 238 239 VM_BUG_ON(resv->region_cache_count <= 0); 240 241 resv->region_cache_count--; 242 nrg = list_first_entry(&resv->region_cache, struct file_region, link); 243 list_del(&nrg->link); 244 245 nrg->from = from; 246 nrg->to = to; 247 248 return nrg; 249 } 250 251 static void copy_hugetlb_cgroup_uncharge_info(struct file_region *nrg, 252 struct file_region *rg) 253 { 254 #ifdef CONFIG_CGROUP_HUGETLB 255 nrg->reservation_counter = rg->reservation_counter; 256 nrg->css = rg->css; 257 if (rg->css) 258 css_get(rg->css); 259 #endif 260 } 261 262 /* Helper that records hugetlb_cgroup uncharge info. */ 263 static void record_hugetlb_cgroup_uncharge_info(struct hugetlb_cgroup *h_cg, 264 struct hstate *h, 265 struct resv_map *resv, 266 struct file_region *nrg) 267 { 268 #ifdef CONFIG_CGROUP_HUGETLB 269 if (h_cg) { 270 nrg->reservation_counter = 271 &h_cg->rsvd_hugepage[hstate_index(h)]; 272 nrg->css = &h_cg->css; 273 if (!resv->pages_per_hpage) 274 resv->pages_per_hpage = pages_per_huge_page(h); 275 /* pages_per_hpage should be the same for all entries in 276 * a resv_map. 277 */ 278 VM_BUG_ON(resv->pages_per_hpage != pages_per_huge_page(h)); 279 } else { 280 nrg->reservation_counter = NULL; 281 nrg->css = NULL; 282 } 283 #endif 284 } 285 286 static bool has_same_uncharge_info(struct file_region *rg, 287 struct file_region *org) 288 { 289 #ifdef CONFIG_CGROUP_HUGETLB 290 return rg && org && 291 rg->reservation_counter == org->reservation_counter && 292 rg->css == org->css; 293 294 #else 295 return true; 296 #endif 297 } 298 299 static void coalesce_file_region(struct resv_map *resv, struct file_region *rg) 300 { 301 struct file_region *nrg = NULL, *prg = NULL; 302 303 prg = list_prev_entry(rg, link); 304 if (&prg->link != &resv->regions && prg->to == rg->from && 305 has_same_uncharge_info(prg, rg)) { 306 prg->to = rg->to; 307 308 list_del(&rg->link); 309 kfree(rg); 310 311 rg = prg; 312 } 313 314 nrg = list_next_entry(rg, link); 315 if (&nrg->link != &resv->regions && nrg->from == rg->to && 316 has_same_uncharge_info(nrg, rg)) { 317 nrg->from = rg->from; 318 319 list_del(&rg->link); 320 kfree(rg); 321 } 322 } 323 324 /* 325 * Must be called with resv->lock held. 326 * 327 * Calling this with regions_needed != NULL will count the number of pages 328 * to be added but will not modify the linked list. And regions_needed will 329 * indicate the number of file_regions needed in the cache to carry out to add 330 * the regions for this range. 331 */ 332 static long add_reservation_in_range(struct resv_map *resv, long f, long t, 333 struct hugetlb_cgroup *h_cg, 334 struct hstate *h, long *regions_needed) 335 { 336 long add = 0; 337 struct list_head *head = &resv->regions; 338 long last_accounted_offset = f; 339 struct file_region *rg = NULL, *trg = NULL, *nrg = NULL; 340 341 if (regions_needed) 342 *regions_needed = 0; 343 344 /* In this loop, we essentially handle an entry for the range 345 * [last_accounted_offset, rg->from), at every iteration, with some 346 * bounds checking. 347 */ 348 list_for_each_entry_safe(rg, trg, head, link) { 349 /* Skip irrelevant regions that start before our range. */ 350 if (rg->from < f) { 351 /* If this region ends after the last accounted offset, 352 * then we need to update last_accounted_offset. 353 */ 354 if (rg->to > last_accounted_offset) 355 last_accounted_offset = rg->to; 356 continue; 357 } 358 359 /* When we find a region that starts beyond our range, we've 360 * finished. 361 */ 362 if (rg->from > t) 363 break; 364 365 /* Add an entry for last_accounted_offset -> rg->from, and 366 * update last_accounted_offset. 367 */ 368 if (rg->from > last_accounted_offset) { 369 add += rg->from - last_accounted_offset; 370 if (!regions_needed) { 371 nrg = get_file_region_entry_from_cache( 372 resv, last_accounted_offset, rg->from); 373 record_hugetlb_cgroup_uncharge_info(h_cg, h, 374 resv, nrg); 375 list_add(&nrg->link, rg->link.prev); 376 coalesce_file_region(resv, nrg); 377 } else 378 *regions_needed += 1; 379 } 380 381 last_accounted_offset = rg->to; 382 } 383 384 /* Handle the case where our range extends beyond 385 * last_accounted_offset. 386 */ 387 if (last_accounted_offset < t) { 388 add += t - last_accounted_offset; 389 if (!regions_needed) { 390 nrg = get_file_region_entry_from_cache( 391 resv, last_accounted_offset, t); 392 record_hugetlb_cgroup_uncharge_info(h_cg, h, resv, nrg); 393 list_add(&nrg->link, rg->link.prev); 394 coalesce_file_region(resv, nrg); 395 } else 396 *regions_needed += 1; 397 } 398 399 VM_BUG_ON(add < 0); 400 return add; 401 } 402 403 /* Must be called with resv->lock acquired. Will drop lock to allocate entries. 404 */ 405 static int allocate_file_region_entries(struct resv_map *resv, 406 int regions_needed) 407 __must_hold(&resv->lock) 408 { 409 struct list_head allocated_regions; 410 int to_allocate = 0, i = 0; 411 struct file_region *trg = NULL, *rg = NULL; 412 413 VM_BUG_ON(regions_needed < 0); 414 415 INIT_LIST_HEAD(&allocated_regions); 416 417 /* 418 * Check for sufficient descriptors in the cache to accommodate 419 * the number of in progress add operations plus regions_needed. 420 * 421 * This is a while loop because when we drop the lock, some other call 422 * to region_add or region_del may have consumed some region_entries, 423 * so we keep looping here until we finally have enough entries for 424 * (adds_in_progress + regions_needed). 425 */ 426 while (resv->region_cache_count < 427 (resv->adds_in_progress + regions_needed)) { 428 to_allocate = resv->adds_in_progress + regions_needed - 429 resv->region_cache_count; 430 431 /* At this point, we should have enough entries in the cache 432 * for all the existings adds_in_progress. We should only be 433 * needing to allocate for regions_needed. 434 */ 435 VM_BUG_ON(resv->region_cache_count < resv->adds_in_progress); 436 437 spin_unlock(&resv->lock); 438 for (i = 0; i < to_allocate; i++) { 439 trg = kmalloc(sizeof(*trg), GFP_KERNEL); 440 if (!trg) 441 goto out_of_memory; 442 list_add(&trg->link, &allocated_regions); 443 } 444 445 spin_lock(&resv->lock); 446 447 list_splice(&allocated_regions, &resv->region_cache); 448 resv->region_cache_count += to_allocate; 449 } 450 451 return 0; 452 453 out_of_memory: 454 list_for_each_entry_safe(rg, trg, &allocated_regions, link) { 455 list_del(&rg->link); 456 kfree(rg); 457 } 458 return -ENOMEM; 459 } 460 461 /* 462 * Add the huge page range represented by [f, t) to the reserve 463 * map. Regions will be taken from the cache to fill in this range. 464 * Sufficient regions should exist in the cache due to the previous 465 * call to region_chg with the same range, but in some cases the cache will not 466 * have sufficient entries due to races with other code doing region_add or 467 * region_del. The extra needed entries will be allocated. 468 * 469 * regions_needed is the out value provided by a previous call to region_chg. 470 * 471 * Return the number of new huge pages added to the map. This number is greater 472 * than or equal to zero. If file_region entries needed to be allocated for 473 * this operation and we were not able to allocate, it returns -ENOMEM. 474 * region_add of regions of length 1 never allocate file_regions and cannot 475 * fail; region_chg will always allocate at least 1 entry and a region_add for 476 * 1 page will only require at most 1 entry. 477 */ 478 static long region_add(struct resv_map *resv, long f, long t, 479 long in_regions_needed, struct hstate *h, 480 struct hugetlb_cgroup *h_cg) 481 { 482 long add = 0, actual_regions_needed = 0; 483 484 spin_lock(&resv->lock); 485 retry: 486 487 /* Count how many regions are actually needed to execute this add. */ 488 add_reservation_in_range(resv, f, t, NULL, NULL, 489 &actual_regions_needed); 490 491 /* 492 * Check for sufficient descriptors in the cache to accommodate 493 * this add operation. Note that actual_regions_needed may be greater 494 * than in_regions_needed, as the resv_map may have been modified since 495 * the region_chg call. In this case, we need to make sure that we 496 * allocate extra entries, such that we have enough for all the 497 * existing adds_in_progress, plus the excess needed for this 498 * operation. 499 */ 500 if (actual_regions_needed > in_regions_needed && 501 resv->region_cache_count < 502 resv->adds_in_progress + 503 (actual_regions_needed - in_regions_needed)) { 504 /* region_add operation of range 1 should never need to 505 * allocate file_region entries. 506 */ 507 VM_BUG_ON(t - f <= 1); 508 509 if (allocate_file_region_entries( 510 resv, actual_regions_needed - in_regions_needed)) { 511 return -ENOMEM; 512 } 513 514 goto retry; 515 } 516 517 add = add_reservation_in_range(resv, f, t, h_cg, h, NULL); 518 519 resv->adds_in_progress -= in_regions_needed; 520 521 spin_unlock(&resv->lock); 522 VM_BUG_ON(add < 0); 523 return add; 524 } 525 526 /* 527 * Examine the existing reserve map and determine how many 528 * huge pages in the specified range [f, t) are NOT currently 529 * represented. This routine is called before a subsequent 530 * call to region_add that will actually modify the reserve 531 * map to add the specified range [f, t). region_chg does 532 * not change the number of huge pages represented by the 533 * map. A number of new file_region structures is added to the cache as a 534 * placeholder, for the subsequent region_add call to use. At least 1 535 * file_region structure is added. 536 * 537 * out_regions_needed is the number of regions added to the 538 * resv->adds_in_progress. This value needs to be provided to a follow up call 539 * to region_add or region_abort for proper accounting. 540 * 541 * Returns the number of huge pages that need to be added to the existing 542 * reservation map for the range [f, t). This number is greater or equal to 543 * zero. -ENOMEM is returned if a new file_region structure or cache entry 544 * is needed and can not be allocated. 545 */ 546 static long region_chg(struct resv_map *resv, long f, long t, 547 long *out_regions_needed) 548 { 549 long chg = 0; 550 551 spin_lock(&resv->lock); 552 553 /* Count how many hugepages in this range are NOT represented. */ 554 chg = add_reservation_in_range(resv, f, t, NULL, NULL, 555 out_regions_needed); 556 557 if (*out_regions_needed == 0) 558 *out_regions_needed = 1; 559 560 if (allocate_file_region_entries(resv, *out_regions_needed)) 561 return -ENOMEM; 562 563 resv->adds_in_progress += *out_regions_needed; 564 565 spin_unlock(&resv->lock); 566 return chg; 567 } 568 569 /* 570 * Abort the in progress add operation. The adds_in_progress field 571 * of the resv_map keeps track of the operations in progress between 572 * calls to region_chg and region_add. Operations are sometimes 573 * aborted after the call to region_chg. In such cases, region_abort 574 * is called to decrement the adds_in_progress counter. regions_needed 575 * is the value returned by the region_chg call, it is used to decrement 576 * the adds_in_progress counter. 577 * 578 * NOTE: The range arguments [f, t) are not needed or used in this 579 * routine. They are kept to make reading the calling code easier as 580 * arguments will match the associated region_chg call. 581 */ 582 static void region_abort(struct resv_map *resv, long f, long t, 583 long regions_needed) 584 { 585 spin_lock(&resv->lock); 586 VM_BUG_ON(!resv->region_cache_count); 587 resv->adds_in_progress -= regions_needed; 588 spin_unlock(&resv->lock); 589 } 590 591 /* 592 * Delete the specified range [f, t) from the reserve map. If the 593 * t parameter is LONG_MAX, this indicates that ALL regions after f 594 * should be deleted. Locate the regions which intersect [f, t) 595 * and either trim, delete or split the existing regions. 596 * 597 * Returns the number of huge pages deleted from the reserve map. 598 * In the normal case, the return value is zero or more. In the 599 * case where a region must be split, a new region descriptor must 600 * be allocated. If the allocation fails, -ENOMEM will be returned. 601 * NOTE: If the parameter t == LONG_MAX, then we will never split 602 * a region and possibly return -ENOMEM. Callers specifying 603 * t == LONG_MAX do not need to check for -ENOMEM error. 604 */ 605 static long region_del(struct resv_map *resv, long f, long t) 606 { 607 struct list_head *head = &resv->regions; 608 struct file_region *rg, *trg; 609 struct file_region *nrg = NULL; 610 long del = 0; 611 612 retry: 613 spin_lock(&resv->lock); 614 list_for_each_entry_safe(rg, trg, head, link) { 615 /* 616 * Skip regions before the range to be deleted. file_region 617 * ranges are normally of the form [from, to). However, there 618 * may be a "placeholder" entry in the map which is of the form 619 * (from, to) with from == to. Check for placeholder entries 620 * at the beginning of the range to be deleted. 621 */ 622 if (rg->to <= f && (rg->to != rg->from || rg->to != f)) 623 continue; 624 625 if (rg->from >= t) 626 break; 627 628 if (f > rg->from && t < rg->to) { /* Must split region */ 629 /* 630 * Check for an entry in the cache before dropping 631 * lock and attempting allocation. 632 */ 633 if (!nrg && 634 resv->region_cache_count > resv->adds_in_progress) { 635 nrg = list_first_entry(&resv->region_cache, 636 struct file_region, 637 link); 638 list_del(&nrg->link); 639 resv->region_cache_count--; 640 } 641 642 if (!nrg) { 643 spin_unlock(&resv->lock); 644 nrg = kmalloc(sizeof(*nrg), GFP_KERNEL); 645 if (!nrg) 646 return -ENOMEM; 647 goto retry; 648 } 649 650 del += t - f; 651 hugetlb_cgroup_uncharge_file_region( 652 resv, rg, t - f); 653 654 /* New entry for end of split region */ 655 nrg->from = t; 656 nrg->to = rg->to; 657 658 copy_hugetlb_cgroup_uncharge_info(nrg, rg); 659 660 INIT_LIST_HEAD(&nrg->link); 661 662 /* Original entry is trimmed */ 663 rg->to = f; 664 665 list_add(&nrg->link, &rg->link); 666 nrg = NULL; 667 break; 668 } 669 670 if (f <= rg->from && t >= rg->to) { /* Remove entire region */ 671 del += rg->to - rg->from; 672 hugetlb_cgroup_uncharge_file_region(resv, rg, 673 rg->to - rg->from); 674 list_del(&rg->link); 675 kfree(rg); 676 continue; 677 } 678 679 if (f <= rg->from) { /* Trim beginning of region */ 680 hugetlb_cgroup_uncharge_file_region(resv, rg, 681 t - rg->from); 682 683 del += t - rg->from; 684 rg->from = t; 685 } else { /* Trim end of region */ 686 hugetlb_cgroup_uncharge_file_region(resv, rg, 687 rg->to - f); 688 689 del += rg->to - f; 690 rg->to = f; 691 } 692 } 693 694 spin_unlock(&resv->lock); 695 kfree(nrg); 696 return del; 697 } 698 699 /* 700 * A rare out of memory error was encountered which prevented removal of 701 * the reserve map region for a page. The huge page itself was free'ed 702 * and removed from the page cache. This routine will adjust the subpool 703 * usage count, and the global reserve count if needed. By incrementing 704 * these counts, the reserve map entry which could not be deleted will 705 * appear as a "reserved" entry instead of simply dangling with incorrect 706 * counts. 707 */ 708 void hugetlb_fix_reserve_counts(struct inode *inode) 709 { 710 struct hugepage_subpool *spool = subpool_inode(inode); 711 long rsv_adjust; 712 713 rsv_adjust = hugepage_subpool_get_pages(spool, 1); 714 if (rsv_adjust) { 715 struct hstate *h = hstate_inode(inode); 716 717 hugetlb_acct_memory(h, 1); 718 } 719 } 720 721 /* 722 * Count and return the number of huge pages in the reserve map 723 * that intersect with the range [f, t). 724 */ 725 static long region_count(struct resv_map *resv, long f, long t) 726 { 727 struct list_head *head = &resv->regions; 728 struct file_region *rg; 729 long chg = 0; 730 731 spin_lock(&resv->lock); 732 /* Locate each segment we overlap with, and count that overlap. */ 733 list_for_each_entry(rg, head, link) { 734 long seg_from; 735 long seg_to; 736 737 if (rg->to <= f) 738 continue; 739 if (rg->from >= t) 740 break; 741 742 seg_from = max(rg->from, f); 743 seg_to = min(rg->to, t); 744 745 chg += seg_to - seg_from; 746 } 747 spin_unlock(&resv->lock); 748 749 return chg; 750 } 751 752 /* 753 * Convert the address within this vma to the page offset within 754 * the mapping, in pagecache page units; huge pages here. 755 */ 756 static pgoff_t vma_hugecache_offset(struct hstate *h, 757 struct vm_area_struct *vma, unsigned long address) 758 { 759 return ((address - vma->vm_start) >> huge_page_shift(h)) + 760 (vma->vm_pgoff >> huge_page_order(h)); 761 } 762 763 pgoff_t linear_hugepage_index(struct vm_area_struct *vma, 764 unsigned long address) 765 { 766 return vma_hugecache_offset(hstate_vma(vma), vma, address); 767 } 768 EXPORT_SYMBOL_GPL(linear_hugepage_index); 769 770 /* 771 * Return the size of the pages allocated when backing a VMA. In the majority 772 * cases this will be same size as used by the page table entries. 773 */ 774 unsigned long vma_kernel_pagesize(struct vm_area_struct *vma) 775 { 776 if (vma->vm_ops && vma->vm_ops->pagesize) 777 return vma->vm_ops->pagesize(vma); 778 return PAGE_SIZE; 779 } 780 EXPORT_SYMBOL_GPL(vma_kernel_pagesize); 781 782 /* 783 * Return the page size being used by the MMU to back a VMA. In the majority 784 * of cases, the page size used by the kernel matches the MMU size. On 785 * architectures where it differs, an architecture-specific 'strong' 786 * version of this symbol is required. 787 */ 788 __weak unsigned long vma_mmu_pagesize(struct vm_area_struct *vma) 789 { 790 return vma_kernel_pagesize(vma); 791 } 792 793 /* 794 * Flags for MAP_PRIVATE reservations. These are stored in the bottom 795 * bits of the reservation map pointer, which are always clear due to 796 * alignment. 797 */ 798 #define HPAGE_RESV_OWNER (1UL << 0) 799 #define HPAGE_RESV_UNMAPPED (1UL << 1) 800 #define HPAGE_RESV_MASK (HPAGE_RESV_OWNER | HPAGE_RESV_UNMAPPED) 801 802 /* 803 * These helpers are used to track how many pages are reserved for 804 * faults in a MAP_PRIVATE mapping. Only the process that called mmap() 805 * is guaranteed to have their future faults succeed. 806 * 807 * With the exception of reset_vma_resv_huge_pages() which is called at fork(), 808 * the reserve counters are updated with the hugetlb_lock held. It is safe 809 * to reset the VMA at fork() time as it is not in use yet and there is no 810 * chance of the global counters getting corrupted as a result of the values. 811 * 812 * The private mapping reservation is represented in a subtly different 813 * manner to a shared mapping. A shared mapping has a region map associated 814 * with the underlying file, this region map represents the backing file 815 * pages which have ever had a reservation assigned which this persists even 816 * after the page is instantiated. A private mapping has a region map 817 * associated with the original mmap which is attached to all VMAs which 818 * reference it, this region map represents those offsets which have consumed 819 * reservation ie. where pages have been instantiated. 820 */ 821 static unsigned long get_vma_private_data(struct vm_area_struct *vma) 822 { 823 return (unsigned long)vma->vm_private_data; 824 } 825 826 static void set_vma_private_data(struct vm_area_struct *vma, 827 unsigned long value) 828 { 829 vma->vm_private_data = (void *)value; 830 } 831 832 static void 833 resv_map_set_hugetlb_cgroup_uncharge_info(struct resv_map *resv_map, 834 struct hugetlb_cgroup *h_cg, 835 struct hstate *h) 836 { 837 #ifdef CONFIG_CGROUP_HUGETLB 838 if (!h_cg || !h) { 839 resv_map->reservation_counter = NULL; 840 resv_map->pages_per_hpage = 0; 841 resv_map->css = NULL; 842 } else { 843 resv_map->reservation_counter = 844 &h_cg->rsvd_hugepage[hstate_index(h)]; 845 resv_map->pages_per_hpage = pages_per_huge_page(h); 846 resv_map->css = &h_cg->css; 847 } 848 #endif 849 } 850 851 struct resv_map *resv_map_alloc(void) 852 { 853 struct resv_map *resv_map = kmalloc(sizeof(*resv_map), GFP_KERNEL); 854 struct file_region *rg = kmalloc(sizeof(*rg), GFP_KERNEL); 855 856 if (!resv_map || !rg) { 857 kfree(resv_map); 858 kfree(rg); 859 return NULL; 860 } 861 862 kref_init(&resv_map->refs); 863 spin_lock_init(&resv_map->lock); 864 INIT_LIST_HEAD(&resv_map->regions); 865 866 resv_map->adds_in_progress = 0; 867 /* 868 * Initialize these to 0. On shared mappings, 0's here indicate these 869 * fields don't do cgroup accounting. On private mappings, these will be 870 * re-initialized to the proper values, to indicate that hugetlb cgroup 871 * reservations are to be un-charged from here. 872 */ 873 resv_map_set_hugetlb_cgroup_uncharge_info(resv_map, NULL, NULL); 874 875 INIT_LIST_HEAD(&resv_map->region_cache); 876 list_add(&rg->link, &resv_map->region_cache); 877 resv_map->region_cache_count = 1; 878 879 return resv_map; 880 } 881 882 void resv_map_release(struct kref *ref) 883 { 884 struct resv_map *resv_map = container_of(ref, struct resv_map, refs); 885 struct list_head *head = &resv_map->region_cache; 886 struct file_region *rg, *trg; 887 888 /* Clear out any active regions before we release the map. */ 889 region_del(resv_map, 0, LONG_MAX); 890 891 /* ... and any entries left in the cache */ 892 list_for_each_entry_safe(rg, trg, head, link) { 893 list_del(&rg->link); 894 kfree(rg); 895 } 896 897 VM_BUG_ON(resv_map->adds_in_progress); 898 899 kfree(resv_map); 900 } 901 902 static inline struct resv_map *inode_resv_map(struct inode *inode) 903 { 904 /* 905 * At inode evict time, i_mapping may not point to the original 906 * address space within the inode. This original address space 907 * contains the pointer to the resv_map. So, always use the 908 * address space embedded within the inode. 909 * The VERY common case is inode->mapping == &inode->i_data but, 910 * this may not be true for device special inodes. 911 */ 912 return (struct resv_map *)(&inode->i_data)->private_data; 913 } 914 915 static struct resv_map *vma_resv_map(struct vm_area_struct *vma) 916 { 917 VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma); 918 if (vma->vm_flags & VM_MAYSHARE) { 919 struct address_space *mapping = vma->vm_file->f_mapping; 920 struct inode *inode = mapping->host; 921 922 return inode_resv_map(inode); 923 924 } else { 925 return (struct resv_map *)(get_vma_private_data(vma) & 926 ~HPAGE_RESV_MASK); 927 } 928 } 929 930 static void set_vma_resv_map(struct vm_area_struct *vma, struct resv_map *map) 931 { 932 VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma); 933 VM_BUG_ON_VMA(vma->vm_flags & VM_MAYSHARE, vma); 934 935 set_vma_private_data(vma, (get_vma_private_data(vma) & 936 HPAGE_RESV_MASK) | (unsigned long)map); 937 } 938 939 static void set_vma_resv_flags(struct vm_area_struct *vma, unsigned long flags) 940 { 941 VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma); 942 VM_BUG_ON_VMA(vma->vm_flags & VM_MAYSHARE, vma); 943 944 set_vma_private_data(vma, get_vma_private_data(vma) | flags); 945 } 946 947 static int is_vma_resv_set(struct vm_area_struct *vma, unsigned long flag) 948 { 949 VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma); 950 951 return (get_vma_private_data(vma) & flag) != 0; 952 } 953 954 /* Reset counters to 0 and clear all HPAGE_RESV_* flags */ 955 void reset_vma_resv_huge_pages(struct vm_area_struct *vma) 956 { 957 VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma); 958 if (!(vma->vm_flags & VM_MAYSHARE)) 959 vma->vm_private_data = (void *)0; 960 } 961 962 /* Returns true if the VMA has associated reserve pages */ 963 static bool vma_has_reserves(struct vm_area_struct *vma, long chg) 964 { 965 if (vma->vm_flags & VM_NORESERVE) { 966 /* 967 * This address is already reserved by other process(chg == 0), 968 * so, we should decrement reserved count. Without decrementing, 969 * reserve count remains after releasing inode, because this 970 * allocated page will go into page cache and is regarded as 971 * coming from reserved pool in releasing step. Currently, we 972 * don't have any other solution to deal with this situation 973 * properly, so add work-around here. 974 */ 975 if (vma->vm_flags & VM_MAYSHARE && chg == 0) 976 return true; 977 else 978 return false; 979 } 980 981 /* Shared mappings always use reserves */ 982 if (vma->vm_flags & VM_MAYSHARE) { 983 /* 984 * We know VM_NORESERVE is not set. Therefore, there SHOULD 985 * be a region map for all pages. The only situation where 986 * there is no region map is if a hole was punched via 987 * fallocate. In this case, there really are no reserves to 988 * use. This situation is indicated if chg != 0. 989 */ 990 if (chg) 991 return false; 992 else 993 return true; 994 } 995 996 /* 997 * Only the process that called mmap() has reserves for 998 * private mappings. 999 */ 1000 if (is_vma_resv_set(vma, HPAGE_RESV_OWNER)) { 1001 /* 1002 * Like the shared case above, a hole punch or truncate 1003 * could have been performed on the private mapping. 1004 * Examine the value of chg to determine if reserves 1005 * actually exist or were previously consumed. 1006 * Very Subtle - The value of chg comes from a previous 1007 * call to vma_needs_reserves(). The reserve map for 1008 * private mappings has different (opposite) semantics 1009 * than that of shared mappings. vma_needs_reserves() 1010 * has already taken this difference in semantics into 1011 * account. Therefore, the meaning of chg is the same 1012 * as in the shared case above. Code could easily be 1013 * combined, but keeping it separate draws attention to 1014 * subtle differences. 1015 */ 1016 if (chg) 1017 return false; 1018 else 1019 return true; 1020 } 1021 1022 return false; 1023 } 1024 1025 static void enqueue_huge_page(struct hstate *h, struct page *page) 1026 { 1027 int nid = page_to_nid(page); 1028 list_move(&page->lru, &h->hugepage_freelists[nid]); 1029 h->free_huge_pages++; 1030 h->free_huge_pages_node[nid]++; 1031 } 1032 1033 static struct page *dequeue_huge_page_node_exact(struct hstate *h, int nid) 1034 { 1035 struct page *page; 1036 bool nocma = !!(current->flags & PF_MEMALLOC_NOCMA); 1037 1038 list_for_each_entry(page, &h->hugepage_freelists[nid], lru) { 1039 if (nocma && is_migrate_cma_page(page)) 1040 continue; 1041 1042 if (PageHWPoison(page)) 1043 continue; 1044 1045 list_move(&page->lru, &h->hugepage_activelist); 1046 set_page_refcounted(page); 1047 h->free_huge_pages--; 1048 h->free_huge_pages_node[nid]--; 1049 return page; 1050 } 1051 1052 return NULL; 1053 } 1054 1055 static struct page *dequeue_huge_page_nodemask(struct hstate *h, gfp_t gfp_mask, int nid, 1056 nodemask_t *nmask) 1057 { 1058 unsigned int cpuset_mems_cookie; 1059 struct zonelist *zonelist; 1060 struct zone *zone; 1061 struct zoneref *z; 1062 int node = NUMA_NO_NODE; 1063 1064 zonelist = node_zonelist(nid, gfp_mask); 1065 1066 retry_cpuset: 1067 cpuset_mems_cookie = read_mems_allowed_begin(); 1068 for_each_zone_zonelist_nodemask(zone, z, zonelist, gfp_zone(gfp_mask), nmask) { 1069 struct page *page; 1070 1071 if (!cpuset_zone_allowed(zone, gfp_mask)) 1072 continue; 1073 /* 1074 * no need to ask again on the same node. Pool is node rather than 1075 * zone aware 1076 */ 1077 if (zone_to_nid(zone) == node) 1078 continue; 1079 node = zone_to_nid(zone); 1080 1081 page = dequeue_huge_page_node_exact(h, node); 1082 if (page) 1083 return page; 1084 } 1085 if (unlikely(read_mems_allowed_retry(cpuset_mems_cookie))) 1086 goto retry_cpuset; 1087 1088 return NULL; 1089 } 1090 1091 static struct page *dequeue_huge_page_vma(struct hstate *h, 1092 struct vm_area_struct *vma, 1093 unsigned long address, int avoid_reserve, 1094 long chg) 1095 { 1096 struct page *page; 1097 struct mempolicy *mpol; 1098 gfp_t gfp_mask; 1099 nodemask_t *nodemask; 1100 int nid; 1101 1102 /* 1103 * A child process with MAP_PRIVATE mappings created by their parent 1104 * have no page reserves. This check ensures that reservations are 1105 * not "stolen". The child may still get SIGKILLed 1106 */ 1107 if (!vma_has_reserves(vma, chg) && 1108 h->free_huge_pages - h->resv_huge_pages == 0) 1109 goto err; 1110 1111 /* If reserves cannot be used, ensure enough pages are in the pool */ 1112 if (avoid_reserve && h->free_huge_pages - h->resv_huge_pages == 0) 1113 goto err; 1114 1115 gfp_mask = htlb_alloc_mask(h); 1116 nid = huge_node(vma, address, gfp_mask, &mpol, &nodemask); 1117 page = dequeue_huge_page_nodemask(h, gfp_mask, nid, nodemask); 1118 if (page && !avoid_reserve && vma_has_reserves(vma, chg)) { 1119 SetPagePrivate(page); 1120 h->resv_huge_pages--; 1121 } 1122 1123 mpol_cond_put(mpol); 1124 return page; 1125 1126 err: 1127 return NULL; 1128 } 1129 1130 /* 1131 * common helper functions for hstate_next_node_to_{alloc|free}. 1132 * We may have allocated or freed a huge page based on a different 1133 * nodes_allowed previously, so h->next_node_to_{alloc|free} might 1134 * be outside of *nodes_allowed. Ensure that we use an allowed 1135 * node for alloc or free. 1136 */ 1137 static int next_node_allowed(int nid, nodemask_t *nodes_allowed) 1138 { 1139 nid = next_node_in(nid, *nodes_allowed); 1140 VM_BUG_ON(nid >= MAX_NUMNODES); 1141 1142 return nid; 1143 } 1144 1145 static int get_valid_node_allowed(int nid, nodemask_t *nodes_allowed) 1146 { 1147 if (!node_isset(nid, *nodes_allowed)) 1148 nid = next_node_allowed(nid, nodes_allowed); 1149 return nid; 1150 } 1151 1152 /* 1153 * returns the previously saved node ["this node"] from which to 1154 * allocate a persistent huge page for the pool and advance the 1155 * next node from which to allocate, handling wrap at end of node 1156 * mask. 1157 */ 1158 static int hstate_next_node_to_alloc(struct hstate *h, 1159 nodemask_t *nodes_allowed) 1160 { 1161 int nid; 1162 1163 VM_BUG_ON(!nodes_allowed); 1164 1165 nid = get_valid_node_allowed(h->next_nid_to_alloc, nodes_allowed); 1166 h->next_nid_to_alloc = next_node_allowed(nid, nodes_allowed); 1167 1168 return nid; 1169 } 1170 1171 /* 1172 * helper for free_pool_huge_page() - return the previously saved 1173 * node ["this node"] from which to free a huge page. Advance the 1174 * next node id whether or not we find a free huge page to free so 1175 * that the next attempt to free addresses the next node. 1176 */ 1177 static int hstate_next_node_to_free(struct hstate *h, nodemask_t *nodes_allowed) 1178 { 1179 int nid; 1180 1181 VM_BUG_ON(!nodes_allowed); 1182 1183 nid = get_valid_node_allowed(h->next_nid_to_free, nodes_allowed); 1184 h->next_nid_to_free = next_node_allowed(nid, nodes_allowed); 1185 1186 return nid; 1187 } 1188 1189 #define for_each_node_mask_to_alloc(hs, nr_nodes, node, mask) \ 1190 for (nr_nodes = nodes_weight(*mask); \ 1191 nr_nodes > 0 && \ 1192 ((node = hstate_next_node_to_alloc(hs, mask)) || 1); \ 1193 nr_nodes--) 1194 1195 #define for_each_node_mask_to_free(hs, nr_nodes, node, mask) \ 1196 for (nr_nodes = nodes_weight(*mask); \ 1197 nr_nodes > 0 && \ 1198 ((node = hstate_next_node_to_free(hs, mask)) || 1); \ 1199 nr_nodes--) 1200 1201 #ifdef CONFIG_ARCH_HAS_GIGANTIC_PAGE 1202 static void destroy_compound_gigantic_page(struct page *page, 1203 unsigned int order) 1204 { 1205 int i; 1206 int nr_pages = 1 << order; 1207 struct page *p = page + 1; 1208 1209 atomic_set(compound_mapcount_ptr(page), 0); 1210 if (hpage_pincount_available(page)) 1211 atomic_set(compound_pincount_ptr(page), 0); 1212 1213 for (i = 1; i < nr_pages; i++, p = mem_map_next(p, page, i)) { 1214 clear_compound_head(p); 1215 set_page_refcounted(p); 1216 } 1217 1218 set_compound_order(page, 0); 1219 page[1].compound_nr = 0; 1220 __ClearPageHead(page); 1221 } 1222 1223 static void free_gigantic_page(struct page *page, unsigned int order) 1224 { 1225 /* 1226 * If the page isn't allocated using the cma allocator, 1227 * cma_release() returns false. 1228 */ 1229 #ifdef CONFIG_CMA 1230 if (cma_release(hugetlb_cma[page_to_nid(page)], page, 1 << order)) 1231 return; 1232 #endif 1233 1234 free_contig_range(page_to_pfn(page), 1 << order); 1235 } 1236 1237 #ifdef CONFIG_CONTIG_ALLOC 1238 static struct page *alloc_gigantic_page(struct hstate *h, gfp_t gfp_mask, 1239 int nid, nodemask_t *nodemask) 1240 { 1241 unsigned long nr_pages = 1UL << huge_page_order(h); 1242 if (nid == NUMA_NO_NODE) 1243 nid = numa_mem_id(); 1244 1245 #ifdef CONFIG_CMA 1246 { 1247 struct page *page; 1248 int node; 1249 1250 if (hugetlb_cma[nid]) { 1251 page = cma_alloc(hugetlb_cma[nid], nr_pages, 1252 huge_page_order(h), true); 1253 if (page) 1254 return page; 1255 } 1256 1257 if (!(gfp_mask & __GFP_THISNODE)) { 1258 for_each_node_mask(node, *nodemask) { 1259 if (node == nid || !hugetlb_cma[node]) 1260 continue; 1261 1262 page = cma_alloc(hugetlb_cma[node], nr_pages, 1263 huge_page_order(h), true); 1264 if (page) 1265 return page; 1266 } 1267 } 1268 } 1269 #endif 1270 1271 return alloc_contig_pages(nr_pages, gfp_mask, nid, nodemask); 1272 } 1273 1274 static void prep_new_huge_page(struct hstate *h, struct page *page, int nid); 1275 static void prep_compound_gigantic_page(struct page *page, unsigned int order); 1276 #else /* !CONFIG_CONTIG_ALLOC */ 1277 static struct page *alloc_gigantic_page(struct hstate *h, gfp_t gfp_mask, 1278 int nid, nodemask_t *nodemask) 1279 { 1280 return NULL; 1281 } 1282 #endif /* CONFIG_CONTIG_ALLOC */ 1283 1284 #else /* !CONFIG_ARCH_HAS_GIGANTIC_PAGE */ 1285 static struct page *alloc_gigantic_page(struct hstate *h, gfp_t gfp_mask, 1286 int nid, nodemask_t *nodemask) 1287 { 1288 return NULL; 1289 } 1290 static inline void free_gigantic_page(struct page *page, unsigned int order) { } 1291 static inline void destroy_compound_gigantic_page(struct page *page, 1292 unsigned int order) { } 1293 #endif 1294 1295 static void update_and_free_page(struct hstate *h, struct page *page) 1296 { 1297 int i; 1298 1299 if (hstate_is_gigantic(h) && !gigantic_page_runtime_supported()) 1300 return; 1301 1302 h->nr_huge_pages--; 1303 h->nr_huge_pages_node[page_to_nid(page)]--; 1304 for (i = 0; i < pages_per_huge_page(h); i++) { 1305 page[i].flags &= ~(1 << PG_locked | 1 << PG_error | 1306 1 << PG_referenced | 1 << PG_dirty | 1307 1 << PG_active | 1 << PG_private | 1308 1 << PG_writeback); 1309 } 1310 VM_BUG_ON_PAGE(hugetlb_cgroup_from_page(page), page); 1311 VM_BUG_ON_PAGE(hugetlb_cgroup_from_page_rsvd(page), page); 1312 set_compound_page_dtor(page, NULL_COMPOUND_DTOR); 1313 set_page_refcounted(page); 1314 if (hstate_is_gigantic(h)) { 1315 /* 1316 * Temporarily drop the hugetlb_lock, because 1317 * we might block in free_gigantic_page(). 1318 */ 1319 spin_unlock(&hugetlb_lock); 1320 destroy_compound_gigantic_page(page, huge_page_order(h)); 1321 free_gigantic_page(page, huge_page_order(h)); 1322 spin_lock(&hugetlb_lock); 1323 } else { 1324 __free_pages(page, huge_page_order(h)); 1325 } 1326 } 1327 1328 struct hstate *size_to_hstate(unsigned long size) 1329 { 1330 struct hstate *h; 1331 1332 for_each_hstate(h) { 1333 if (huge_page_size(h) == size) 1334 return h; 1335 } 1336 return NULL; 1337 } 1338 1339 /* 1340 * Test to determine whether the hugepage is "active/in-use" (i.e. being linked 1341 * to hstate->hugepage_activelist.) 1342 * 1343 * This function can be called for tail pages, but never returns true for them. 1344 */ 1345 bool page_huge_active(struct page *page) 1346 { 1347 VM_BUG_ON_PAGE(!PageHuge(page), page); 1348 return PageHead(page) && PagePrivate(&page[1]); 1349 } 1350 1351 /* never called for tail page */ 1352 static void set_page_huge_active(struct page *page) 1353 { 1354 VM_BUG_ON_PAGE(!PageHeadHuge(page), page); 1355 SetPagePrivate(&page[1]); 1356 } 1357 1358 static void clear_page_huge_active(struct page *page) 1359 { 1360 VM_BUG_ON_PAGE(!PageHeadHuge(page), page); 1361 ClearPagePrivate(&page[1]); 1362 } 1363 1364 /* 1365 * Internal hugetlb specific page flag. Do not use outside of the hugetlb 1366 * code 1367 */ 1368 static inline bool PageHugeTemporary(struct page *page) 1369 { 1370 if (!PageHuge(page)) 1371 return false; 1372 1373 return (unsigned long)page[2].mapping == -1U; 1374 } 1375 1376 static inline void SetPageHugeTemporary(struct page *page) 1377 { 1378 page[2].mapping = (void *)-1U; 1379 } 1380 1381 static inline void ClearPageHugeTemporary(struct page *page) 1382 { 1383 page[2].mapping = NULL; 1384 } 1385 1386 static void __free_huge_page(struct page *page) 1387 { 1388 /* 1389 * Can't pass hstate in here because it is called from the 1390 * compound page destructor. 1391 */ 1392 struct hstate *h = page_hstate(page); 1393 int nid = page_to_nid(page); 1394 struct hugepage_subpool *spool = 1395 (struct hugepage_subpool *)page_private(page); 1396 bool restore_reserve; 1397 1398 VM_BUG_ON_PAGE(page_count(page), page); 1399 VM_BUG_ON_PAGE(page_mapcount(page), page); 1400 1401 set_page_private(page, 0); 1402 page->mapping = NULL; 1403 restore_reserve = PagePrivate(page); 1404 ClearPagePrivate(page); 1405 1406 /* 1407 * If PagePrivate() was set on page, page allocation consumed a 1408 * reservation. If the page was associated with a subpool, there 1409 * would have been a page reserved in the subpool before allocation 1410 * via hugepage_subpool_get_pages(). Since we are 'restoring' the 1411 * reservtion, do not call hugepage_subpool_put_pages() as this will 1412 * remove the reserved page from the subpool. 1413 */ 1414 if (!restore_reserve) { 1415 /* 1416 * A return code of zero implies that the subpool will be 1417 * under its minimum size if the reservation is not restored 1418 * after page is free. Therefore, force restore_reserve 1419 * operation. 1420 */ 1421 if (hugepage_subpool_put_pages(spool, 1) == 0) 1422 restore_reserve = true; 1423 } 1424 1425 spin_lock(&hugetlb_lock); 1426 clear_page_huge_active(page); 1427 hugetlb_cgroup_uncharge_page(hstate_index(h), 1428 pages_per_huge_page(h), page); 1429 hugetlb_cgroup_uncharge_page_rsvd(hstate_index(h), 1430 pages_per_huge_page(h), page); 1431 if (restore_reserve) 1432 h->resv_huge_pages++; 1433 1434 if (PageHugeTemporary(page)) { 1435 list_del(&page->lru); 1436 ClearPageHugeTemporary(page); 1437 update_and_free_page(h, page); 1438 } else if (h->surplus_huge_pages_node[nid]) { 1439 /* remove the page from active list */ 1440 list_del(&page->lru); 1441 update_and_free_page(h, page); 1442 h->surplus_huge_pages--; 1443 h->surplus_huge_pages_node[nid]--; 1444 } else { 1445 arch_clear_hugepage_flags(page); 1446 enqueue_huge_page(h, page); 1447 } 1448 spin_unlock(&hugetlb_lock); 1449 } 1450 1451 /* 1452 * As free_huge_page() can be called from a non-task context, we have 1453 * to defer the actual freeing in a workqueue to prevent potential 1454 * hugetlb_lock deadlock. 1455 * 1456 * free_hpage_workfn() locklessly retrieves the linked list of pages to 1457 * be freed and frees them one-by-one. As the page->mapping pointer is 1458 * going to be cleared in __free_huge_page() anyway, it is reused as the 1459 * llist_node structure of a lockless linked list of huge pages to be freed. 1460 */ 1461 static LLIST_HEAD(hpage_freelist); 1462 1463 static void free_hpage_workfn(struct work_struct *work) 1464 { 1465 struct llist_node *node; 1466 struct page *page; 1467 1468 node = llist_del_all(&hpage_freelist); 1469 1470 while (node) { 1471 page = container_of((struct address_space **)node, 1472 struct page, mapping); 1473 node = node->next; 1474 __free_huge_page(page); 1475 } 1476 } 1477 static DECLARE_WORK(free_hpage_work, free_hpage_workfn); 1478 1479 void free_huge_page(struct page *page) 1480 { 1481 /* 1482 * Defer freeing if in non-task context to avoid hugetlb_lock deadlock. 1483 */ 1484 if (!in_task()) { 1485 /* 1486 * Only call schedule_work() if hpage_freelist is previously 1487 * empty. Otherwise, schedule_work() had been called but the 1488 * workfn hasn't retrieved the list yet. 1489 */ 1490 if (llist_add((struct llist_node *)&page->mapping, 1491 &hpage_freelist)) 1492 schedule_work(&free_hpage_work); 1493 return; 1494 } 1495 1496 __free_huge_page(page); 1497 } 1498 1499 static void prep_new_huge_page(struct hstate *h, struct page *page, int nid) 1500 { 1501 INIT_LIST_HEAD(&page->lru); 1502 set_compound_page_dtor(page, HUGETLB_PAGE_DTOR); 1503 set_hugetlb_cgroup(page, NULL); 1504 set_hugetlb_cgroup_rsvd(page, NULL); 1505 spin_lock(&hugetlb_lock); 1506 h->nr_huge_pages++; 1507 h->nr_huge_pages_node[nid]++; 1508 spin_unlock(&hugetlb_lock); 1509 } 1510 1511 static void prep_compound_gigantic_page(struct page *page, unsigned int order) 1512 { 1513 int i; 1514 int nr_pages = 1 << order; 1515 struct page *p = page + 1; 1516 1517 /* we rely on prep_new_huge_page to set the destructor */ 1518 set_compound_order(page, order); 1519 __ClearPageReserved(page); 1520 __SetPageHead(page); 1521 for (i = 1; i < nr_pages; i++, p = mem_map_next(p, page, i)) { 1522 /* 1523 * For gigantic hugepages allocated through bootmem at 1524 * boot, it's safer to be consistent with the not-gigantic 1525 * hugepages and clear the PG_reserved bit from all tail pages 1526 * too. Otherwise drivers using get_user_pages() to access tail 1527 * pages may get the reference counting wrong if they see 1528 * PG_reserved set on a tail page (despite the head page not 1529 * having PG_reserved set). Enforcing this consistency between 1530 * head and tail pages allows drivers to optimize away a check 1531 * on the head page when they need know if put_page() is needed 1532 * after get_user_pages(). 1533 */ 1534 __ClearPageReserved(p); 1535 set_page_count(p, 0); 1536 set_compound_head(p, page); 1537 } 1538 atomic_set(compound_mapcount_ptr(page), -1); 1539 1540 if (hpage_pincount_available(page)) 1541 atomic_set(compound_pincount_ptr(page), 0); 1542 } 1543 1544 /* 1545 * PageHuge() only returns true for hugetlbfs pages, but not for normal or 1546 * transparent huge pages. See the PageTransHuge() documentation for more 1547 * details. 1548 */ 1549 int PageHuge(struct page *page) 1550 { 1551 if (!PageCompound(page)) 1552 return 0; 1553 1554 page = compound_head(page); 1555 return page[1].compound_dtor == HUGETLB_PAGE_DTOR; 1556 } 1557 EXPORT_SYMBOL_GPL(PageHuge); 1558 1559 /* 1560 * PageHeadHuge() only returns true for hugetlbfs head page, but not for 1561 * normal or transparent huge pages. 1562 */ 1563 int PageHeadHuge(struct page *page_head) 1564 { 1565 if (!PageHead(page_head)) 1566 return 0; 1567 1568 return page_head[1].compound_dtor == HUGETLB_PAGE_DTOR; 1569 } 1570 1571 /* 1572 * Find and lock address space (mapping) in write mode. 1573 * 1574 * Upon entry, the page is locked which means that page_mapping() is 1575 * stable. Due to locking order, we can only trylock_write. If we can 1576 * not get the lock, simply return NULL to caller. 1577 */ 1578 struct address_space *hugetlb_page_mapping_lock_write(struct page *hpage) 1579 { 1580 struct address_space *mapping = page_mapping(hpage); 1581 1582 if (!mapping) 1583 return mapping; 1584 1585 if (i_mmap_trylock_write(mapping)) 1586 return mapping; 1587 1588 return NULL; 1589 } 1590 1591 pgoff_t __basepage_index(struct page *page) 1592 { 1593 struct page *page_head = compound_head(page); 1594 pgoff_t index = page_index(page_head); 1595 unsigned long compound_idx; 1596 1597 if (!PageHuge(page_head)) 1598 return page_index(page); 1599 1600 if (compound_order(page_head) >= MAX_ORDER) 1601 compound_idx = page_to_pfn(page) - page_to_pfn(page_head); 1602 else 1603 compound_idx = page - page_head; 1604 1605 return (index << compound_order(page_head)) + compound_idx; 1606 } 1607 1608 static struct page *alloc_buddy_huge_page(struct hstate *h, 1609 gfp_t gfp_mask, int nid, nodemask_t *nmask, 1610 nodemask_t *node_alloc_noretry) 1611 { 1612 int order = huge_page_order(h); 1613 struct page *page; 1614 bool alloc_try_hard = true; 1615 1616 /* 1617 * By default we always try hard to allocate the page with 1618 * __GFP_RETRY_MAYFAIL flag. However, if we are allocating pages in 1619 * a loop (to adjust global huge page counts) and previous allocation 1620 * failed, do not continue to try hard on the same node. Use the 1621 * node_alloc_noretry bitmap to manage this state information. 1622 */ 1623 if (node_alloc_noretry && node_isset(nid, *node_alloc_noretry)) 1624 alloc_try_hard = false; 1625 gfp_mask |= __GFP_COMP|__GFP_NOWARN; 1626 if (alloc_try_hard) 1627 gfp_mask |= __GFP_RETRY_MAYFAIL; 1628 if (nid == NUMA_NO_NODE) 1629 nid = numa_mem_id(); 1630 page = __alloc_pages_nodemask(gfp_mask, order, nid, nmask); 1631 if (page) 1632 __count_vm_event(HTLB_BUDDY_PGALLOC); 1633 else 1634 __count_vm_event(HTLB_BUDDY_PGALLOC_FAIL); 1635 1636 /* 1637 * If we did not specify __GFP_RETRY_MAYFAIL, but still got a page this 1638 * indicates an overall state change. Clear bit so that we resume 1639 * normal 'try hard' allocations. 1640 */ 1641 if (node_alloc_noretry && page && !alloc_try_hard) 1642 node_clear(nid, *node_alloc_noretry); 1643 1644 /* 1645 * If we tried hard to get a page but failed, set bit so that 1646 * subsequent attempts will not try as hard until there is an 1647 * overall state change. 1648 */ 1649 if (node_alloc_noretry && !page && alloc_try_hard) 1650 node_set(nid, *node_alloc_noretry); 1651 1652 return page; 1653 } 1654 1655 /* 1656 * Common helper to allocate a fresh hugetlb page. All specific allocators 1657 * should use this function to get new hugetlb pages 1658 */ 1659 static struct page *alloc_fresh_huge_page(struct hstate *h, 1660 gfp_t gfp_mask, int nid, nodemask_t *nmask, 1661 nodemask_t *node_alloc_noretry) 1662 { 1663 struct page *page; 1664 1665 if (hstate_is_gigantic(h)) 1666 page = alloc_gigantic_page(h, gfp_mask, nid, nmask); 1667 else 1668 page = alloc_buddy_huge_page(h, gfp_mask, 1669 nid, nmask, node_alloc_noretry); 1670 if (!page) 1671 return NULL; 1672 1673 if (hstate_is_gigantic(h)) 1674 prep_compound_gigantic_page(page, huge_page_order(h)); 1675 prep_new_huge_page(h, page, page_to_nid(page)); 1676 1677 return page; 1678 } 1679 1680 /* 1681 * Allocates a fresh page to the hugetlb allocator pool in the node interleaved 1682 * manner. 1683 */ 1684 static int alloc_pool_huge_page(struct hstate *h, nodemask_t *nodes_allowed, 1685 nodemask_t *node_alloc_noretry) 1686 { 1687 struct page *page; 1688 int nr_nodes, node; 1689 gfp_t gfp_mask = htlb_alloc_mask(h) | __GFP_THISNODE; 1690 1691 for_each_node_mask_to_alloc(h, nr_nodes, node, nodes_allowed) { 1692 page = alloc_fresh_huge_page(h, gfp_mask, node, nodes_allowed, 1693 node_alloc_noretry); 1694 if (page) 1695 break; 1696 } 1697 1698 if (!page) 1699 return 0; 1700 1701 put_page(page); /* free it into the hugepage allocator */ 1702 1703 return 1; 1704 } 1705 1706 /* 1707 * Free huge page from pool from next node to free. 1708 * Attempt to keep persistent huge pages more or less 1709 * balanced over allowed nodes. 1710 * Called with hugetlb_lock locked. 1711 */ 1712 static int free_pool_huge_page(struct hstate *h, nodemask_t *nodes_allowed, 1713 bool acct_surplus) 1714 { 1715 int nr_nodes, node; 1716 int ret = 0; 1717 1718 for_each_node_mask_to_free(h, nr_nodes, node, nodes_allowed) { 1719 /* 1720 * If we're returning unused surplus pages, only examine 1721 * nodes with surplus pages. 1722 */ 1723 if ((!acct_surplus || h->surplus_huge_pages_node[node]) && 1724 !list_empty(&h->hugepage_freelists[node])) { 1725 struct page *page = 1726 list_entry(h->hugepage_freelists[node].next, 1727 struct page, lru); 1728 list_del(&page->lru); 1729 h->free_huge_pages--; 1730 h->free_huge_pages_node[node]--; 1731 if (acct_surplus) { 1732 h->surplus_huge_pages--; 1733 h->surplus_huge_pages_node[node]--; 1734 } 1735 update_and_free_page(h, page); 1736 ret = 1; 1737 break; 1738 } 1739 } 1740 1741 return ret; 1742 } 1743 1744 /* 1745 * Dissolve a given free hugepage into free buddy pages. This function does 1746 * nothing for in-use hugepages and non-hugepages. 1747 * This function returns values like below: 1748 * 1749 * -EBUSY: failed to dissolved free hugepages or the hugepage is in-use 1750 * (allocated or reserved.) 1751 * 0: successfully dissolved free hugepages or the page is not a 1752 * hugepage (considered as already dissolved) 1753 */ 1754 int dissolve_free_huge_page(struct page *page) 1755 { 1756 int rc = -EBUSY; 1757 1758 /* Not to disrupt normal path by vainly holding hugetlb_lock */ 1759 if (!PageHuge(page)) 1760 return 0; 1761 1762 spin_lock(&hugetlb_lock); 1763 if (!PageHuge(page)) { 1764 rc = 0; 1765 goto out; 1766 } 1767 1768 if (!page_count(page)) { 1769 struct page *head = compound_head(page); 1770 struct hstate *h = page_hstate(head); 1771 int nid = page_to_nid(head); 1772 if (h->free_huge_pages - h->resv_huge_pages == 0) 1773 goto out; 1774 /* 1775 * Move PageHWPoison flag from head page to the raw error page, 1776 * which makes any subpages rather than the error page reusable. 1777 */ 1778 if (PageHWPoison(head) && page != head) { 1779 SetPageHWPoison(page); 1780 ClearPageHWPoison(head); 1781 } 1782 list_del(&head->lru); 1783 h->free_huge_pages--; 1784 h->free_huge_pages_node[nid]--; 1785 h->max_huge_pages--; 1786 update_and_free_page(h, head); 1787 rc = 0; 1788 } 1789 out: 1790 spin_unlock(&hugetlb_lock); 1791 return rc; 1792 } 1793 1794 /* 1795 * Dissolve free hugepages in a given pfn range. Used by memory hotplug to 1796 * make specified memory blocks removable from the system. 1797 * Note that this will dissolve a free gigantic hugepage completely, if any 1798 * part of it lies within the given range. 1799 * Also note that if dissolve_free_huge_page() returns with an error, all 1800 * free hugepages that were dissolved before that error are lost. 1801 */ 1802 int dissolve_free_huge_pages(unsigned long start_pfn, unsigned long end_pfn) 1803 { 1804 unsigned long pfn; 1805 struct page *page; 1806 int rc = 0; 1807 1808 if (!hugepages_supported()) 1809 return rc; 1810 1811 for (pfn = start_pfn; pfn < end_pfn; pfn += 1 << minimum_order) { 1812 page = pfn_to_page(pfn); 1813 rc = dissolve_free_huge_page(page); 1814 if (rc) 1815 break; 1816 } 1817 1818 return rc; 1819 } 1820 1821 /* 1822 * Allocates a fresh surplus page from the page allocator. 1823 */ 1824 static struct page *alloc_surplus_huge_page(struct hstate *h, gfp_t gfp_mask, 1825 int nid, nodemask_t *nmask) 1826 { 1827 struct page *page = NULL; 1828 1829 if (hstate_is_gigantic(h)) 1830 return NULL; 1831 1832 spin_lock(&hugetlb_lock); 1833 if (h->surplus_huge_pages >= h->nr_overcommit_huge_pages) 1834 goto out_unlock; 1835 spin_unlock(&hugetlb_lock); 1836 1837 page = alloc_fresh_huge_page(h, gfp_mask, nid, nmask, NULL); 1838 if (!page) 1839 return NULL; 1840 1841 spin_lock(&hugetlb_lock); 1842 /* 1843 * We could have raced with the pool size change. 1844 * Double check that and simply deallocate the new page 1845 * if we would end up overcommiting the surpluses. Abuse 1846 * temporary page to workaround the nasty free_huge_page 1847 * codeflow 1848 */ 1849 if (h->surplus_huge_pages >= h->nr_overcommit_huge_pages) { 1850 SetPageHugeTemporary(page); 1851 spin_unlock(&hugetlb_lock); 1852 put_page(page); 1853 return NULL; 1854 } else { 1855 h->surplus_huge_pages++; 1856 h->surplus_huge_pages_node[page_to_nid(page)]++; 1857 } 1858 1859 out_unlock: 1860 spin_unlock(&hugetlb_lock); 1861 1862 return page; 1863 } 1864 1865 static struct page *alloc_migrate_huge_page(struct hstate *h, gfp_t gfp_mask, 1866 int nid, nodemask_t *nmask) 1867 { 1868 struct page *page; 1869 1870 if (hstate_is_gigantic(h)) 1871 return NULL; 1872 1873 page = alloc_fresh_huge_page(h, gfp_mask, nid, nmask, NULL); 1874 if (!page) 1875 return NULL; 1876 1877 /* 1878 * We do not account these pages as surplus because they are only 1879 * temporary and will be released properly on the last reference 1880 */ 1881 SetPageHugeTemporary(page); 1882 1883 return page; 1884 } 1885 1886 /* 1887 * Use the VMA's mpolicy to allocate a huge page from the buddy. 1888 */ 1889 static 1890 struct page *alloc_buddy_huge_page_with_mpol(struct hstate *h, 1891 struct vm_area_struct *vma, unsigned long addr) 1892 { 1893 struct page *page; 1894 struct mempolicy *mpol; 1895 gfp_t gfp_mask = htlb_alloc_mask(h); 1896 int nid; 1897 nodemask_t *nodemask; 1898 1899 nid = huge_node(vma, addr, gfp_mask, &mpol, &nodemask); 1900 page = alloc_surplus_huge_page(h, gfp_mask, nid, nodemask); 1901 mpol_cond_put(mpol); 1902 1903 return page; 1904 } 1905 1906 /* page migration callback function */ 1907 struct page *alloc_huge_page_nodemask(struct hstate *h, int preferred_nid, 1908 nodemask_t *nmask, gfp_t gfp_mask) 1909 { 1910 spin_lock(&hugetlb_lock); 1911 if (h->free_huge_pages - h->resv_huge_pages > 0) { 1912 struct page *page; 1913 1914 page = dequeue_huge_page_nodemask(h, gfp_mask, preferred_nid, nmask); 1915 if (page) { 1916 spin_unlock(&hugetlb_lock); 1917 return page; 1918 } 1919 } 1920 spin_unlock(&hugetlb_lock); 1921 1922 return alloc_migrate_huge_page(h, gfp_mask, preferred_nid, nmask); 1923 } 1924 1925 /* mempolicy aware migration callback */ 1926 struct page *alloc_huge_page_vma(struct hstate *h, struct vm_area_struct *vma, 1927 unsigned long address) 1928 { 1929 struct mempolicy *mpol; 1930 nodemask_t *nodemask; 1931 struct page *page; 1932 gfp_t gfp_mask; 1933 int node; 1934 1935 gfp_mask = htlb_alloc_mask(h); 1936 node = huge_node(vma, address, gfp_mask, &mpol, &nodemask); 1937 page = alloc_huge_page_nodemask(h, node, nodemask, gfp_mask); 1938 mpol_cond_put(mpol); 1939 1940 return page; 1941 } 1942 1943 /* 1944 * Increase the hugetlb pool such that it can accommodate a reservation 1945 * of size 'delta'. 1946 */ 1947 static int gather_surplus_pages(struct hstate *h, long delta) 1948 __must_hold(&hugetlb_lock) 1949 { 1950 struct list_head surplus_list; 1951 struct page *page, *tmp; 1952 int ret; 1953 long i; 1954 long needed, allocated; 1955 bool alloc_ok = true; 1956 1957 needed = (h->resv_huge_pages + delta) - h->free_huge_pages; 1958 if (needed <= 0) { 1959 h->resv_huge_pages += delta; 1960 return 0; 1961 } 1962 1963 allocated = 0; 1964 INIT_LIST_HEAD(&surplus_list); 1965 1966 ret = -ENOMEM; 1967 retry: 1968 spin_unlock(&hugetlb_lock); 1969 for (i = 0; i < needed; i++) { 1970 page = alloc_surplus_huge_page(h, htlb_alloc_mask(h), 1971 NUMA_NO_NODE, NULL); 1972 if (!page) { 1973 alloc_ok = false; 1974 break; 1975 } 1976 list_add(&page->lru, &surplus_list); 1977 cond_resched(); 1978 } 1979 allocated += i; 1980 1981 /* 1982 * After retaking hugetlb_lock, we need to recalculate 'needed' 1983 * because either resv_huge_pages or free_huge_pages may have changed. 1984 */ 1985 spin_lock(&hugetlb_lock); 1986 needed = (h->resv_huge_pages + delta) - 1987 (h->free_huge_pages + allocated); 1988 if (needed > 0) { 1989 if (alloc_ok) 1990 goto retry; 1991 /* 1992 * We were not able to allocate enough pages to 1993 * satisfy the entire reservation so we free what 1994 * we've allocated so far. 1995 */ 1996 goto free; 1997 } 1998 /* 1999 * The surplus_list now contains _at_least_ the number of extra pages 2000 * needed to accommodate the reservation. Add the appropriate number 2001 * of pages to the hugetlb pool and free the extras back to the buddy 2002 * allocator. Commit the entire reservation here to prevent another 2003 * process from stealing the pages as they are added to the pool but 2004 * before they are reserved. 2005 */ 2006 needed += allocated; 2007 h->resv_huge_pages += delta; 2008 ret = 0; 2009 2010 /* Free the needed pages to the hugetlb pool */ 2011 list_for_each_entry_safe(page, tmp, &surplus_list, lru) { 2012 if ((--needed) < 0) 2013 break; 2014 /* 2015 * This page is now managed by the hugetlb allocator and has 2016 * no users -- drop the buddy allocator's reference. 2017 */ 2018 VM_BUG_ON_PAGE(!put_page_testzero(page), page); 2019 enqueue_huge_page(h, page); 2020 } 2021 free: 2022 spin_unlock(&hugetlb_lock); 2023 2024 /* Free unnecessary surplus pages to the buddy allocator */ 2025 list_for_each_entry_safe(page, tmp, &surplus_list, lru) 2026 put_page(page); 2027 spin_lock(&hugetlb_lock); 2028 2029 return ret; 2030 } 2031 2032 /* 2033 * This routine has two main purposes: 2034 * 1) Decrement the reservation count (resv_huge_pages) by the value passed 2035 * in unused_resv_pages. This corresponds to the prior adjustments made 2036 * to the associated reservation map. 2037 * 2) Free any unused surplus pages that may have been allocated to satisfy 2038 * the reservation. As many as unused_resv_pages may be freed. 2039 * 2040 * Called with hugetlb_lock held. However, the lock could be dropped (and 2041 * reacquired) during calls to cond_resched_lock. Whenever dropping the lock, 2042 * we must make sure nobody else can claim pages we are in the process of 2043 * freeing. Do this by ensuring resv_huge_page always is greater than the 2044 * number of huge pages we plan to free when dropping the lock. 2045 */ 2046 static void return_unused_surplus_pages(struct hstate *h, 2047 unsigned long unused_resv_pages) 2048 { 2049 unsigned long nr_pages; 2050 2051 /* Cannot return gigantic pages currently */ 2052 if (hstate_is_gigantic(h)) 2053 goto out; 2054 2055 /* 2056 * Part (or even all) of the reservation could have been backed 2057 * by pre-allocated pages. Only free surplus pages. 2058 */ 2059 nr_pages = min(unused_resv_pages, h->surplus_huge_pages); 2060 2061 /* 2062 * We want to release as many surplus pages as possible, spread 2063 * evenly across all nodes with memory. Iterate across these nodes 2064 * until we can no longer free unreserved surplus pages. This occurs 2065 * when the nodes with surplus pages have no free pages. 2066 * free_pool_huge_page() will balance the freed pages across the 2067 * on-line nodes with memory and will handle the hstate accounting. 2068 * 2069 * Note that we decrement resv_huge_pages as we free the pages. If 2070 * we drop the lock, resv_huge_pages will still be sufficiently large 2071 * to cover subsequent pages we may free. 2072 */ 2073 while (nr_pages--) { 2074 h->resv_huge_pages--; 2075 unused_resv_pages--; 2076 if (!free_pool_huge_page(h, &node_states[N_MEMORY], 1)) 2077 goto out; 2078 cond_resched_lock(&hugetlb_lock); 2079 } 2080 2081 out: 2082 /* Fully uncommit the reservation */ 2083 h->resv_huge_pages -= unused_resv_pages; 2084 } 2085 2086 2087 /* 2088 * vma_needs_reservation, vma_commit_reservation and vma_end_reservation 2089 * are used by the huge page allocation routines to manage reservations. 2090 * 2091 * vma_needs_reservation is called to determine if the huge page at addr 2092 * within the vma has an associated reservation. If a reservation is 2093 * needed, the value 1 is returned. The caller is then responsible for 2094 * managing the global reservation and subpool usage counts. After 2095 * the huge page has been allocated, vma_commit_reservation is called 2096 * to add the page to the reservation map. If the page allocation fails, 2097 * the reservation must be ended instead of committed. vma_end_reservation 2098 * is called in such cases. 2099 * 2100 * In the normal case, vma_commit_reservation returns the same value 2101 * as the preceding vma_needs_reservation call. The only time this 2102 * is not the case is if a reserve map was changed between calls. It 2103 * is the responsibility of the caller to notice the difference and 2104 * take appropriate action. 2105 * 2106 * vma_add_reservation is used in error paths where a reservation must 2107 * be restored when a newly allocated huge page must be freed. It is 2108 * to be called after calling vma_needs_reservation to determine if a 2109 * reservation exists. 2110 */ 2111 enum vma_resv_mode { 2112 VMA_NEEDS_RESV, 2113 VMA_COMMIT_RESV, 2114 VMA_END_RESV, 2115 VMA_ADD_RESV, 2116 }; 2117 static long __vma_reservation_common(struct hstate *h, 2118 struct vm_area_struct *vma, unsigned long addr, 2119 enum vma_resv_mode mode) 2120 { 2121 struct resv_map *resv; 2122 pgoff_t idx; 2123 long ret; 2124 long dummy_out_regions_needed; 2125 2126 resv = vma_resv_map(vma); 2127 if (!resv) 2128 return 1; 2129 2130 idx = vma_hugecache_offset(h, vma, addr); 2131 switch (mode) { 2132 case VMA_NEEDS_RESV: 2133 ret = region_chg(resv, idx, idx + 1, &dummy_out_regions_needed); 2134 /* We assume that vma_reservation_* routines always operate on 2135 * 1 page, and that adding to resv map a 1 page entry can only 2136 * ever require 1 region. 2137 */ 2138 VM_BUG_ON(dummy_out_regions_needed != 1); 2139 break; 2140 case VMA_COMMIT_RESV: 2141 ret = region_add(resv, idx, idx + 1, 1, NULL, NULL); 2142 /* region_add calls of range 1 should never fail. */ 2143 VM_BUG_ON(ret < 0); 2144 break; 2145 case VMA_END_RESV: 2146 region_abort(resv, idx, idx + 1, 1); 2147 ret = 0; 2148 break; 2149 case VMA_ADD_RESV: 2150 if (vma->vm_flags & VM_MAYSHARE) { 2151 ret = region_add(resv, idx, idx + 1, 1, NULL, NULL); 2152 /* region_add calls of range 1 should never fail. */ 2153 VM_BUG_ON(ret < 0); 2154 } else { 2155 region_abort(resv, idx, idx + 1, 1); 2156 ret = region_del(resv, idx, idx + 1); 2157 } 2158 break; 2159 default: 2160 BUG(); 2161 } 2162 2163 if (vma->vm_flags & VM_MAYSHARE) 2164 return ret; 2165 else if (is_vma_resv_set(vma, HPAGE_RESV_OWNER) && ret >= 0) { 2166 /* 2167 * In most cases, reserves always exist for private mappings. 2168 * However, a file associated with mapping could have been 2169 * hole punched or truncated after reserves were consumed. 2170 * As subsequent fault on such a range will not use reserves. 2171 * Subtle - The reserve map for private mappings has the 2172 * opposite meaning than that of shared mappings. If NO 2173 * entry is in the reserve map, it means a reservation exists. 2174 * If an entry exists in the reserve map, it means the 2175 * reservation has already been consumed. As a result, the 2176 * return value of this routine is the opposite of the 2177 * value returned from reserve map manipulation routines above. 2178 */ 2179 if (ret) 2180 return 0; 2181 else 2182 return 1; 2183 } 2184 else 2185 return ret < 0 ? ret : 0; 2186 } 2187 2188 static long vma_needs_reservation(struct hstate *h, 2189 struct vm_area_struct *vma, unsigned long addr) 2190 { 2191 return __vma_reservation_common(h, vma, addr, VMA_NEEDS_RESV); 2192 } 2193 2194 static long vma_commit_reservation(struct hstate *h, 2195 struct vm_area_struct *vma, unsigned long addr) 2196 { 2197 return __vma_reservation_common(h, vma, addr, VMA_COMMIT_RESV); 2198 } 2199 2200 static void vma_end_reservation(struct hstate *h, 2201 struct vm_area_struct *vma, unsigned long addr) 2202 { 2203 (void)__vma_reservation_common(h, vma, addr, VMA_END_RESV); 2204 } 2205 2206 static long vma_add_reservation(struct hstate *h, 2207 struct vm_area_struct *vma, unsigned long addr) 2208 { 2209 return __vma_reservation_common(h, vma, addr, VMA_ADD_RESV); 2210 } 2211 2212 /* 2213 * This routine is called to restore a reservation on error paths. In the 2214 * specific error paths, a huge page was allocated (via alloc_huge_page) 2215 * and is about to be freed. If a reservation for the page existed, 2216 * alloc_huge_page would have consumed the reservation and set PagePrivate 2217 * in the newly allocated page. When the page is freed via free_huge_page, 2218 * the global reservation count will be incremented if PagePrivate is set. 2219 * However, free_huge_page can not adjust the reserve map. Adjust the 2220 * reserve map here to be consistent with global reserve count adjustments 2221 * to be made by free_huge_page. 2222 */ 2223 static void restore_reserve_on_error(struct hstate *h, 2224 struct vm_area_struct *vma, unsigned long address, 2225 struct page *page) 2226 { 2227 if (unlikely(PagePrivate(page))) { 2228 long rc = vma_needs_reservation(h, vma, address); 2229 2230 if (unlikely(rc < 0)) { 2231 /* 2232 * Rare out of memory condition in reserve map 2233 * manipulation. Clear PagePrivate so that 2234 * global reserve count will not be incremented 2235 * by free_huge_page. This will make it appear 2236 * as though the reservation for this page was 2237 * consumed. This may prevent the task from 2238 * faulting in the page at a later time. This 2239 * is better than inconsistent global huge page 2240 * accounting of reserve counts. 2241 */ 2242 ClearPagePrivate(page); 2243 } else if (rc) { 2244 rc = vma_add_reservation(h, vma, address); 2245 if (unlikely(rc < 0)) 2246 /* 2247 * See above comment about rare out of 2248 * memory condition. 2249 */ 2250 ClearPagePrivate(page); 2251 } else 2252 vma_end_reservation(h, vma, address); 2253 } 2254 } 2255 2256 struct page *alloc_huge_page(struct vm_area_struct *vma, 2257 unsigned long addr, int avoid_reserve) 2258 { 2259 struct hugepage_subpool *spool = subpool_vma(vma); 2260 struct hstate *h = hstate_vma(vma); 2261 struct page *page; 2262 long map_chg, map_commit; 2263 long gbl_chg; 2264 int ret, idx; 2265 struct hugetlb_cgroup *h_cg; 2266 bool deferred_reserve; 2267 2268 idx = hstate_index(h); 2269 /* 2270 * Examine the region/reserve map to determine if the process 2271 * has a reservation for the page to be allocated. A return 2272 * code of zero indicates a reservation exists (no change). 2273 */ 2274 map_chg = gbl_chg = vma_needs_reservation(h, vma, addr); 2275 if (map_chg < 0) 2276 return ERR_PTR(-ENOMEM); 2277 2278 /* 2279 * Processes that did not create the mapping will have no 2280 * reserves as indicated by the region/reserve map. Check 2281 * that the allocation will not exceed the subpool limit. 2282 * Allocations for MAP_NORESERVE mappings also need to be 2283 * checked against any subpool limit. 2284 */ 2285 if (map_chg || avoid_reserve) { 2286 gbl_chg = hugepage_subpool_get_pages(spool, 1); 2287 if (gbl_chg < 0) { 2288 vma_end_reservation(h, vma, addr); 2289 return ERR_PTR(-ENOSPC); 2290 } 2291 2292 /* 2293 * Even though there was no reservation in the region/reserve 2294 * map, there could be reservations associated with the 2295 * subpool that can be used. This would be indicated if the 2296 * return value of hugepage_subpool_get_pages() is zero. 2297 * However, if avoid_reserve is specified we still avoid even 2298 * the subpool reservations. 2299 */ 2300 if (avoid_reserve) 2301 gbl_chg = 1; 2302 } 2303 2304 /* If this allocation is not consuming a reservation, charge it now. 2305 */ 2306 deferred_reserve = map_chg || avoid_reserve || !vma_resv_map(vma); 2307 if (deferred_reserve) { 2308 ret = hugetlb_cgroup_charge_cgroup_rsvd( 2309 idx, pages_per_huge_page(h), &h_cg); 2310 if (ret) 2311 goto out_subpool_put; 2312 } 2313 2314 ret = hugetlb_cgroup_charge_cgroup(idx, pages_per_huge_page(h), &h_cg); 2315 if (ret) 2316 goto out_uncharge_cgroup_reservation; 2317 2318 spin_lock(&hugetlb_lock); 2319 /* 2320 * glb_chg is passed to indicate whether or not a page must be taken 2321 * from the global free pool (global change). gbl_chg == 0 indicates 2322 * a reservation exists for the allocation. 2323 */ 2324 page = dequeue_huge_page_vma(h, vma, addr, avoid_reserve, gbl_chg); 2325 if (!page) { 2326 spin_unlock(&hugetlb_lock); 2327 page = alloc_buddy_huge_page_with_mpol(h, vma, addr); 2328 if (!page) 2329 goto out_uncharge_cgroup; 2330 if (!avoid_reserve && vma_has_reserves(vma, gbl_chg)) { 2331 SetPagePrivate(page); 2332 h->resv_huge_pages--; 2333 } 2334 spin_lock(&hugetlb_lock); 2335 list_add(&page->lru, &h->hugepage_activelist); 2336 /* Fall through */ 2337 } 2338 hugetlb_cgroup_commit_charge(idx, pages_per_huge_page(h), h_cg, page); 2339 /* If allocation is not consuming a reservation, also store the 2340 * hugetlb_cgroup pointer on the page. 2341 */ 2342 if (deferred_reserve) { 2343 hugetlb_cgroup_commit_charge_rsvd(idx, pages_per_huge_page(h), 2344 h_cg, page); 2345 } 2346 2347 spin_unlock(&hugetlb_lock); 2348 2349 set_page_private(page, (unsigned long)spool); 2350 2351 map_commit = vma_commit_reservation(h, vma, addr); 2352 if (unlikely(map_chg > map_commit)) { 2353 /* 2354 * The page was added to the reservation map between 2355 * vma_needs_reservation and vma_commit_reservation. 2356 * This indicates a race with hugetlb_reserve_pages. 2357 * Adjust for the subpool count incremented above AND 2358 * in hugetlb_reserve_pages for the same page. Also, 2359 * the reservation count added in hugetlb_reserve_pages 2360 * no longer applies. 2361 */ 2362 long rsv_adjust; 2363 2364 rsv_adjust = hugepage_subpool_put_pages(spool, 1); 2365 hugetlb_acct_memory(h, -rsv_adjust); 2366 if (deferred_reserve) 2367 hugetlb_cgroup_uncharge_page_rsvd(hstate_index(h), 2368 pages_per_huge_page(h), page); 2369 } 2370 return page; 2371 2372 out_uncharge_cgroup: 2373 hugetlb_cgroup_uncharge_cgroup(idx, pages_per_huge_page(h), h_cg); 2374 out_uncharge_cgroup_reservation: 2375 if (deferred_reserve) 2376 hugetlb_cgroup_uncharge_cgroup_rsvd(idx, pages_per_huge_page(h), 2377 h_cg); 2378 out_subpool_put: 2379 if (map_chg || avoid_reserve) 2380 hugepage_subpool_put_pages(spool, 1); 2381 vma_end_reservation(h, vma, addr); 2382 return ERR_PTR(-ENOSPC); 2383 } 2384 2385 int alloc_bootmem_huge_page(struct hstate *h) 2386 __attribute__ ((weak, alias("__alloc_bootmem_huge_page"))); 2387 int __alloc_bootmem_huge_page(struct hstate *h) 2388 { 2389 struct huge_bootmem_page *m; 2390 int nr_nodes, node; 2391 2392 for_each_node_mask_to_alloc(h, nr_nodes, node, &node_states[N_MEMORY]) { 2393 void *addr; 2394 2395 addr = memblock_alloc_try_nid_raw( 2396 huge_page_size(h), huge_page_size(h), 2397 0, MEMBLOCK_ALLOC_ACCESSIBLE, node); 2398 if (addr) { 2399 /* 2400 * Use the beginning of the huge page to store the 2401 * huge_bootmem_page struct (until gather_bootmem 2402 * puts them into the mem_map). 2403 */ 2404 m = addr; 2405 goto found; 2406 } 2407 } 2408 return 0; 2409 2410 found: 2411 BUG_ON(!IS_ALIGNED(virt_to_phys(m), huge_page_size(h))); 2412 /* Put them into a private list first because mem_map is not up yet */ 2413 INIT_LIST_HEAD(&m->list); 2414 list_add(&m->list, &huge_boot_pages); 2415 m->hstate = h; 2416 return 1; 2417 } 2418 2419 static void __init prep_compound_huge_page(struct page *page, 2420 unsigned int order) 2421 { 2422 if (unlikely(order > (MAX_ORDER - 1))) 2423 prep_compound_gigantic_page(page, order); 2424 else 2425 prep_compound_page(page, order); 2426 } 2427 2428 /* Put bootmem huge pages into the standard lists after mem_map is up */ 2429 static void __init gather_bootmem_prealloc(void) 2430 { 2431 struct huge_bootmem_page *m; 2432 2433 list_for_each_entry(m, &huge_boot_pages, list) { 2434 struct page *page = virt_to_page(m); 2435 struct hstate *h = m->hstate; 2436 2437 WARN_ON(page_count(page) != 1); 2438 prep_compound_huge_page(page, h->order); 2439 WARN_ON(PageReserved(page)); 2440 prep_new_huge_page(h, page, page_to_nid(page)); 2441 put_page(page); /* free it into the hugepage allocator */ 2442 2443 /* 2444 * If we had gigantic hugepages allocated at boot time, we need 2445 * to restore the 'stolen' pages to totalram_pages in order to 2446 * fix confusing memory reports from free(1) and another 2447 * side-effects, like CommitLimit going negative. 2448 */ 2449 if (hstate_is_gigantic(h)) 2450 adjust_managed_page_count(page, 1 << h->order); 2451 cond_resched(); 2452 } 2453 } 2454 2455 static void __init hugetlb_hstate_alloc_pages(struct hstate *h) 2456 { 2457 unsigned long i; 2458 nodemask_t *node_alloc_noretry; 2459 2460 if (!hstate_is_gigantic(h)) { 2461 /* 2462 * Bit mask controlling how hard we retry per-node allocations. 2463 * Ignore errors as lower level routines can deal with 2464 * node_alloc_noretry == NULL. If this kmalloc fails at boot 2465 * time, we are likely in bigger trouble. 2466 */ 2467 node_alloc_noretry = kmalloc(sizeof(*node_alloc_noretry), 2468 GFP_KERNEL); 2469 } else { 2470 /* allocations done at boot time */ 2471 node_alloc_noretry = NULL; 2472 } 2473 2474 /* bit mask controlling how hard we retry per-node allocations */ 2475 if (node_alloc_noretry) 2476 nodes_clear(*node_alloc_noretry); 2477 2478 for (i = 0; i < h->max_huge_pages; ++i) { 2479 if (hstate_is_gigantic(h)) { 2480 if (hugetlb_cma_size) { 2481 pr_warn_once("HugeTLB: hugetlb_cma is enabled, skip boot time allocation\n"); 2482 break; 2483 } 2484 if (!alloc_bootmem_huge_page(h)) 2485 break; 2486 } else if (!alloc_pool_huge_page(h, 2487 &node_states[N_MEMORY], 2488 node_alloc_noretry)) 2489 break; 2490 cond_resched(); 2491 } 2492 if (i < h->max_huge_pages) { 2493 char buf[32]; 2494 2495 string_get_size(huge_page_size(h), 1, STRING_UNITS_2, buf, 32); 2496 pr_warn("HugeTLB: allocating %lu of page size %s failed. Only allocated %lu hugepages.\n", 2497 h->max_huge_pages, buf, i); 2498 h->max_huge_pages = i; 2499 } 2500 2501 kfree(node_alloc_noretry); 2502 } 2503 2504 static void __init hugetlb_init_hstates(void) 2505 { 2506 struct hstate *h; 2507 2508 for_each_hstate(h) { 2509 if (minimum_order > huge_page_order(h)) 2510 minimum_order = huge_page_order(h); 2511 2512 /* oversize hugepages were init'ed in early boot */ 2513 if (!hstate_is_gigantic(h)) 2514 hugetlb_hstate_alloc_pages(h); 2515 } 2516 VM_BUG_ON(minimum_order == UINT_MAX); 2517 } 2518 2519 static void __init report_hugepages(void) 2520 { 2521 struct hstate *h; 2522 2523 for_each_hstate(h) { 2524 char buf[32]; 2525 2526 string_get_size(huge_page_size(h), 1, STRING_UNITS_2, buf, 32); 2527 pr_info("HugeTLB registered %s page size, pre-allocated %ld pages\n", 2528 buf, h->free_huge_pages); 2529 } 2530 } 2531 2532 #ifdef CONFIG_HIGHMEM 2533 static void try_to_free_low(struct hstate *h, unsigned long count, 2534 nodemask_t *nodes_allowed) 2535 { 2536 int i; 2537 2538 if (hstate_is_gigantic(h)) 2539 return; 2540 2541 for_each_node_mask(i, *nodes_allowed) { 2542 struct page *page, *next; 2543 struct list_head *freel = &h->hugepage_freelists[i]; 2544 list_for_each_entry_safe(page, next, freel, lru) { 2545 if (count >= h->nr_huge_pages) 2546 return; 2547 if (PageHighMem(page)) 2548 continue; 2549 list_del(&page->lru); 2550 update_and_free_page(h, page); 2551 h->free_huge_pages--; 2552 h->free_huge_pages_node[page_to_nid(page)]--; 2553 } 2554 } 2555 } 2556 #else 2557 static inline void try_to_free_low(struct hstate *h, unsigned long count, 2558 nodemask_t *nodes_allowed) 2559 { 2560 } 2561 #endif 2562 2563 /* 2564 * Increment or decrement surplus_huge_pages. Keep node-specific counters 2565 * balanced by operating on them in a round-robin fashion. 2566 * Returns 1 if an adjustment was made. 2567 */ 2568 static int adjust_pool_surplus(struct hstate *h, nodemask_t *nodes_allowed, 2569 int delta) 2570 { 2571 int nr_nodes, node; 2572 2573 VM_BUG_ON(delta != -1 && delta != 1); 2574 2575 if (delta < 0) { 2576 for_each_node_mask_to_alloc(h, nr_nodes, node, nodes_allowed) { 2577 if (h->surplus_huge_pages_node[node]) 2578 goto found; 2579 } 2580 } else { 2581 for_each_node_mask_to_free(h, nr_nodes, node, nodes_allowed) { 2582 if (h->surplus_huge_pages_node[node] < 2583 h->nr_huge_pages_node[node]) 2584 goto found; 2585 } 2586 } 2587 return 0; 2588 2589 found: 2590 h->surplus_huge_pages += delta; 2591 h->surplus_huge_pages_node[node] += delta; 2592 return 1; 2593 } 2594 2595 #define persistent_huge_pages(h) (h->nr_huge_pages - h->surplus_huge_pages) 2596 static int set_max_huge_pages(struct hstate *h, unsigned long count, int nid, 2597 nodemask_t *nodes_allowed) 2598 { 2599 unsigned long min_count, ret; 2600 NODEMASK_ALLOC(nodemask_t, node_alloc_noretry, GFP_KERNEL); 2601 2602 /* 2603 * Bit mask controlling how hard we retry per-node allocations. 2604 * If we can not allocate the bit mask, do not attempt to allocate 2605 * the requested huge pages. 2606 */ 2607 if (node_alloc_noretry) 2608 nodes_clear(*node_alloc_noretry); 2609 else 2610 return -ENOMEM; 2611 2612 spin_lock(&hugetlb_lock); 2613 2614 /* 2615 * Check for a node specific request. 2616 * Changing node specific huge page count may require a corresponding 2617 * change to the global count. In any case, the passed node mask 2618 * (nodes_allowed) will restrict alloc/free to the specified node. 2619 */ 2620 if (nid != NUMA_NO_NODE) { 2621 unsigned long old_count = count; 2622 2623 count += h->nr_huge_pages - h->nr_huge_pages_node[nid]; 2624 /* 2625 * User may have specified a large count value which caused the 2626 * above calculation to overflow. In this case, they wanted 2627 * to allocate as many huge pages as possible. Set count to 2628 * largest possible value to align with their intention. 2629 */ 2630 if (count < old_count) 2631 count = ULONG_MAX; 2632 } 2633 2634 /* 2635 * Gigantic pages runtime allocation depend on the capability for large 2636 * page range allocation. 2637 * If the system does not provide this feature, return an error when 2638 * the user tries to allocate gigantic pages but let the user free the 2639 * boottime allocated gigantic pages. 2640 */ 2641 if (hstate_is_gigantic(h) && !IS_ENABLED(CONFIG_CONTIG_ALLOC)) { 2642 if (count > persistent_huge_pages(h)) { 2643 spin_unlock(&hugetlb_lock); 2644 NODEMASK_FREE(node_alloc_noretry); 2645 return -EINVAL; 2646 } 2647 /* Fall through to decrease pool */ 2648 } 2649 2650 /* 2651 * Increase the pool size 2652 * First take pages out of surplus state. Then make up the 2653 * remaining difference by allocating fresh huge pages. 2654 * 2655 * We might race with alloc_surplus_huge_page() here and be unable 2656 * to convert a surplus huge page to a normal huge page. That is 2657 * not critical, though, it just means the overall size of the 2658 * pool might be one hugepage larger than it needs to be, but 2659 * within all the constraints specified by the sysctls. 2660 */ 2661 while (h->surplus_huge_pages && count > persistent_huge_pages(h)) { 2662 if (!adjust_pool_surplus(h, nodes_allowed, -1)) 2663 break; 2664 } 2665 2666 while (count > persistent_huge_pages(h)) { 2667 /* 2668 * If this allocation races such that we no longer need the 2669 * page, free_huge_page will handle it by freeing the page 2670 * and reducing the surplus. 2671 */ 2672 spin_unlock(&hugetlb_lock); 2673 2674 /* yield cpu to avoid soft lockup */ 2675 cond_resched(); 2676 2677 ret = alloc_pool_huge_page(h, nodes_allowed, 2678 node_alloc_noretry); 2679 spin_lock(&hugetlb_lock); 2680 if (!ret) 2681 goto out; 2682 2683 /* Bail for signals. Probably ctrl-c from user */ 2684 if (signal_pending(current)) 2685 goto out; 2686 } 2687 2688 /* 2689 * Decrease the pool size 2690 * First return free pages to the buddy allocator (being careful 2691 * to keep enough around to satisfy reservations). Then place 2692 * pages into surplus state as needed so the pool will shrink 2693 * to the desired size as pages become free. 2694 * 2695 * By placing pages into the surplus state independent of the 2696 * overcommit value, we are allowing the surplus pool size to 2697 * exceed overcommit. There are few sane options here. Since 2698 * alloc_surplus_huge_page() is checking the global counter, 2699 * though, we'll note that we're not allowed to exceed surplus 2700 * and won't grow the pool anywhere else. Not until one of the 2701 * sysctls are changed, or the surplus pages go out of use. 2702 */ 2703 min_count = h->resv_huge_pages + h->nr_huge_pages - h->free_huge_pages; 2704 min_count = max(count, min_count); 2705 try_to_free_low(h, min_count, nodes_allowed); 2706 while (min_count < persistent_huge_pages(h)) { 2707 if (!free_pool_huge_page(h, nodes_allowed, 0)) 2708 break; 2709 cond_resched_lock(&hugetlb_lock); 2710 } 2711 while (count < persistent_huge_pages(h)) { 2712 if (!adjust_pool_surplus(h, nodes_allowed, 1)) 2713 break; 2714 } 2715 out: 2716 h->max_huge_pages = persistent_huge_pages(h); 2717 spin_unlock(&hugetlb_lock); 2718 2719 NODEMASK_FREE(node_alloc_noretry); 2720 2721 return 0; 2722 } 2723 2724 #define HSTATE_ATTR_RO(_name) \ 2725 static struct kobj_attribute _name##_attr = __ATTR_RO(_name) 2726 2727 #define HSTATE_ATTR(_name) \ 2728 static struct kobj_attribute _name##_attr = \ 2729 __ATTR(_name, 0644, _name##_show, _name##_store) 2730 2731 static struct kobject *hugepages_kobj; 2732 static struct kobject *hstate_kobjs[HUGE_MAX_HSTATE]; 2733 2734 static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp); 2735 2736 static struct hstate *kobj_to_hstate(struct kobject *kobj, int *nidp) 2737 { 2738 int i; 2739 2740 for (i = 0; i < HUGE_MAX_HSTATE; i++) 2741 if (hstate_kobjs[i] == kobj) { 2742 if (nidp) 2743 *nidp = NUMA_NO_NODE; 2744 return &hstates[i]; 2745 } 2746 2747 return kobj_to_node_hstate(kobj, nidp); 2748 } 2749 2750 static ssize_t nr_hugepages_show_common(struct kobject *kobj, 2751 struct kobj_attribute *attr, char *buf) 2752 { 2753 struct hstate *h; 2754 unsigned long nr_huge_pages; 2755 int nid; 2756 2757 h = kobj_to_hstate(kobj, &nid); 2758 if (nid == NUMA_NO_NODE) 2759 nr_huge_pages = h->nr_huge_pages; 2760 else 2761 nr_huge_pages = h->nr_huge_pages_node[nid]; 2762 2763 return sysfs_emit(buf, "%lu\n", nr_huge_pages); 2764 } 2765 2766 static ssize_t __nr_hugepages_store_common(bool obey_mempolicy, 2767 struct hstate *h, int nid, 2768 unsigned long count, size_t len) 2769 { 2770 int err; 2771 nodemask_t nodes_allowed, *n_mask; 2772 2773 if (hstate_is_gigantic(h) && !gigantic_page_runtime_supported()) 2774 return -EINVAL; 2775 2776 if (nid == NUMA_NO_NODE) { 2777 /* 2778 * global hstate attribute 2779 */ 2780 if (!(obey_mempolicy && 2781 init_nodemask_of_mempolicy(&nodes_allowed))) 2782 n_mask = &node_states[N_MEMORY]; 2783 else 2784 n_mask = &nodes_allowed; 2785 } else { 2786 /* 2787 * Node specific request. count adjustment happens in 2788 * set_max_huge_pages() after acquiring hugetlb_lock. 2789 */ 2790 init_nodemask_of_node(&nodes_allowed, nid); 2791 n_mask = &nodes_allowed; 2792 } 2793 2794 err = set_max_huge_pages(h, count, nid, n_mask); 2795 2796 return err ? err : len; 2797 } 2798 2799 static ssize_t nr_hugepages_store_common(bool obey_mempolicy, 2800 struct kobject *kobj, const char *buf, 2801 size_t len) 2802 { 2803 struct hstate *h; 2804 unsigned long count; 2805 int nid; 2806 int err; 2807 2808 err = kstrtoul(buf, 10, &count); 2809 if (err) 2810 return err; 2811 2812 h = kobj_to_hstate(kobj, &nid); 2813 return __nr_hugepages_store_common(obey_mempolicy, h, nid, count, len); 2814 } 2815 2816 static ssize_t nr_hugepages_show(struct kobject *kobj, 2817 struct kobj_attribute *attr, char *buf) 2818 { 2819 return nr_hugepages_show_common(kobj, attr, buf); 2820 } 2821 2822 static ssize_t nr_hugepages_store(struct kobject *kobj, 2823 struct kobj_attribute *attr, const char *buf, size_t len) 2824 { 2825 return nr_hugepages_store_common(false, kobj, buf, len); 2826 } 2827 HSTATE_ATTR(nr_hugepages); 2828 2829 #ifdef CONFIG_NUMA 2830 2831 /* 2832 * hstate attribute for optionally mempolicy-based constraint on persistent 2833 * huge page alloc/free. 2834 */ 2835 static ssize_t nr_hugepages_mempolicy_show(struct kobject *kobj, 2836 struct kobj_attribute *attr, 2837 char *buf) 2838 { 2839 return nr_hugepages_show_common(kobj, attr, buf); 2840 } 2841 2842 static ssize_t nr_hugepages_mempolicy_store(struct kobject *kobj, 2843 struct kobj_attribute *attr, const char *buf, size_t len) 2844 { 2845 return nr_hugepages_store_common(true, kobj, buf, len); 2846 } 2847 HSTATE_ATTR(nr_hugepages_mempolicy); 2848 #endif 2849 2850 2851 static ssize_t nr_overcommit_hugepages_show(struct kobject *kobj, 2852 struct kobj_attribute *attr, char *buf) 2853 { 2854 struct hstate *h = kobj_to_hstate(kobj, NULL); 2855 return sysfs_emit(buf, "%lu\n", h->nr_overcommit_huge_pages); 2856 } 2857 2858 static ssize_t nr_overcommit_hugepages_store(struct kobject *kobj, 2859 struct kobj_attribute *attr, const char *buf, size_t count) 2860 { 2861 int err; 2862 unsigned long input; 2863 struct hstate *h = kobj_to_hstate(kobj, NULL); 2864 2865 if (hstate_is_gigantic(h)) 2866 return -EINVAL; 2867 2868 err = kstrtoul(buf, 10, &input); 2869 if (err) 2870 return err; 2871 2872 spin_lock(&hugetlb_lock); 2873 h->nr_overcommit_huge_pages = input; 2874 spin_unlock(&hugetlb_lock); 2875 2876 return count; 2877 } 2878 HSTATE_ATTR(nr_overcommit_hugepages); 2879 2880 static ssize_t free_hugepages_show(struct kobject *kobj, 2881 struct kobj_attribute *attr, char *buf) 2882 { 2883 struct hstate *h; 2884 unsigned long free_huge_pages; 2885 int nid; 2886 2887 h = kobj_to_hstate(kobj, &nid); 2888 if (nid == NUMA_NO_NODE) 2889 free_huge_pages = h->free_huge_pages; 2890 else 2891 free_huge_pages = h->free_huge_pages_node[nid]; 2892 2893 return sysfs_emit(buf, "%lu\n", free_huge_pages); 2894 } 2895 HSTATE_ATTR_RO(free_hugepages); 2896 2897 static ssize_t resv_hugepages_show(struct kobject *kobj, 2898 struct kobj_attribute *attr, char *buf) 2899 { 2900 struct hstate *h = kobj_to_hstate(kobj, NULL); 2901 return sysfs_emit(buf, "%lu\n", h->resv_huge_pages); 2902 } 2903 HSTATE_ATTR_RO(resv_hugepages); 2904 2905 static ssize_t surplus_hugepages_show(struct kobject *kobj, 2906 struct kobj_attribute *attr, char *buf) 2907 { 2908 struct hstate *h; 2909 unsigned long surplus_huge_pages; 2910 int nid; 2911 2912 h = kobj_to_hstate(kobj, &nid); 2913 if (nid == NUMA_NO_NODE) 2914 surplus_huge_pages = h->surplus_huge_pages; 2915 else 2916 surplus_huge_pages = h->surplus_huge_pages_node[nid]; 2917 2918 return sysfs_emit(buf, "%lu\n", surplus_huge_pages); 2919 } 2920 HSTATE_ATTR_RO(surplus_hugepages); 2921 2922 static struct attribute *hstate_attrs[] = { 2923 &nr_hugepages_attr.attr, 2924 &nr_overcommit_hugepages_attr.attr, 2925 &free_hugepages_attr.attr, 2926 &resv_hugepages_attr.attr, 2927 &surplus_hugepages_attr.attr, 2928 #ifdef CONFIG_NUMA 2929 &nr_hugepages_mempolicy_attr.attr, 2930 #endif 2931 NULL, 2932 }; 2933 2934 static const struct attribute_group hstate_attr_group = { 2935 .attrs = hstate_attrs, 2936 }; 2937 2938 static int hugetlb_sysfs_add_hstate(struct hstate *h, struct kobject *parent, 2939 struct kobject **hstate_kobjs, 2940 const struct attribute_group *hstate_attr_group) 2941 { 2942 int retval; 2943 int hi = hstate_index(h); 2944 2945 hstate_kobjs[hi] = kobject_create_and_add(h->name, parent); 2946 if (!hstate_kobjs[hi]) 2947 return -ENOMEM; 2948 2949 retval = sysfs_create_group(hstate_kobjs[hi], hstate_attr_group); 2950 if (retval) 2951 kobject_put(hstate_kobjs[hi]); 2952 2953 return retval; 2954 } 2955 2956 static void __init hugetlb_sysfs_init(void) 2957 { 2958 struct hstate *h; 2959 int err; 2960 2961 hugepages_kobj = kobject_create_and_add("hugepages", mm_kobj); 2962 if (!hugepages_kobj) 2963 return; 2964 2965 for_each_hstate(h) { 2966 err = hugetlb_sysfs_add_hstate(h, hugepages_kobj, 2967 hstate_kobjs, &hstate_attr_group); 2968 if (err) 2969 pr_err("HugeTLB: Unable to add hstate %s", h->name); 2970 } 2971 } 2972 2973 #ifdef CONFIG_NUMA 2974 2975 /* 2976 * node_hstate/s - associate per node hstate attributes, via their kobjects, 2977 * with node devices in node_devices[] using a parallel array. The array 2978 * index of a node device or _hstate == node id. 2979 * This is here to avoid any static dependency of the node device driver, in 2980 * the base kernel, on the hugetlb module. 2981 */ 2982 struct node_hstate { 2983 struct kobject *hugepages_kobj; 2984 struct kobject *hstate_kobjs[HUGE_MAX_HSTATE]; 2985 }; 2986 static struct node_hstate node_hstates[MAX_NUMNODES]; 2987 2988 /* 2989 * A subset of global hstate attributes for node devices 2990 */ 2991 static struct attribute *per_node_hstate_attrs[] = { 2992 &nr_hugepages_attr.attr, 2993 &free_hugepages_attr.attr, 2994 &surplus_hugepages_attr.attr, 2995 NULL, 2996 }; 2997 2998 static const struct attribute_group per_node_hstate_attr_group = { 2999 .attrs = per_node_hstate_attrs, 3000 }; 3001 3002 /* 3003 * kobj_to_node_hstate - lookup global hstate for node device hstate attr kobj. 3004 * Returns node id via non-NULL nidp. 3005 */ 3006 static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp) 3007 { 3008 int nid; 3009 3010 for (nid = 0; nid < nr_node_ids; nid++) { 3011 struct node_hstate *nhs = &node_hstates[nid]; 3012 int i; 3013 for (i = 0; i < HUGE_MAX_HSTATE; i++) 3014 if (nhs->hstate_kobjs[i] == kobj) { 3015 if (nidp) 3016 *nidp = nid; 3017 return &hstates[i]; 3018 } 3019 } 3020 3021 BUG(); 3022 return NULL; 3023 } 3024 3025 /* 3026 * Unregister hstate attributes from a single node device. 3027 * No-op if no hstate attributes attached. 3028 */ 3029 static void hugetlb_unregister_node(struct node *node) 3030 { 3031 struct hstate *h; 3032 struct node_hstate *nhs = &node_hstates[node->dev.id]; 3033 3034 if (!nhs->hugepages_kobj) 3035 return; /* no hstate attributes */ 3036 3037 for_each_hstate(h) { 3038 int idx = hstate_index(h); 3039 if (nhs->hstate_kobjs[idx]) { 3040 kobject_put(nhs->hstate_kobjs[idx]); 3041 nhs->hstate_kobjs[idx] = NULL; 3042 } 3043 } 3044 3045 kobject_put(nhs->hugepages_kobj); 3046 nhs->hugepages_kobj = NULL; 3047 } 3048 3049 3050 /* 3051 * Register hstate attributes for a single node device. 3052 * No-op if attributes already registered. 3053 */ 3054 static void hugetlb_register_node(struct node *node) 3055 { 3056 struct hstate *h; 3057 struct node_hstate *nhs = &node_hstates[node->dev.id]; 3058 int err; 3059 3060 if (nhs->hugepages_kobj) 3061 return; /* already allocated */ 3062 3063 nhs->hugepages_kobj = kobject_create_and_add("hugepages", 3064 &node->dev.kobj); 3065 if (!nhs->hugepages_kobj) 3066 return; 3067 3068 for_each_hstate(h) { 3069 err = hugetlb_sysfs_add_hstate(h, nhs->hugepages_kobj, 3070 nhs->hstate_kobjs, 3071 &per_node_hstate_attr_group); 3072 if (err) { 3073 pr_err("HugeTLB: Unable to add hstate %s for node %d\n", 3074 h->name, node->dev.id); 3075 hugetlb_unregister_node(node); 3076 break; 3077 } 3078 } 3079 } 3080 3081 /* 3082 * hugetlb init time: register hstate attributes for all registered node 3083 * devices of nodes that have memory. All on-line nodes should have 3084 * registered their associated device by this time. 3085 */ 3086 static void __init hugetlb_register_all_nodes(void) 3087 { 3088 int nid; 3089 3090 for_each_node_state(nid, N_MEMORY) { 3091 struct node *node = node_devices[nid]; 3092 if (node->dev.id == nid) 3093 hugetlb_register_node(node); 3094 } 3095 3096 /* 3097 * Let the node device driver know we're here so it can 3098 * [un]register hstate attributes on node hotplug. 3099 */ 3100 register_hugetlbfs_with_node(hugetlb_register_node, 3101 hugetlb_unregister_node); 3102 } 3103 #else /* !CONFIG_NUMA */ 3104 3105 static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp) 3106 { 3107 BUG(); 3108 if (nidp) 3109 *nidp = -1; 3110 return NULL; 3111 } 3112 3113 static void hugetlb_register_all_nodes(void) { } 3114 3115 #endif 3116 3117 static int __init hugetlb_init(void) 3118 { 3119 int i; 3120 3121 if (!hugepages_supported()) { 3122 if (hugetlb_max_hstate || default_hstate_max_huge_pages) 3123 pr_warn("HugeTLB: huge pages not supported, ignoring associated command-line parameters\n"); 3124 return 0; 3125 } 3126 3127 /* 3128 * Make sure HPAGE_SIZE (HUGETLB_PAGE_ORDER) hstate exists. Some 3129 * architectures depend on setup being done here. 3130 */ 3131 hugetlb_add_hstate(HUGETLB_PAGE_ORDER); 3132 if (!parsed_default_hugepagesz) { 3133 /* 3134 * If we did not parse a default huge page size, set 3135 * default_hstate_idx to HPAGE_SIZE hstate. And, if the 3136 * number of huge pages for this default size was implicitly 3137 * specified, set that here as well. 3138 * Note that the implicit setting will overwrite an explicit 3139 * setting. A warning will be printed in this case. 3140 */ 3141 default_hstate_idx = hstate_index(size_to_hstate(HPAGE_SIZE)); 3142 if (default_hstate_max_huge_pages) { 3143 if (default_hstate.max_huge_pages) { 3144 char buf[32]; 3145 3146 string_get_size(huge_page_size(&default_hstate), 3147 1, STRING_UNITS_2, buf, 32); 3148 pr_warn("HugeTLB: Ignoring hugepages=%lu associated with %s page size\n", 3149 default_hstate.max_huge_pages, buf); 3150 pr_warn("HugeTLB: Using hugepages=%lu for number of default huge pages\n", 3151 default_hstate_max_huge_pages); 3152 } 3153 default_hstate.max_huge_pages = 3154 default_hstate_max_huge_pages; 3155 } 3156 } 3157 3158 hugetlb_cma_check(); 3159 hugetlb_init_hstates(); 3160 gather_bootmem_prealloc(); 3161 report_hugepages(); 3162 3163 hugetlb_sysfs_init(); 3164 hugetlb_register_all_nodes(); 3165 hugetlb_cgroup_file_init(); 3166 3167 #ifdef CONFIG_SMP 3168 num_fault_mutexes = roundup_pow_of_two(8 * num_possible_cpus()); 3169 #else 3170 num_fault_mutexes = 1; 3171 #endif 3172 hugetlb_fault_mutex_table = 3173 kmalloc_array(num_fault_mutexes, sizeof(struct mutex), 3174 GFP_KERNEL); 3175 BUG_ON(!hugetlb_fault_mutex_table); 3176 3177 for (i = 0; i < num_fault_mutexes; i++) 3178 mutex_init(&hugetlb_fault_mutex_table[i]); 3179 return 0; 3180 } 3181 subsys_initcall(hugetlb_init); 3182 3183 /* Overwritten by architectures with more huge page sizes */ 3184 bool __init __attribute((weak)) arch_hugetlb_valid_size(unsigned long size) 3185 { 3186 return size == HPAGE_SIZE; 3187 } 3188 3189 void __init hugetlb_add_hstate(unsigned int order) 3190 { 3191 struct hstate *h; 3192 unsigned long i; 3193 3194 if (size_to_hstate(PAGE_SIZE << order)) { 3195 return; 3196 } 3197 BUG_ON(hugetlb_max_hstate >= HUGE_MAX_HSTATE); 3198 BUG_ON(order == 0); 3199 h = &hstates[hugetlb_max_hstate++]; 3200 h->order = order; 3201 h->mask = ~((1ULL << (order + PAGE_SHIFT)) - 1); 3202 for (i = 0; i < MAX_NUMNODES; ++i) 3203 INIT_LIST_HEAD(&h->hugepage_freelists[i]); 3204 INIT_LIST_HEAD(&h->hugepage_activelist); 3205 h->next_nid_to_alloc = first_memory_node; 3206 h->next_nid_to_free = first_memory_node; 3207 snprintf(h->name, HSTATE_NAME_LEN, "hugepages-%lukB", 3208 huge_page_size(h)/1024); 3209 3210 parsed_hstate = h; 3211 } 3212 3213 /* 3214 * hugepages command line processing 3215 * hugepages normally follows a valid hugepagsz or default_hugepagsz 3216 * specification. If not, ignore the hugepages value. hugepages can also 3217 * be the first huge page command line option in which case it implicitly 3218 * specifies the number of huge pages for the default size. 3219 */ 3220 static int __init hugepages_setup(char *s) 3221 { 3222 unsigned long *mhp; 3223 static unsigned long *last_mhp; 3224 3225 if (!parsed_valid_hugepagesz) { 3226 pr_warn("HugeTLB: hugepages=%s does not follow a valid hugepagesz, ignoring\n", s); 3227 parsed_valid_hugepagesz = true; 3228 return 0; 3229 } 3230 3231 /* 3232 * !hugetlb_max_hstate means we haven't parsed a hugepagesz= parameter 3233 * yet, so this hugepages= parameter goes to the "default hstate". 3234 * Otherwise, it goes with the previously parsed hugepagesz or 3235 * default_hugepagesz. 3236 */ 3237 else if (!hugetlb_max_hstate) 3238 mhp = &default_hstate_max_huge_pages; 3239 else 3240 mhp = &parsed_hstate->max_huge_pages; 3241 3242 if (mhp == last_mhp) { 3243 pr_warn("HugeTLB: hugepages= specified twice without interleaving hugepagesz=, ignoring hugepages=%s\n", s); 3244 return 0; 3245 } 3246 3247 if (sscanf(s, "%lu", mhp) <= 0) 3248 *mhp = 0; 3249 3250 /* 3251 * Global state is always initialized later in hugetlb_init. 3252 * But we need to allocate >= MAX_ORDER hstates here early to still 3253 * use the bootmem allocator. 3254 */ 3255 if (hugetlb_max_hstate && parsed_hstate->order >= MAX_ORDER) 3256 hugetlb_hstate_alloc_pages(parsed_hstate); 3257 3258 last_mhp = mhp; 3259 3260 return 1; 3261 } 3262 __setup("hugepages=", hugepages_setup); 3263 3264 /* 3265 * hugepagesz command line processing 3266 * A specific huge page size can only be specified once with hugepagesz. 3267 * hugepagesz is followed by hugepages on the command line. The global 3268 * variable 'parsed_valid_hugepagesz' is used to determine if prior 3269 * hugepagesz argument was valid. 3270 */ 3271 static int __init hugepagesz_setup(char *s) 3272 { 3273 unsigned long size; 3274 struct hstate *h; 3275 3276 parsed_valid_hugepagesz = false; 3277 size = (unsigned long)memparse(s, NULL); 3278 3279 if (!arch_hugetlb_valid_size(size)) { 3280 pr_err("HugeTLB: unsupported hugepagesz=%s\n", s); 3281 return 0; 3282 } 3283 3284 h = size_to_hstate(size); 3285 if (h) { 3286 /* 3287 * hstate for this size already exists. This is normally 3288 * an error, but is allowed if the existing hstate is the 3289 * default hstate. More specifically, it is only allowed if 3290 * the number of huge pages for the default hstate was not 3291 * previously specified. 3292 */ 3293 if (!parsed_default_hugepagesz || h != &default_hstate || 3294 default_hstate.max_huge_pages) { 3295 pr_warn("HugeTLB: hugepagesz=%s specified twice, ignoring\n", s); 3296 return 0; 3297 } 3298 3299 /* 3300 * No need to call hugetlb_add_hstate() as hstate already 3301 * exists. But, do set parsed_hstate so that a following 3302 * hugepages= parameter will be applied to this hstate. 3303 */ 3304 parsed_hstate = h; 3305 parsed_valid_hugepagesz = true; 3306 return 1; 3307 } 3308 3309 hugetlb_add_hstate(ilog2(size) - PAGE_SHIFT); 3310 parsed_valid_hugepagesz = true; 3311 return 1; 3312 } 3313 __setup("hugepagesz=", hugepagesz_setup); 3314 3315 /* 3316 * default_hugepagesz command line input 3317 * Only one instance of default_hugepagesz allowed on command line. 3318 */ 3319 static int __init default_hugepagesz_setup(char *s) 3320 { 3321 unsigned long size; 3322 3323 parsed_valid_hugepagesz = false; 3324 if (parsed_default_hugepagesz) { 3325 pr_err("HugeTLB: default_hugepagesz previously specified, ignoring %s\n", s); 3326 return 0; 3327 } 3328 3329 size = (unsigned long)memparse(s, NULL); 3330 3331 if (!arch_hugetlb_valid_size(size)) { 3332 pr_err("HugeTLB: unsupported default_hugepagesz=%s\n", s); 3333 return 0; 3334 } 3335 3336 hugetlb_add_hstate(ilog2(size) - PAGE_SHIFT); 3337 parsed_valid_hugepagesz = true; 3338 parsed_default_hugepagesz = true; 3339 default_hstate_idx = hstate_index(size_to_hstate(size)); 3340 3341 /* 3342 * The number of default huge pages (for this size) could have been 3343 * specified as the first hugetlb parameter: hugepages=X. If so, 3344 * then default_hstate_max_huge_pages is set. If the default huge 3345 * page size is gigantic (>= MAX_ORDER), then the pages must be 3346 * allocated here from bootmem allocator. 3347 */ 3348 if (default_hstate_max_huge_pages) { 3349 default_hstate.max_huge_pages = default_hstate_max_huge_pages; 3350 if (hstate_is_gigantic(&default_hstate)) 3351 hugetlb_hstate_alloc_pages(&default_hstate); 3352 default_hstate_max_huge_pages = 0; 3353 } 3354 3355 return 1; 3356 } 3357 __setup("default_hugepagesz=", default_hugepagesz_setup); 3358 3359 static unsigned int allowed_mems_nr(struct hstate *h) 3360 { 3361 int node; 3362 unsigned int nr = 0; 3363 nodemask_t *mpol_allowed; 3364 unsigned int *array = h->free_huge_pages_node; 3365 gfp_t gfp_mask = htlb_alloc_mask(h); 3366 3367 mpol_allowed = policy_nodemask_current(gfp_mask); 3368 3369 for_each_node_mask(node, cpuset_current_mems_allowed) { 3370 if (!mpol_allowed || 3371 (mpol_allowed && node_isset(node, *mpol_allowed))) 3372 nr += array[node]; 3373 } 3374 3375 return nr; 3376 } 3377 3378 #ifdef CONFIG_SYSCTL 3379 static int proc_hugetlb_doulongvec_minmax(struct ctl_table *table, int write, 3380 void *buffer, size_t *length, 3381 loff_t *ppos, unsigned long *out) 3382 { 3383 struct ctl_table dup_table; 3384 3385 /* 3386 * In order to avoid races with __do_proc_doulongvec_minmax(), we 3387 * can duplicate the @table and alter the duplicate of it. 3388 */ 3389 dup_table = *table; 3390 dup_table.data = out; 3391 3392 return proc_doulongvec_minmax(&dup_table, write, buffer, length, ppos); 3393 } 3394 3395 static int hugetlb_sysctl_handler_common(bool obey_mempolicy, 3396 struct ctl_table *table, int write, 3397 void *buffer, size_t *length, loff_t *ppos) 3398 { 3399 struct hstate *h = &default_hstate; 3400 unsigned long tmp = h->max_huge_pages; 3401 int ret; 3402 3403 if (!hugepages_supported()) 3404 return -EOPNOTSUPP; 3405 3406 ret = proc_hugetlb_doulongvec_minmax(table, write, buffer, length, ppos, 3407 &tmp); 3408 if (ret) 3409 goto out; 3410 3411 if (write) 3412 ret = __nr_hugepages_store_common(obey_mempolicy, h, 3413 NUMA_NO_NODE, tmp, *length); 3414 out: 3415 return ret; 3416 } 3417 3418 int hugetlb_sysctl_handler(struct ctl_table *table, int write, 3419 void *buffer, size_t *length, loff_t *ppos) 3420 { 3421 3422 return hugetlb_sysctl_handler_common(false, table, write, 3423 buffer, length, ppos); 3424 } 3425 3426 #ifdef CONFIG_NUMA 3427 int hugetlb_mempolicy_sysctl_handler(struct ctl_table *table, int write, 3428 void *buffer, size_t *length, loff_t *ppos) 3429 { 3430 return hugetlb_sysctl_handler_common(true, table, write, 3431 buffer, length, ppos); 3432 } 3433 #endif /* CONFIG_NUMA */ 3434 3435 int hugetlb_overcommit_handler(struct ctl_table *table, int write, 3436 void *buffer, size_t *length, loff_t *ppos) 3437 { 3438 struct hstate *h = &default_hstate; 3439 unsigned long tmp; 3440 int ret; 3441 3442 if (!hugepages_supported()) 3443 return -EOPNOTSUPP; 3444 3445 tmp = h->nr_overcommit_huge_pages; 3446 3447 if (write && hstate_is_gigantic(h)) 3448 return -EINVAL; 3449 3450 ret = proc_hugetlb_doulongvec_minmax(table, write, buffer, length, ppos, 3451 &tmp); 3452 if (ret) 3453 goto out; 3454 3455 if (write) { 3456 spin_lock(&hugetlb_lock); 3457 h->nr_overcommit_huge_pages = tmp; 3458 spin_unlock(&hugetlb_lock); 3459 } 3460 out: 3461 return ret; 3462 } 3463 3464 #endif /* CONFIG_SYSCTL */ 3465 3466 void hugetlb_report_meminfo(struct seq_file *m) 3467 { 3468 struct hstate *h; 3469 unsigned long total = 0; 3470 3471 if (!hugepages_supported()) 3472 return; 3473 3474 for_each_hstate(h) { 3475 unsigned long count = h->nr_huge_pages; 3476 3477 total += (PAGE_SIZE << huge_page_order(h)) * count; 3478 3479 if (h == &default_hstate) 3480 seq_printf(m, 3481 "HugePages_Total: %5lu\n" 3482 "HugePages_Free: %5lu\n" 3483 "HugePages_Rsvd: %5lu\n" 3484 "HugePages_Surp: %5lu\n" 3485 "Hugepagesize: %8lu kB\n", 3486 count, 3487 h->free_huge_pages, 3488 h->resv_huge_pages, 3489 h->surplus_huge_pages, 3490 (PAGE_SIZE << huge_page_order(h)) / 1024); 3491 } 3492 3493 seq_printf(m, "Hugetlb: %8lu kB\n", total / 1024); 3494 } 3495 3496 int hugetlb_report_node_meminfo(char *buf, int len, int nid) 3497 { 3498 struct hstate *h = &default_hstate; 3499 3500 if (!hugepages_supported()) 3501 return 0; 3502 3503 return sysfs_emit_at(buf, len, 3504 "Node %d HugePages_Total: %5u\n" 3505 "Node %d HugePages_Free: %5u\n" 3506 "Node %d HugePages_Surp: %5u\n", 3507 nid, h->nr_huge_pages_node[nid], 3508 nid, h->free_huge_pages_node[nid], 3509 nid, h->surplus_huge_pages_node[nid]); 3510 } 3511 3512 void hugetlb_show_meminfo(void) 3513 { 3514 struct hstate *h; 3515 int nid; 3516 3517 if (!hugepages_supported()) 3518 return; 3519 3520 for_each_node_state(nid, N_MEMORY) 3521 for_each_hstate(h) 3522 pr_info("Node %d hugepages_total=%u hugepages_free=%u hugepages_surp=%u hugepages_size=%lukB\n", 3523 nid, 3524 h->nr_huge_pages_node[nid], 3525 h->free_huge_pages_node[nid], 3526 h->surplus_huge_pages_node[nid], 3527 1UL << (huge_page_order(h) + PAGE_SHIFT - 10)); 3528 } 3529 3530 void hugetlb_report_usage(struct seq_file *m, struct mm_struct *mm) 3531 { 3532 seq_printf(m, "HugetlbPages:\t%8lu kB\n", 3533 atomic_long_read(&mm->hugetlb_usage) << (PAGE_SHIFT - 10)); 3534 } 3535 3536 /* Return the number pages of memory we physically have, in PAGE_SIZE units. */ 3537 unsigned long hugetlb_total_pages(void) 3538 { 3539 struct hstate *h; 3540 unsigned long nr_total_pages = 0; 3541 3542 for_each_hstate(h) 3543 nr_total_pages += h->nr_huge_pages * pages_per_huge_page(h); 3544 return nr_total_pages; 3545 } 3546 3547 static int hugetlb_acct_memory(struct hstate *h, long delta) 3548 { 3549 int ret = -ENOMEM; 3550 3551 spin_lock(&hugetlb_lock); 3552 /* 3553 * When cpuset is configured, it breaks the strict hugetlb page 3554 * reservation as the accounting is done on a global variable. Such 3555 * reservation is completely rubbish in the presence of cpuset because 3556 * the reservation is not checked against page availability for the 3557 * current cpuset. Application can still potentially OOM'ed by kernel 3558 * with lack of free htlb page in cpuset that the task is in. 3559 * Attempt to enforce strict accounting with cpuset is almost 3560 * impossible (or too ugly) because cpuset is too fluid that 3561 * task or memory node can be dynamically moved between cpusets. 3562 * 3563 * The change of semantics for shared hugetlb mapping with cpuset is 3564 * undesirable. However, in order to preserve some of the semantics, 3565 * we fall back to check against current free page availability as 3566 * a best attempt and hopefully to minimize the impact of changing 3567 * semantics that cpuset has. 3568 * 3569 * Apart from cpuset, we also have memory policy mechanism that 3570 * also determines from which node the kernel will allocate memory 3571 * in a NUMA system. So similar to cpuset, we also should consider 3572 * the memory policy of the current task. Similar to the description 3573 * above. 3574 */ 3575 if (delta > 0) { 3576 if (gather_surplus_pages(h, delta) < 0) 3577 goto out; 3578 3579 if (delta > allowed_mems_nr(h)) { 3580 return_unused_surplus_pages(h, delta); 3581 goto out; 3582 } 3583 } 3584 3585 ret = 0; 3586 if (delta < 0) 3587 return_unused_surplus_pages(h, (unsigned long) -delta); 3588 3589 out: 3590 spin_unlock(&hugetlb_lock); 3591 return ret; 3592 } 3593 3594 static void hugetlb_vm_op_open(struct vm_area_struct *vma) 3595 { 3596 struct resv_map *resv = vma_resv_map(vma); 3597 3598 /* 3599 * This new VMA should share its siblings reservation map if present. 3600 * The VMA will only ever have a valid reservation map pointer where 3601 * it is being copied for another still existing VMA. As that VMA 3602 * has a reference to the reservation map it cannot disappear until 3603 * after this open call completes. It is therefore safe to take a 3604 * new reference here without additional locking. 3605 */ 3606 if (resv && is_vma_resv_set(vma, HPAGE_RESV_OWNER)) 3607 kref_get(&resv->refs); 3608 } 3609 3610 static void hugetlb_vm_op_close(struct vm_area_struct *vma) 3611 { 3612 struct hstate *h = hstate_vma(vma); 3613 struct resv_map *resv = vma_resv_map(vma); 3614 struct hugepage_subpool *spool = subpool_vma(vma); 3615 unsigned long reserve, start, end; 3616 long gbl_reserve; 3617 3618 if (!resv || !is_vma_resv_set(vma, HPAGE_RESV_OWNER)) 3619 return; 3620 3621 start = vma_hugecache_offset(h, vma, vma->vm_start); 3622 end = vma_hugecache_offset(h, vma, vma->vm_end); 3623 3624 reserve = (end - start) - region_count(resv, start, end); 3625 hugetlb_cgroup_uncharge_counter(resv, start, end); 3626 if (reserve) { 3627 /* 3628 * Decrement reserve counts. The global reserve count may be 3629 * adjusted if the subpool has a minimum size. 3630 */ 3631 gbl_reserve = hugepage_subpool_put_pages(spool, reserve); 3632 hugetlb_acct_memory(h, -gbl_reserve); 3633 } 3634 3635 kref_put(&resv->refs, resv_map_release); 3636 } 3637 3638 static int hugetlb_vm_op_split(struct vm_area_struct *vma, unsigned long addr) 3639 { 3640 if (addr & ~(huge_page_mask(hstate_vma(vma)))) 3641 return -EINVAL; 3642 return 0; 3643 } 3644 3645 static unsigned long hugetlb_vm_op_pagesize(struct vm_area_struct *vma) 3646 { 3647 struct hstate *hstate = hstate_vma(vma); 3648 3649 return 1UL << huge_page_shift(hstate); 3650 } 3651 3652 /* 3653 * We cannot handle pagefaults against hugetlb pages at all. They cause 3654 * handle_mm_fault() to try to instantiate regular-sized pages in the 3655 * hugegpage VMA. do_page_fault() is supposed to trap this, so BUG is we get 3656 * this far. 3657 */ 3658 static vm_fault_t hugetlb_vm_op_fault(struct vm_fault *vmf) 3659 { 3660 BUG(); 3661 return 0; 3662 } 3663 3664 /* 3665 * When a new function is introduced to vm_operations_struct and added 3666 * to hugetlb_vm_ops, please consider adding the function to shm_vm_ops. 3667 * This is because under System V memory model, mappings created via 3668 * shmget/shmat with "huge page" specified are backed by hugetlbfs files, 3669 * their original vm_ops are overwritten with shm_vm_ops. 3670 */ 3671 const struct vm_operations_struct hugetlb_vm_ops = { 3672 .fault = hugetlb_vm_op_fault, 3673 .open = hugetlb_vm_op_open, 3674 .close = hugetlb_vm_op_close, 3675 .may_split = hugetlb_vm_op_split, 3676 .pagesize = hugetlb_vm_op_pagesize, 3677 }; 3678 3679 static pte_t make_huge_pte(struct vm_area_struct *vma, struct page *page, 3680 int writable) 3681 { 3682 pte_t entry; 3683 3684 if (writable) { 3685 entry = huge_pte_mkwrite(huge_pte_mkdirty(mk_huge_pte(page, 3686 vma->vm_page_prot))); 3687 } else { 3688 entry = huge_pte_wrprotect(mk_huge_pte(page, 3689 vma->vm_page_prot)); 3690 } 3691 entry = pte_mkyoung(entry); 3692 entry = pte_mkhuge(entry); 3693 entry = arch_make_huge_pte(entry, vma, page, writable); 3694 3695 return entry; 3696 } 3697 3698 static void set_huge_ptep_writable(struct vm_area_struct *vma, 3699 unsigned long address, pte_t *ptep) 3700 { 3701 pte_t entry; 3702 3703 entry = huge_pte_mkwrite(huge_pte_mkdirty(huge_ptep_get(ptep))); 3704 if (huge_ptep_set_access_flags(vma, address, ptep, entry, 1)) 3705 update_mmu_cache(vma, address, ptep); 3706 } 3707 3708 bool is_hugetlb_entry_migration(pte_t pte) 3709 { 3710 swp_entry_t swp; 3711 3712 if (huge_pte_none(pte) || pte_present(pte)) 3713 return false; 3714 swp = pte_to_swp_entry(pte); 3715 if (is_migration_entry(swp)) 3716 return true; 3717 else 3718 return false; 3719 } 3720 3721 static bool is_hugetlb_entry_hwpoisoned(pte_t pte) 3722 { 3723 swp_entry_t swp; 3724 3725 if (huge_pte_none(pte) || pte_present(pte)) 3726 return false; 3727 swp = pte_to_swp_entry(pte); 3728 if (is_hwpoison_entry(swp)) 3729 return true; 3730 else 3731 return false; 3732 } 3733 3734 int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src, 3735 struct vm_area_struct *vma) 3736 { 3737 pte_t *src_pte, *dst_pte, entry, dst_entry; 3738 struct page *ptepage; 3739 unsigned long addr; 3740 int cow; 3741 struct hstate *h = hstate_vma(vma); 3742 unsigned long sz = huge_page_size(h); 3743 struct address_space *mapping = vma->vm_file->f_mapping; 3744 struct mmu_notifier_range range; 3745 int ret = 0; 3746 3747 cow = (vma->vm_flags & (VM_SHARED | VM_MAYWRITE)) == VM_MAYWRITE; 3748 3749 if (cow) { 3750 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, src, 3751 vma->vm_start, 3752 vma->vm_end); 3753 mmu_notifier_invalidate_range_start(&range); 3754 } else { 3755 /* 3756 * For shared mappings i_mmap_rwsem must be held to call 3757 * huge_pte_alloc, otherwise the returned ptep could go 3758 * away if part of a shared pmd and another thread calls 3759 * huge_pmd_unshare. 3760 */ 3761 i_mmap_lock_read(mapping); 3762 } 3763 3764 for (addr = vma->vm_start; addr < vma->vm_end; addr += sz) { 3765 spinlock_t *src_ptl, *dst_ptl; 3766 src_pte = huge_pte_offset(src, addr, sz); 3767 if (!src_pte) 3768 continue; 3769 dst_pte = huge_pte_alloc(dst, addr, sz); 3770 if (!dst_pte) { 3771 ret = -ENOMEM; 3772 break; 3773 } 3774 3775 /* 3776 * If the pagetables are shared don't copy or take references. 3777 * dst_pte == src_pte is the common case of src/dest sharing. 3778 * 3779 * However, src could have 'unshared' and dst shares with 3780 * another vma. If dst_pte !none, this implies sharing. 3781 * Check here before taking page table lock, and once again 3782 * after taking the lock below. 3783 */ 3784 dst_entry = huge_ptep_get(dst_pte); 3785 if ((dst_pte == src_pte) || !huge_pte_none(dst_entry)) 3786 continue; 3787 3788 dst_ptl = huge_pte_lock(h, dst, dst_pte); 3789 src_ptl = huge_pte_lockptr(h, src, src_pte); 3790 spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING); 3791 entry = huge_ptep_get(src_pte); 3792 dst_entry = huge_ptep_get(dst_pte); 3793 if (huge_pte_none(entry) || !huge_pte_none(dst_entry)) { 3794 /* 3795 * Skip if src entry none. Also, skip in the 3796 * unlikely case dst entry !none as this implies 3797 * sharing with another vma. 3798 */ 3799 ; 3800 } else if (unlikely(is_hugetlb_entry_migration(entry) || 3801 is_hugetlb_entry_hwpoisoned(entry))) { 3802 swp_entry_t swp_entry = pte_to_swp_entry(entry); 3803 3804 if (is_write_migration_entry(swp_entry) && cow) { 3805 /* 3806 * COW mappings require pages in both 3807 * parent and child to be set to read. 3808 */ 3809 make_migration_entry_read(&swp_entry); 3810 entry = swp_entry_to_pte(swp_entry); 3811 set_huge_swap_pte_at(src, addr, src_pte, 3812 entry, sz); 3813 } 3814 set_huge_swap_pte_at(dst, addr, dst_pte, entry, sz); 3815 } else { 3816 if (cow) { 3817 /* 3818 * No need to notify as we are downgrading page 3819 * table protection not changing it to point 3820 * to a new page. 3821 * 3822 * See Documentation/vm/mmu_notifier.rst 3823 */ 3824 huge_ptep_set_wrprotect(src, addr, src_pte); 3825 } 3826 entry = huge_ptep_get(src_pte); 3827 ptepage = pte_page(entry); 3828 get_page(ptepage); 3829 page_dup_rmap(ptepage, true); 3830 set_huge_pte_at(dst, addr, dst_pte, entry); 3831 hugetlb_count_add(pages_per_huge_page(h), dst); 3832 } 3833 spin_unlock(src_ptl); 3834 spin_unlock(dst_ptl); 3835 } 3836 3837 if (cow) 3838 mmu_notifier_invalidate_range_end(&range); 3839 else 3840 i_mmap_unlock_read(mapping); 3841 3842 return ret; 3843 } 3844 3845 void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct *vma, 3846 unsigned long start, unsigned long end, 3847 struct page *ref_page) 3848 { 3849 struct mm_struct *mm = vma->vm_mm; 3850 unsigned long address; 3851 pte_t *ptep; 3852 pte_t pte; 3853 spinlock_t *ptl; 3854 struct page *page; 3855 struct hstate *h = hstate_vma(vma); 3856 unsigned long sz = huge_page_size(h); 3857 struct mmu_notifier_range range; 3858 3859 WARN_ON(!is_vm_hugetlb_page(vma)); 3860 BUG_ON(start & ~huge_page_mask(h)); 3861 BUG_ON(end & ~huge_page_mask(h)); 3862 3863 /* 3864 * This is a hugetlb vma, all the pte entries should point 3865 * to huge page. 3866 */ 3867 tlb_change_page_size(tlb, sz); 3868 tlb_start_vma(tlb, vma); 3869 3870 /* 3871 * If sharing possible, alert mmu notifiers of worst case. 3872 */ 3873 mmu_notifier_range_init(&range, MMU_NOTIFY_UNMAP, 0, vma, mm, start, 3874 end); 3875 adjust_range_if_pmd_sharing_possible(vma, &range.start, &range.end); 3876 mmu_notifier_invalidate_range_start(&range); 3877 address = start; 3878 for (; address < end; address += sz) { 3879 ptep = huge_pte_offset(mm, address, sz); 3880 if (!ptep) 3881 continue; 3882 3883 ptl = huge_pte_lock(h, mm, ptep); 3884 if (huge_pmd_unshare(mm, vma, &address, ptep)) { 3885 spin_unlock(ptl); 3886 /* 3887 * We just unmapped a page of PMDs by clearing a PUD. 3888 * The caller's TLB flush range should cover this area. 3889 */ 3890 continue; 3891 } 3892 3893 pte = huge_ptep_get(ptep); 3894 if (huge_pte_none(pte)) { 3895 spin_unlock(ptl); 3896 continue; 3897 } 3898 3899 /* 3900 * Migrating hugepage or HWPoisoned hugepage is already 3901 * unmapped and its refcount is dropped, so just clear pte here. 3902 */ 3903 if (unlikely(!pte_present(pte))) { 3904 huge_pte_clear(mm, address, ptep, sz); 3905 spin_unlock(ptl); 3906 continue; 3907 } 3908 3909 page = pte_page(pte); 3910 /* 3911 * If a reference page is supplied, it is because a specific 3912 * page is being unmapped, not a range. Ensure the page we 3913 * are about to unmap is the actual page of interest. 3914 */ 3915 if (ref_page) { 3916 if (page != ref_page) { 3917 spin_unlock(ptl); 3918 continue; 3919 } 3920 /* 3921 * Mark the VMA as having unmapped its page so that 3922 * future faults in this VMA will fail rather than 3923 * looking like data was lost 3924 */ 3925 set_vma_resv_flags(vma, HPAGE_RESV_UNMAPPED); 3926 } 3927 3928 pte = huge_ptep_get_and_clear(mm, address, ptep); 3929 tlb_remove_huge_tlb_entry(h, tlb, ptep, address); 3930 if (huge_pte_dirty(pte)) 3931 set_page_dirty(page); 3932 3933 hugetlb_count_sub(pages_per_huge_page(h), mm); 3934 page_remove_rmap(page, true); 3935 3936 spin_unlock(ptl); 3937 tlb_remove_page_size(tlb, page, huge_page_size(h)); 3938 /* 3939 * Bail out after unmapping reference page if supplied 3940 */ 3941 if (ref_page) 3942 break; 3943 } 3944 mmu_notifier_invalidate_range_end(&range); 3945 tlb_end_vma(tlb, vma); 3946 } 3947 3948 void __unmap_hugepage_range_final(struct mmu_gather *tlb, 3949 struct vm_area_struct *vma, unsigned long start, 3950 unsigned long end, struct page *ref_page) 3951 { 3952 __unmap_hugepage_range(tlb, vma, start, end, ref_page); 3953 3954 /* 3955 * Clear this flag so that x86's huge_pmd_share page_table_shareable 3956 * test will fail on a vma being torn down, and not grab a page table 3957 * on its way out. We're lucky that the flag has such an appropriate 3958 * name, and can in fact be safely cleared here. We could clear it 3959 * before the __unmap_hugepage_range above, but all that's necessary 3960 * is to clear it before releasing the i_mmap_rwsem. This works 3961 * because in the context this is called, the VMA is about to be 3962 * destroyed and the i_mmap_rwsem is held. 3963 */ 3964 vma->vm_flags &= ~VM_MAYSHARE; 3965 } 3966 3967 void unmap_hugepage_range(struct vm_area_struct *vma, unsigned long start, 3968 unsigned long end, struct page *ref_page) 3969 { 3970 struct mm_struct *mm; 3971 struct mmu_gather tlb; 3972 unsigned long tlb_start = start; 3973 unsigned long tlb_end = end; 3974 3975 /* 3976 * If shared PMDs were possibly used within this vma range, adjust 3977 * start/end for worst case tlb flushing. 3978 * Note that we can not be sure if PMDs are shared until we try to 3979 * unmap pages. However, we want to make sure TLB flushing covers 3980 * the largest possible range. 3981 */ 3982 adjust_range_if_pmd_sharing_possible(vma, &tlb_start, &tlb_end); 3983 3984 mm = vma->vm_mm; 3985 3986 tlb_gather_mmu(&tlb, mm, tlb_start, tlb_end); 3987 __unmap_hugepage_range(&tlb, vma, start, end, ref_page); 3988 tlb_finish_mmu(&tlb, tlb_start, tlb_end); 3989 } 3990 3991 /* 3992 * This is called when the original mapper is failing to COW a MAP_PRIVATE 3993 * mappping it owns the reserve page for. The intention is to unmap the page 3994 * from other VMAs and let the children be SIGKILLed if they are faulting the 3995 * same region. 3996 */ 3997 static void unmap_ref_private(struct mm_struct *mm, struct vm_area_struct *vma, 3998 struct page *page, unsigned long address) 3999 { 4000 struct hstate *h = hstate_vma(vma); 4001 struct vm_area_struct *iter_vma; 4002 struct address_space *mapping; 4003 pgoff_t pgoff; 4004 4005 /* 4006 * vm_pgoff is in PAGE_SIZE units, hence the different calculation 4007 * from page cache lookup which is in HPAGE_SIZE units. 4008 */ 4009 address = address & huge_page_mask(h); 4010 pgoff = ((address - vma->vm_start) >> PAGE_SHIFT) + 4011 vma->vm_pgoff; 4012 mapping = vma->vm_file->f_mapping; 4013 4014 /* 4015 * Take the mapping lock for the duration of the table walk. As 4016 * this mapping should be shared between all the VMAs, 4017 * __unmap_hugepage_range() is called as the lock is already held 4018 */ 4019 i_mmap_lock_write(mapping); 4020 vma_interval_tree_foreach(iter_vma, &mapping->i_mmap, pgoff, pgoff) { 4021 /* Do not unmap the current VMA */ 4022 if (iter_vma == vma) 4023 continue; 4024 4025 /* 4026 * Shared VMAs have their own reserves and do not affect 4027 * MAP_PRIVATE accounting but it is possible that a shared 4028 * VMA is using the same page so check and skip such VMAs. 4029 */ 4030 if (iter_vma->vm_flags & VM_MAYSHARE) 4031 continue; 4032 4033 /* 4034 * Unmap the page from other VMAs without their own reserves. 4035 * They get marked to be SIGKILLed if they fault in these 4036 * areas. This is because a future no-page fault on this VMA 4037 * could insert a zeroed page instead of the data existing 4038 * from the time of fork. This would look like data corruption 4039 */ 4040 if (!is_vma_resv_set(iter_vma, HPAGE_RESV_OWNER)) 4041 unmap_hugepage_range(iter_vma, address, 4042 address + huge_page_size(h), page); 4043 } 4044 i_mmap_unlock_write(mapping); 4045 } 4046 4047 /* 4048 * Hugetlb_cow() should be called with page lock of the original hugepage held. 4049 * Called with hugetlb_instantiation_mutex held and pte_page locked so we 4050 * cannot race with other handlers or page migration. 4051 * Keep the pte_same checks anyway to make transition from the mutex easier. 4052 */ 4053 static vm_fault_t hugetlb_cow(struct mm_struct *mm, struct vm_area_struct *vma, 4054 unsigned long address, pte_t *ptep, 4055 struct page *pagecache_page, spinlock_t *ptl) 4056 { 4057 pte_t pte; 4058 struct hstate *h = hstate_vma(vma); 4059 struct page *old_page, *new_page; 4060 int outside_reserve = 0; 4061 vm_fault_t ret = 0; 4062 unsigned long haddr = address & huge_page_mask(h); 4063 struct mmu_notifier_range range; 4064 4065 pte = huge_ptep_get(ptep); 4066 old_page = pte_page(pte); 4067 4068 retry_avoidcopy: 4069 /* If no-one else is actually using this page, avoid the copy 4070 * and just make the page writable */ 4071 if (page_mapcount(old_page) == 1 && PageAnon(old_page)) { 4072 page_move_anon_rmap(old_page, vma); 4073 set_huge_ptep_writable(vma, haddr, ptep); 4074 return 0; 4075 } 4076 4077 /* 4078 * If the process that created a MAP_PRIVATE mapping is about to 4079 * perform a COW due to a shared page count, attempt to satisfy 4080 * the allocation without using the existing reserves. The pagecache 4081 * page is used to determine if the reserve at this address was 4082 * consumed or not. If reserves were used, a partial faulted mapping 4083 * at the time of fork() could consume its reserves on COW instead 4084 * of the full address range. 4085 */ 4086 if (is_vma_resv_set(vma, HPAGE_RESV_OWNER) && 4087 old_page != pagecache_page) 4088 outside_reserve = 1; 4089 4090 get_page(old_page); 4091 4092 /* 4093 * Drop page table lock as buddy allocator may be called. It will 4094 * be acquired again before returning to the caller, as expected. 4095 */ 4096 spin_unlock(ptl); 4097 new_page = alloc_huge_page(vma, haddr, outside_reserve); 4098 4099 if (IS_ERR(new_page)) { 4100 /* 4101 * If a process owning a MAP_PRIVATE mapping fails to COW, 4102 * it is due to references held by a child and an insufficient 4103 * huge page pool. To guarantee the original mappers 4104 * reliability, unmap the page from child processes. The child 4105 * may get SIGKILLed if it later faults. 4106 */ 4107 if (outside_reserve) { 4108 put_page(old_page); 4109 BUG_ON(huge_pte_none(pte)); 4110 unmap_ref_private(mm, vma, old_page, haddr); 4111 BUG_ON(huge_pte_none(pte)); 4112 spin_lock(ptl); 4113 ptep = huge_pte_offset(mm, haddr, huge_page_size(h)); 4114 if (likely(ptep && 4115 pte_same(huge_ptep_get(ptep), pte))) 4116 goto retry_avoidcopy; 4117 /* 4118 * race occurs while re-acquiring page table 4119 * lock, and our job is done. 4120 */ 4121 return 0; 4122 } 4123 4124 ret = vmf_error(PTR_ERR(new_page)); 4125 goto out_release_old; 4126 } 4127 4128 /* 4129 * When the original hugepage is shared one, it does not have 4130 * anon_vma prepared. 4131 */ 4132 if (unlikely(anon_vma_prepare(vma))) { 4133 ret = VM_FAULT_OOM; 4134 goto out_release_all; 4135 } 4136 4137 copy_user_huge_page(new_page, old_page, address, vma, 4138 pages_per_huge_page(h)); 4139 __SetPageUptodate(new_page); 4140 4141 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, mm, haddr, 4142 haddr + huge_page_size(h)); 4143 mmu_notifier_invalidate_range_start(&range); 4144 4145 /* 4146 * Retake the page table lock to check for racing updates 4147 * before the page tables are altered 4148 */ 4149 spin_lock(ptl); 4150 ptep = huge_pte_offset(mm, haddr, huge_page_size(h)); 4151 if (likely(ptep && pte_same(huge_ptep_get(ptep), pte))) { 4152 ClearPagePrivate(new_page); 4153 4154 /* Break COW */ 4155 huge_ptep_clear_flush(vma, haddr, ptep); 4156 mmu_notifier_invalidate_range(mm, range.start, range.end); 4157 set_huge_pte_at(mm, haddr, ptep, 4158 make_huge_pte(vma, new_page, 1)); 4159 page_remove_rmap(old_page, true); 4160 hugepage_add_new_anon_rmap(new_page, vma, haddr); 4161 set_page_huge_active(new_page); 4162 /* Make the old page be freed below */ 4163 new_page = old_page; 4164 } 4165 spin_unlock(ptl); 4166 mmu_notifier_invalidate_range_end(&range); 4167 out_release_all: 4168 restore_reserve_on_error(h, vma, haddr, new_page); 4169 put_page(new_page); 4170 out_release_old: 4171 put_page(old_page); 4172 4173 spin_lock(ptl); /* Caller expects lock to be held */ 4174 return ret; 4175 } 4176 4177 /* Return the pagecache page at a given address within a VMA */ 4178 static struct page *hugetlbfs_pagecache_page(struct hstate *h, 4179 struct vm_area_struct *vma, unsigned long address) 4180 { 4181 struct address_space *mapping; 4182 pgoff_t idx; 4183 4184 mapping = vma->vm_file->f_mapping; 4185 idx = vma_hugecache_offset(h, vma, address); 4186 4187 return find_lock_page(mapping, idx); 4188 } 4189 4190 /* 4191 * Return whether there is a pagecache page to back given address within VMA. 4192 * Caller follow_hugetlb_page() holds page_table_lock so we cannot lock_page. 4193 */ 4194 static bool hugetlbfs_pagecache_present(struct hstate *h, 4195 struct vm_area_struct *vma, unsigned long address) 4196 { 4197 struct address_space *mapping; 4198 pgoff_t idx; 4199 struct page *page; 4200 4201 mapping = vma->vm_file->f_mapping; 4202 idx = vma_hugecache_offset(h, vma, address); 4203 4204 page = find_get_page(mapping, idx); 4205 if (page) 4206 put_page(page); 4207 return page != NULL; 4208 } 4209 4210 int huge_add_to_page_cache(struct page *page, struct address_space *mapping, 4211 pgoff_t idx) 4212 { 4213 struct inode *inode = mapping->host; 4214 struct hstate *h = hstate_inode(inode); 4215 int err = add_to_page_cache(page, mapping, idx, GFP_KERNEL); 4216 4217 if (err) 4218 return err; 4219 ClearPagePrivate(page); 4220 4221 /* 4222 * set page dirty so that it will not be removed from cache/file 4223 * by non-hugetlbfs specific code paths. 4224 */ 4225 set_page_dirty(page); 4226 4227 spin_lock(&inode->i_lock); 4228 inode->i_blocks += blocks_per_huge_page(h); 4229 spin_unlock(&inode->i_lock); 4230 return 0; 4231 } 4232 4233 static vm_fault_t hugetlb_no_page(struct mm_struct *mm, 4234 struct vm_area_struct *vma, 4235 struct address_space *mapping, pgoff_t idx, 4236 unsigned long address, pte_t *ptep, unsigned int flags) 4237 { 4238 struct hstate *h = hstate_vma(vma); 4239 vm_fault_t ret = VM_FAULT_SIGBUS; 4240 int anon_rmap = 0; 4241 unsigned long size; 4242 struct page *page; 4243 pte_t new_pte; 4244 spinlock_t *ptl; 4245 unsigned long haddr = address & huge_page_mask(h); 4246 bool new_page = false; 4247 4248 /* 4249 * Currently, we are forced to kill the process in the event the 4250 * original mapper has unmapped pages from the child due to a failed 4251 * COW. Warn that such a situation has occurred as it may not be obvious 4252 */ 4253 if (is_vma_resv_set(vma, HPAGE_RESV_UNMAPPED)) { 4254 pr_warn_ratelimited("PID %d killed due to inadequate hugepage pool\n", 4255 current->pid); 4256 return ret; 4257 } 4258 4259 /* 4260 * We can not race with truncation due to holding i_mmap_rwsem. 4261 * i_size is modified when holding i_mmap_rwsem, so check here 4262 * once for faults beyond end of file. 4263 */ 4264 size = i_size_read(mapping->host) >> huge_page_shift(h); 4265 if (idx >= size) 4266 goto out; 4267 4268 retry: 4269 page = find_lock_page(mapping, idx); 4270 if (!page) { 4271 /* 4272 * Check for page in userfault range 4273 */ 4274 if (userfaultfd_missing(vma)) { 4275 u32 hash; 4276 struct vm_fault vmf = { 4277 .vma = vma, 4278 .address = haddr, 4279 .flags = flags, 4280 /* 4281 * Hard to debug if it ends up being 4282 * used by a callee that assumes 4283 * something about the other 4284 * uninitialized fields... same as in 4285 * memory.c 4286 */ 4287 }; 4288 4289 /* 4290 * hugetlb_fault_mutex and i_mmap_rwsem must be 4291 * dropped before handling userfault. Reacquire 4292 * after handling fault to make calling code simpler. 4293 */ 4294 hash = hugetlb_fault_mutex_hash(mapping, idx); 4295 mutex_unlock(&hugetlb_fault_mutex_table[hash]); 4296 i_mmap_unlock_read(mapping); 4297 ret = handle_userfault(&vmf, VM_UFFD_MISSING); 4298 i_mmap_lock_read(mapping); 4299 mutex_lock(&hugetlb_fault_mutex_table[hash]); 4300 goto out; 4301 } 4302 4303 page = alloc_huge_page(vma, haddr, 0); 4304 if (IS_ERR(page)) { 4305 /* 4306 * Returning error will result in faulting task being 4307 * sent SIGBUS. The hugetlb fault mutex prevents two 4308 * tasks from racing to fault in the same page which 4309 * could result in false unable to allocate errors. 4310 * Page migration does not take the fault mutex, but 4311 * does a clear then write of pte's under page table 4312 * lock. Page fault code could race with migration, 4313 * notice the clear pte and try to allocate a page 4314 * here. Before returning error, get ptl and make 4315 * sure there really is no pte entry. 4316 */ 4317 ptl = huge_pte_lock(h, mm, ptep); 4318 if (!huge_pte_none(huge_ptep_get(ptep))) { 4319 ret = 0; 4320 spin_unlock(ptl); 4321 goto out; 4322 } 4323 spin_unlock(ptl); 4324 ret = vmf_error(PTR_ERR(page)); 4325 goto out; 4326 } 4327 clear_huge_page(page, address, pages_per_huge_page(h)); 4328 __SetPageUptodate(page); 4329 new_page = true; 4330 4331 if (vma->vm_flags & VM_MAYSHARE) { 4332 int err = huge_add_to_page_cache(page, mapping, idx); 4333 if (err) { 4334 put_page(page); 4335 if (err == -EEXIST) 4336 goto retry; 4337 goto out; 4338 } 4339 } else { 4340 lock_page(page); 4341 if (unlikely(anon_vma_prepare(vma))) { 4342 ret = VM_FAULT_OOM; 4343 goto backout_unlocked; 4344 } 4345 anon_rmap = 1; 4346 } 4347 } else { 4348 /* 4349 * If memory error occurs between mmap() and fault, some process 4350 * don't have hwpoisoned swap entry for errored virtual address. 4351 * So we need to block hugepage fault by PG_hwpoison bit check. 4352 */ 4353 if (unlikely(PageHWPoison(page))) { 4354 ret = VM_FAULT_HWPOISON | 4355 VM_FAULT_SET_HINDEX(hstate_index(h)); 4356 goto backout_unlocked; 4357 } 4358 } 4359 4360 /* 4361 * If we are going to COW a private mapping later, we examine the 4362 * pending reservations for this page now. This will ensure that 4363 * any allocations necessary to record that reservation occur outside 4364 * the spinlock. 4365 */ 4366 if ((flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED)) { 4367 if (vma_needs_reservation(h, vma, haddr) < 0) { 4368 ret = VM_FAULT_OOM; 4369 goto backout_unlocked; 4370 } 4371 /* Just decrements count, does not deallocate */ 4372 vma_end_reservation(h, vma, haddr); 4373 } 4374 4375 ptl = huge_pte_lock(h, mm, ptep); 4376 ret = 0; 4377 if (!huge_pte_none(huge_ptep_get(ptep))) 4378 goto backout; 4379 4380 if (anon_rmap) { 4381 ClearPagePrivate(page); 4382 hugepage_add_new_anon_rmap(page, vma, haddr); 4383 } else 4384 page_dup_rmap(page, true); 4385 new_pte = make_huge_pte(vma, page, ((vma->vm_flags & VM_WRITE) 4386 && (vma->vm_flags & VM_SHARED))); 4387 set_huge_pte_at(mm, haddr, ptep, new_pte); 4388 4389 hugetlb_count_add(pages_per_huge_page(h), mm); 4390 if ((flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED)) { 4391 /* Optimization, do the COW without a second fault */ 4392 ret = hugetlb_cow(mm, vma, address, ptep, page, ptl); 4393 } 4394 4395 spin_unlock(ptl); 4396 4397 /* 4398 * Only make newly allocated pages active. Existing pages found 4399 * in the pagecache could be !page_huge_active() if they have been 4400 * isolated for migration. 4401 */ 4402 if (new_page) 4403 set_page_huge_active(page); 4404 4405 unlock_page(page); 4406 out: 4407 return ret; 4408 4409 backout: 4410 spin_unlock(ptl); 4411 backout_unlocked: 4412 unlock_page(page); 4413 restore_reserve_on_error(h, vma, haddr, page); 4414 put_page(page); 4415 goto out; 4416 } 4417 4418 #ifdef CONFIG_SMP 4419 u32 hugetlb_fault_mutex_hash(struct address_space *mapping, pgoff_t idx) 4420 { 4421 unsigned long key[2]; 4422 u32 hash; 4423 4424 key[0] = (unsigned long) mapping; 4425 key[1] = idx; 4426 4427 hash = jhash2((u32 *)&key, sizeof(key)/(sizeof(u32)), 0); 4428 4429 return hash & (num_fault_mutexes - 1); 4430 } 4431 #else 4432 /* 4433 * For uniprocesor systems we always use a single mutex, so just 4434 * return 0 and avoid the hashing overhead. 4435 */ 4436 u32 hugetlb_fault_mutex_hash(struct address_space *mapping, pgoff_t idx) 4437 { 4438 return 0; 4439 } 4440 #endif 4441 4442 vm_fault_t hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma, 4443 unsigned long address, unsigned int flags) 4444 { 4445 pte_t *ptep, entry; 4446 spinlock_t *ptl; 4447 vm_fault_t ret; 4448 u32 hash; 4449 pgoff_t idx; 4450 struct page *page = NULL; 4451 struct page *pagecache_page = NULL; 4452 struct hstate *h = hstate_vma(vma); 4453 struct address_space *mapping; 4454 int need_wait_lock = 0; 4455 unsigned long haddr = address & huge_page_mask(h); 4456 4457 ptep = huge_pte_offset(mm, haddr, huge_page_size(h)); 4458 if (ptep) { 4459 /* 4460 * Since we hold no locks, ptep could be stale. That is 4461 * OK as we are only making decisions based on content and 4462 * not actually modifying content here. 4463 */ 4464 entry = huge_ptep_get(ptep); 4465 if (unlikely(is_hugetlb_entry_migration(entry))) { 4466 migration_entry_wait_huge(vma, mm, ptep); 4467 return 0; 4468 } else if (unlikely(is_hugetlb_entry_hwpoisoned(entry))) 4469 return VM_FAULT_HWPOISON_LARGE | 4470 VM_FAULT_SET_HINDEX(hstate_index(h)); 4471 } 4472 4473 /* 4474 * Acquire i_mmap_rwsem before calling huge_pte_alloc and hold 4475 * until finished with ptep. This serves two purposes: 4476 * 1) It prevents huge_pmd_unshare from being called elsewhere 4477 * and making the ptep no longer valid. 4478 * 2) It synchronizes us with i_size modifications during truncation. 4479 * 4480 * ptep could have already be assigned via huge_pte_offset. That 4481 * is OK, as huge_pte_alloc will return the same value unless 4482 * something has changed. 4483 */ 4484 mapping = vma->vm_file->f_mapping; 4485 i_mmap_lock_read(mapping); 4486 ptep = huge_pte_alloc(mm, haddr, huge_page_size(h)); 4487 if (!ptep) { 4488 i_mmap_unlock_read(mapping); 4489 return VM_FAULT_OOM; 4490 } 4491 4492 /* 4493 * Serialize hugepage allocation and instantiation, so that we don't 4494 * get spurious allocation failures if two CPUs race to instantiate 4495 * the same page in the page cache. 4496 */ 4497 idx = vma_hugecache_offset(h, vma, haddr); 4498 hash = hugetlb_fault_mutex_hash(mapping, idx); 4499 mutex_lock(&hugetlb_fault_mutex_table[hash]); 4500 4501 entry = huge_ptep_get(ptep); 4502 if (huge_pte_none(entry)) { 4503 ret = hugetlb_no_page(mm, vma, mapping, idx, address, ptep, flags); 4504 goto out_mutex; 4505 } 4506 4507 ret = 0; 4508 4509 /* 4510 * entry could be a migration/hwpoison entry at this point, so this 4511 * check prevents the kernel from going below assuming that we have 4512 * an active hugepage in pagecache. This goto expects the 2nd page 4513 * fault, and is_hugetlb_entry_(migration|hwpoisoned) check will 4514 * properly handle it. 4515 */ 4516 if (!pte_present(entry)) 4517 goto out_mutex; 4518 4519 /* 4520 * If we are going to COW the mapping later, we examine the pending 4521 * reservations for this page now. This will ensure that any 4522 * allocations necessary to record that reservation occur outside the 4523 * spinlock. For private mappings, we also lookup the pagecache 4524 * page now as it is used to determine if a reservation has been 4525 * consumed. 4526 */ 4527 if ((flags & FAULT_FLAG_WRITE) && !huge_pte_write(entry)) { 4528 if (vma_needs_reservation(h, vma, haddr) < 0) { 4529 ret = VM_FAULT_OOM; 4530 goto out_mutex; 4531 } 4532 /* Just decrements count, does not deallocate */ 4533 vma_end_reservation(h, vma, haddr); 4534 4535 if (!(vma->vm_flags & VM_MAYSHARE)) 4536 pagecache_page = hugetlbfs_pagecache_page(h, 4537 vma, haddr); 4538 } 4539 4540 ptl = huge_pte_lock(h, mm, ptep); 4541 4542 /* Check for a racing update before calling hugetlb_cow */ 4543 if (unlikely(!pte_same(entry, huge_ptep_get(ptep)))) 4544 goto out_ptl; 4545 4546 /* 4547 * hugetlb_cow() requires page locks of pte_page(entry) and 4548 * pagecache_page, so here we need take the former one 4549 * when page != pagecache_page or !pagecache_page. 4550 */ 4551 page = pte_page(entry); 4552 if (page != pagecache_page) 4553 if (!trylock_page(page)) { 4554 need_wait_lock = 1; 4555 goto out_ptl; 4556 } 4557 4558 get_page(page); 4559 4560 if (flags & FAULT_FLAG_WRITE) { 4561 if (!huge_pte_write(entry)) { 4562 ret = hugetlb_cow(mm, vma, address, ptep, 4563 pagecache_page, ptl); 4564 goto out_put_page; 4565 } 4566 entry = huge_pte_mkdirty(entry); 4567 } 4568 entry = pte_mkyoung(entry); 4569 if (huge_ptep_set_access_flags(vma, haddr, ptep, entry, 4570 flags & FAULT_FLAG_WRITE)) 4571 update_mmu_cache(vma, haddr, ptep); 4572 out_put_page: 4573 if (page != pagecache_page) 4574 unlock_page(page); 4575 put_page(page); 4576 out_ptl: 4577 spin_unlock(ptl); 4578 4579 if (pagecache_page) { 4580 unlock_page(pagecache_page); 4581 put_page(pagecache_page); 4582 } 4583 out_mutex: 4584 mutex_unlock(&hugetlb_fault_mutex_table[hash]); 4585 i_mmap_unlock_read(mapping); 4586 /* 4587 * Generally it's safe to hold refcount during waiting page lock. But 4588 * here we just wait to defer the next page fault to avoid busy loop and 4589 * the page is not used after unlocked before returning from the current 4590 * page fault. So we are safe from accessing freed page, even if we wait 4591 * here without taking refcount. 4592 */ 4593 if (need_wait_lock) 4594 wait_on_page_locked(page); 4595 return ret; 4596 } 4597 4598 /* 4599 * Used by userfaultfd UFFDIO_COPY. Based on mcopy_atomic_pte with 4600 * modifications for huge pages. 4601 */ 4602 int hugetlb_mcopy_atomic_pte(struct mm_struct *dst_mm, 4603 pte_t *dst_pte, 4604 struct vm_area_struct *dst_vma, 4605 unsigned long dst_addr, 4606 unsigned long src_addr, 4607 struct page **pagep) 4608 { 4609 struct address_space *mapping; 4610 pgoff_t idx; 4611 unsigned long size; 4612 int vm_shared = dst_vma->vm_flags & VM_SHARED; 4613 struct hstate *h = hstate_vma(dst_vma); 4614 pte_t _dst_pte; 4615 spinlock_t *ptl; 4616 int ret; 4617 struct page *page; 4618 4619 if (!*pagep) { 4620 ret = -ENOMEM; 4621 page = alloc_huge_page(dst_vma, dst_addr, 0); 4622 if (IS_ERR(page)) 4623 goto out; 4624 4625 ret = copy_huge_page_from_user(page, 4626 (const void __user *) src_addr, 4627 pages_per_huge_page(h), false); 4628 4629 /* fallback to copy_from_user outside mmap_lock */ 4630 if (unlikely(ret)) { 4631 ret = -ENOENT; 4632 *pagep = page; 4633 /* don't free the page */ 4634 goto out; 4635 } 4636 } else { 4637 page = *pagep; 4638 *pagep = NULL; 4639 } 4640 4641 /* 4642 * The memory barrier inside __SetPageUptodate makes sure that 4643 * preceding stores to the page contents become visible before 4644 * the set_pte_at() write. 4645 */ 4646 __SetPageUptodate(page); 4647 4648 mapping = dst_vma->vm_file->f_mapping; 4649 idx = vma_hugecache_offset(h, dst_vma, dst_addr); 4650 4651 /* 4652 * If shared, add to page cache 4653 */ 4654 if (vm_shared) { 4655 size = i_size_read(mapping->host) >> huge_page_shift(h); 4656 ret = -EFAULT; 4657 if (idx >= size) 4658 goto out_release_nounlock; 4659 4660 /* 4661 * Serialization between remove_inode_hugepages() and 4662 * huge_add_to_page_cache() below happens through the 4663 * hugetlb_fault_mutex_table that here must be hold by 4664 * the caller. 4665 */ 4666 ret = huge_add_to_page_cache(page, mapping, idx); 4667 if (ret) 4668 goto out_release_nounlock; 4669 } 4670 4671 ptl = huge_pte_lockptr(h, dst_mm, dst_pte); 4672 spin_lock(ptl); 4673 4674 /* 4675 * Recheck the i_size after holding PT lock to make sure not 4676 * to leave any page mapped (as page_mapped()) beyond the end 4677 * of the i_size (remove_inode_hugepages() is strict about 4678 * enforcing that). If we bail out here, we'll also leave a 4679 * page in the radix tree in the vm_shared case beyond the end 4680 * of the i_size, but remove_inode_hugepages() will take care 4681 * of it as soon as we drop the hugetlb_fault_mutex_table. 4682 */ 4683 size = i_size_read(mapping->host) >> huge_page_shift(h); 4684 ret = -EFAULT; 4685 if (idx >= size) 4686 goto out_release_unlock; 4687 4688 ret = -EEXIST; 4689 if (!huge_pte_none(huge_ptep_get(dst_pte))) 4690 goto out_release_unlock; 4691 4692 if (vm_shared) { 4693 page_dup_rmap(page, true); 4694 } else { 4695 ClearPagePrivate(page); 4696 hugepage_add_new_anon_rmap(page, dst_vma, dst_addr); 4697 } 4698 4699 _dst_pte = make_huge_pte(dst_vma, page, dst_vma->vm_flags & VM_WRITE); 4700 if (dst_vma->vm_flags & VM_WRITE) 4701 _dst_pte = huge_pte_mkdirty(_dst_pte); 4702 _dst_pte = pte_mkyoung(_dst_pte); 4703 4704 set_huge_pte_at(dst_mm, dst_addr, dst_pte, _dst_pte); 4705 4706 (void)huge_ptep_set_access_flags(dst_vma, dst_addr, dst_pte, _dst_pte, 4707 dst_vma->vm_flags & VM_WRITE); 4708 hugetlb_count_add(pages_per_huge_page(h), dst_mm); 4709 4710 /* No need to invalidate - it was non-present before */ 4711 update_mmu_cache(dst_vma, dst_addr, dst_pte); 4712 4713 spin_unlock(ptl); 4714 set_page_huge_active(page); 4715 if (vm_shared) 4716 unlock_page(page); 4717 ret = 0; 4718 out: 4719 return ret; 4720 out_release_unlock: 4721 spin_unlock(ptl); 4722 if (vm_shared) 4723 unlock_page(page); 4724 out_release_nounlock: 4725 put_page(page); 4726 goto out; 4727 } 4728 4729 long follow_hugetlb_page(struct mm_struct *mm, struct vm_area_struct *vma, 4730 struct page **pages, struct vm_area_struct **vmas, 4731 unsigned long *position, unsigned long *nr_pages, 4732 long i, unsigned int flags, int *locked) 4733 { 4734 unsigned long pfn_offset; 4735 unsigned long vaddr = *position; 4736 unsigned long remainder = *nr_pages; 4737 struct hstate *h = hstate_vma(vma); 4738 int err = -EFAULT; 4739 4740 while (vaddr < vma->vm_end && remainder) { 4741 pte_t *pte; 4742 spinlock_t *ptl = NULL; 4743 int absent; 4744 struct page *page; 4745 4746 /* 4747 * If we have a pending SIGKILL, don't keep faulting pages and 4748 * potentially allocating memory. 4749 */ 4750 if (fatal_signal_pending(current)) { 4751 remainder = 0; 4752 break; 4753 } 4754 4755 /* 4756 * Some archs (sparc64, sh*) have multiple pte_ts to 4757 * each hugepage. We have to make sure we get the 4758 * first, for the page indexing below to work. 4759 * 4760 * Note that page table lock is not held when pte is null. 4761 */ 4762 pte = huge_pte_offset(mm, vaddr & huge_page_mask(h), 4763 huge_page_size(h)); 4764 if (pte) 4765 ptl = huge_pte_lock(h, mm, pte); 4766 absent = !pte || huge_pte_none(huge_ptep_get(pte)); 4767 4768 /* 4769 * When coredumping, it suits get_dump_page if we just return 4770 * an error where there's an empty slot with no huge pagecache 4771 * to back it. This way, we avoid allocating a hugepage, and 4772 * the sparse dumpfile avoids allocating disk blocks, but its 4773 * huge holes still show up with zeroes where they need to be. 4774 */ 4775 if (absent && (flags & FOLL_DUMP) && 4776 !hugetlbfs_pagecache_present(h, vma, vaddr)) { 4777 if (pte) 4778 spin_unlock(ptl); 4779 remainder = 0; 4780 break; 4781 } 4782 4783 /* 4784 * We need call hugetlb_fault for both hugepages under migration 4785 * (in which case hugetlb_fault waits for the migration,) and 4786 * hwpoisoned hugepages (in which case we need to prevent the 4787 * caller from accessing to them.) In order to do this, we use 4788 * here is_swap_pte instead of is_hugetlb_entry_migration and 4789 * is_hugetlb_entry_hwpoisoned. This is because it simply covers 4790 * both cases, and because we can't follow correct pages 4791 * directly from any kind of swap entries. 4792 */ 4793 if (absent || is_swap_pte(huge_ptep_get(pte)) || 4794 ((flags & FOLL_WRITE) && 4795 !huge_pte_write(huge_ptep_get(pte)))) { 4796 vm_fault_t ret; 4797 unsigned int fault_flags = 0; 4798 4799 if (pte) 4800 spin_unlock(ptl); 4801 if (flags & FOLL_WRITE) 4802 fault_flags |= FAULT_FLAG_WRITE; 4803 if (locked) 4804 fault_flags |= FAULT_FLAG_ALLOW_RETRY | 4805 FAULT_FLAG_KILLABLE; 4806 if (flags & FOLL_NOWAIT) 4807 fault_flags |= FAULT_FLAG_ALLOW_RETRY | 4808 FAULT_FLAG_RETRY_NOWAIT; 4809 if (flags & FOLL_TRIED) { 4810 /* 4811 * Note: FAULT_FLAG_ALLOW_RETRY and 4812 * FAULT_FLAG_TRIED can co-exist 4813 */ 4814 fault_flags |= FAULT_FLAG_TRIED; 4815 } 4816 ret = hugetlb_fault(mm, vma, vaddr, fault_flags); 4817 if (ret & VM_FAULT_ERROR) { 4818 err = vm_fault_to_errno(ret, flags); 4819 remainder = 0; 4820 break; 4821 } 4822 if (ret & VM_FAULT_RETRY) { 4823 if (locked && 4824 !(fault_flags & FAULT_FLAG_RETRY_NOWAIT)) 4825 *locked = 0; 4826 *nr_pages = 0; 4827 /* 4828 * VM_FAULT_RETRY must not return an 4829 * error, it will return zero 4830 * instead. 4831 * 4832 * No need to update "position" as the 4833 * caller will not check it after 4834 * *nr_pages is set to 0. 4835 */ 4836 return i; 4837 } 4838 continue; 4839 } 4840 4841 pfn_offset = (vaddr & ~huge_page_mask(h)) >> PAGE_SHIFT; 4842 page = pte_page(huge_ptep_get(pte)); 4843 4844 /* 4845 * If subpage information not requested, update counters 4846 * and skip the same_page loop below. 4847 */ 4848 if (!pages && !vmas && !pfn_offset && 4849 (vaddr + huge_page_size(h) < vma->vm_end) && 4850 (remainder >= pages_per_huge_page(h))) { 4851 vaddr += huge_page_size(h); 4852 remainder -= pages_per_huge_page(h); 4853 i += pages_per_huge_page(h); 4854 spin_unlock(ptl); 4855 continue; 4856 } 4857 4858 same_page: 4859 if (pages) { 4860 pages[i] = mem_map_offset(page, pfn_offset); 4861 /* 4862 * try_grab_page() should always succeed here, because: 4863 * a) we hold the ptl lock, and b) we've just checked 4864 * that the huge page is present in the page tables. If 4865 * the huge page is present, then the tail pages must 4866 * also be present. The ptl prevents the head page and 4867 * tail pages from being rearranged in any way. So this 4868 * page must be available at this point, unless the page 4869 * refcount overflowed: 4870 */ 4871 if (WARN_ON_ONCE(!try_grab_page(pages[i], flags))) { 4872 spin_unlock(ptl); 4873 remainder = 0; 4874 err = -ENOMEM; 4875 break; 4876 } 4877 } 4878 4879 if (vmas) 4880 vmas[i] = vma; 4881 4882 vaddr += PAGE_SIZE; 4883 ++pfn_offset; 4884 --remainder; 4885 ++i; 4886 if (vaddr < vma->vm_end && remainder && 4887 pfn_offset < pages_per_huge_page(h)) { 4888 /* 4889 * We use pfn_offset to avoid touching the pageframes 4890 * of this compound page. 4891 */ 4892 goto same_page; 4893 } 4894 spin_unlock(ptl); 4895 } 4896 *nr_pages = remainder; 4897 /* 4898 * setting position is actually required only if remainder is 4899 * not zero but it's faster not to add a "if (remainder)" 4900 * branch. 4901 */ 4902 *position = vaddr; 4903 4904 return i ? i : err; 4905 } 4906 4907 #ifndef __HAVE_ARCH_FLUSH_HUGETLB_TLB_RANGE 4908 /* 4909 * ARCHes with special requirements for evicting HUGETLB backing TLB entries can 4910 * implement this. 4911 */ 4912 #define flush_hugetlb_tlb_range(vma, addr, end) flush_tlb_range(vma, addr, end) 4913 #endif 4914 4915 unsigned long hugetlb_change_protection(struct vm_area_struct *vma, 4916 unsigned long address, unsigned long end, pgprot_t newprot) 4917 { 4918 struct mm_struct *mm = vma->vm_mm; 4919 unsigned long start = address; 4920 pte_t *ptep; 4921 pte_t pte; 4922 struct hstate *h = hstate_vma(vma); 4923 unsigned long pages = 0; 4924 bool shared_pmd = false; 4925 struct mmu_notifier_range range; 4926 4927 /* 4928 * In the case of shared PMDs, the area to flush could be beyond 4929 * start/end. Set range.start/range.end to cover the maximum possible 4930 * range if PMD sharing is possible. 4931 */ 4932 mmu_notifier_range_init(&range, MMU_NOTIFY_PROTECTION_VMA, 4933 0, vma, mm, start, end); 4934 adjust_range_if_pmd_sharing_possible(vma, &range.start, &range.end); 4935 4936 BUG_ON(address >= end); 4937 flush_cache_range(vma, range.start, range.end); 4938 4939 mmu_notifier_invalidate_range_start(&range); 4940 i_mmap_lock_write(vma->vm_file->f_mapping); 4941 for (; address < end; address += huge_page_size(h)) { 4942 spinlock_t *ptl; 4943 ptep = huge_pte_offset(mm, address, huge_page_size(h)); 4944 if (!ptep) 4945 continue; 4946 ptl = huge_pte_lock(h, mm, ptep); 4947 if (huge_pmd_unshare(mm, vma, &address, ptep)) { 4948 pages++; 4949 spin_unlock(ptl); 4950 shared_pmd = true; 4951 continue; 4952 } 4953 pte = huge_ptep_get(ptep); 4954 if (unlikely(is_hugetlb_entry_hwpoisoned(pte))) { 4955 spin_unlock(ptl); 4956 continue; 4957 } 4958 if (unlikely(is_hugetlb_entry_migration(pte))) { 4959 swp_entry_t entry = pte_to_swp_entry(pte); 4960 4961 if (is_write_migration_entry(entry)) { 4962 pte_t newpte; 4963 4964 make_migration_entry_read(&entry); 4965 newpte = swp_entry_to_pte(entry); 4966 set_huge_swap_pte_at(mm, address, ptep, 4967 newpte, huge_page_size(h)); 4968 pages++; 4969 } 4970 spin_unlock(ptl); 4971 continue; 4972 } 4973 if (!huge_pte_none(pte)) { 4974 pte_t old_pte; 4975 4976 old_pte = huge_ptep_modify_prot_start(vma, address, ptep); 4977 pte = pte_mkhuge(huge_pte_modify(old_pte, newprot)); 4978 pte = arch_make_huge_pte(pte, vma, NULL, 0); 4979 huge_ptep_modify_prot_commit(vma, address, ptep, old_pte, pte); 4980 pages++; 4981 } 4982 spin_unlock(ptl); 4983 } 4984 /* 4985 * Must flush TLB before releasing i_mmap_rwsem: x86's huge_pmd_unshare 4986 * may have cleared our pud entry and done put_page on the page table: 4987 * once we release i_mmap_rwsem, another task can do the final put_page 4988 * and that page table be reused and filled with junk. If we actually 4989 * did unshare a page of pmds, flush the range corresponding to the pud. 4990 */ 4991 if (shared_pmd) 4992 flush_hugetlb_tlb_range(vma, range.start, range.end); 4993 else 4994 flush_hugetlb_tlb_range(vma, start, end); 4995 /* 4996 * No need to call mmu_notifier_invalidate_range() we are downgrading 4997 * page table protection not changing it to point to a new page. 4998 * 4999 * See Documentation/vm/mmu_notifier.rst 5000 */ 5001 i_mmap_unlock_write(vma->vm_file->f_mapping); 5002 mmu_notifier_invalidate_range_end(&range); 5003 5004 return pages << h->order; 5005 } 5006 5007 int hugetlb_reserve_pages(struct inode *inode, 5008 long from, long to, 5009 struct vm_area_struct *vma, 5010 vm_flags_t vm_flags) 5011 { 5012 long ret, chg, add = -1; 5013 struct hstate *h = hstate_inode(inode); 5014 struct hugepage_subpool *spool = subpool_inode(inode); 5015 struct resv_map *resv_map; 5016 struct hugetlb_cgroup *h_cg = NULL; 5017 long gbl_reserve, regions_needed = 0; 5018 5019 /* This should never happen */ 5020 if (from > to) { 5021 VM_WARN(1, "%s called with a negative range\n", __func__); 5022 return -EINVAL; 5023 } 5024 5025 /* 5026 * Only apply hugepage reservation if asked. At fault time, an 5027 * attempt will be made for VM_NORESERVE to allocate a page 5028 * without using reserves 5029 */ 5030 if (vm_flags & VM_NORESERVE) 5031 return 0; 5032 5033 /* 5034 * Shared mappings base their reservation on the number of pages that 5035 * are already allocated on behalf of the file. Private mappings need 5036 * to reserve the full area even if read-only as mprotect() may be 5037 * called to make the mapping read-write. Assume !vma is a shm mapping 5038 */ 5039 if (!vma || vma->vm_flags & VM_MAYSHARE) { 5040 /* 5041 * resv_map can not be NULL as hugetlb_reserve_pages is only 5042 * called for inodes for which resv_maps were created (see 5043 * hugetlbfs_get_inode). 5044 */ 5045 resv_map = inode_resv_map(inode); 5046 5047 chg = region_chg(resv_map, from, to, ®ions_needed); 5048 5049 } else { 5050 /* Private mapping. */ 5051 resv_map = resv_map_alloc(); 5052 if (!resv_map) 5053 return -ENOMEM; 5054 5055 chg = to - from; 5056 5057 set_vma_resv_map(vma, resv_map); 5058 set_vma_resv_flags(vma, HPAGE_RESV_OWNER); 5059 } 5060 5061 if (chg < 0) { 5062 ret = chg; 5063 goto out_err; 5064 } 5065 5066 ret = hugetlb_cgroup_charge_cgroup_rsvd( 5067 hstate_index(h), chg * pages_per_huge_page(h), &h_cg); 5068 5069 if (ret < 0) { 5070 ret = -ENOMEM; 5071 goto out_err; 5072 } 5073 5074 if (vma && !(vma->vm_flags & VM_MAYSHARE) && h_cg) { 5075 /* For private mappings, the hugetlb_cgroup uncharge info hangs 5076 * of the resv_map. 5077 */ 5078 resv_map_set_hugetlb_cgroup_uncharge_info(resv_map, h_cg, h); 5079 } 5080 5081 /* 5082 * There must be enough pages in the subpool for the mapping. If 5083 * the subpool has a minimum size, there may be some global 5084 * reservations already in place (gbl_reserve). 5085 */ 5086 gbl_reserve = hugepage_subpool_get_pages(spool, chg); 5087 if (gbl_reserve < 0) { 5088 ret = -ENOSPC; 5089 goto out_uncharge_cgroup; 5090 } 5091 5092 /* 5093 * Check enough hugepages are available for the reservation. 5094 * Hand the pages back to the subpool if there are not 5095 */ 5096 ret = hugetlb_acct_memory(h, gbl_reserve); 5097 if (ret < 0) { 5098 goto out_put_pages; 5099 } 5100 5101 /* 5102 * Account for the reservations made. Shared mappings record regions 5103 * that have reservations as they are shared by multiple VMAs. 5104 * When the last VMA disappears, the region map says how much 5105 * the reservation was and the page cache tells how much of 5106 * the reservation was consumed. Private mappings are per-VMA and 5107 * only the consumed reservations are tracked. When the VMA 5108 * disappears, the original reservation is the VMA size and the 5109 * consumed reservations are stored in the map. Hence, nothing 5110 * else has to be done for private mappings here 5111 */ 5112 if (!vma || vma->vm_flags & VM_MAYSHARE) { 5113 add = region_add(resv_map, from, to, regions_needed, h, h_cg); 5114 5115 if (unlikely(add < 0)) { 5116 hugetlb_acct_memory(h, -gbl_reserve); 5117 ret = add; 5118 goto out_put_pages; 5119 } else if (unlikely(chg > add)) { 5120 /* 5121 * pages in this range were added to the reserve 5122 * map between region_chg and region_add. This 5123 * indicates a race with alloc_huge_page. Adjust 5124 * the subpool and reserve counts modified above 5125 * based on the difference. 5126 */ 5127 long rsv_adjust; 5128 5129 hugetlb_cgroup_uncharge_cgroup_rsvd( 5130 hstate_index(h), 5131 (chg - add) * pages_per_huge_page(h), h_cg); 5132 5133 rsv_adjust = hugepage_subpool_put_pages(spool, 5134 chg - add); 5135 hugetlb_acct_memory(h, -rsv_adjust); 5136 } 5137 } 5138 return 0; 5139 out_put_pages: 5140 /* put back original number of pages, chg */ 5141 (void)hugepage_subpool_put_pages(spool, chg); 5142 out_uncharge_cgroup: 5143 hugetlb_cgroup_uncharge_cgroup_rsvd(hstate_index(h), 5144 chg * pages_per_huge_page(h), h_cg); 5145 out_err: 5146 if (!vma || vma->vm_flags & VM_MAYSHARE) 5147 /* Only call region_abort if the region_chg succeeded but the 5148 * region_add failed or didn't run. 5149 */ 5150 if (chg >= 0 && add < 0) 5151 region_abort(resv_map, from, to, regions_needed); 5152 if (vma && is_vma_resv_set(vma, HPAGE_RESV_OWNER)) 5153 kref_put(&resv_map->refs, resv_map_release); 5154 return ret; 5155 } 5156 5157 long hugetlb_unreserve_pages(struct inode *inode, long start, long end, 5158 long freed) 5159 { 5160 struct hstate *h = hstate_inode(inode); 5161 struct resv_map *resv_map = inode_resv_map(inode); 5162 long chg = 0; 5163 struct hugepage_subpool *spool = subpool_inode(inode); 5164 long gbl_reserve; 5165 5166 /* 5167 * Since this routine can be called in the evict inode path for all 5168 * hugetlbfs inodes, resv_map could be NULL. 5169 */ 5170 if (resv_map) { 5171 chg = region_del(resv_map, start, end); 5172 /* 5173 * region_del() can fail in the rare case where a region 5174 * must be split and another region descriptor can not be 5175 * allocated. If end == LONG_MAX, it will not fail. 5176 */ 5177 if (chg < 0) 5178 return chg; 5179 } 5180 5181 spin_lock(&inode->i_lock); 5182 inode->i_blocks -= (blocks_per_huge_page(h) * freed); 5183 spin_unlock(&inode->i_lock); 5184 5185 /* 5186 * If the subpool has a minimum size, the number of global 5187 * reservations to be released may be adjusted. 5188 */ 5189 gbl_reserve = hugepage_subpool_put_pages(spool, (chg - freed)); 5190 hugetlb_acct_memory(h, -gbl_reserve); 5191 5192 return 0; 5193 } 5194 5195 #ifdef CONFIG_ARCH_WANT_HUGE_PMD_SHARE 5196 static unsigned long page_table_shareable(struct vm_area_struct *svma, 5197 struct vm_area_struct *vma, 5198 unsigned long addr, pgoff_t idx) 5199 { 5200 unsigned long saddr = ((idx - svma->vm_pgoff) << PAGE_SHIFT) + 5201 svma->vm_start; 5202 unsigned long sbase = saddr & PUD_MASK; 5203 unsigned long s_end = sbase + PUD_SIZE; 5204 5205 /* Allow segments to share if only one is marked locked */ 5206 unsigned long vm_flags = vma->vm_flags & VM_LOCKED_CLEAR_MASK; 5207 unsigned long svm_flags = svma->vm_flags & VM_LOCKED_CLEAR_MASK; 5208 5209 /* 5210 * match the virtual addresses, permission and the alignment of the 5211 * page table page. 5212 */ 5213 if (pmd_index(addr) != pmd_index(saddr) || 5214 vm_flags != svm_flags || 5215 sbase < svma->vm_start || svma->vm_end < s_end) 5216 return 0; 5217 5218 return saddr; 5219 } 5220 5221 static bool vma_shareable(struct vm_area_struct *vma, unsigned long addr) 5222 { 5223 unsigned long base = addr & PUD_MASK; 5224 unsigned long end = base + PUD_SIZE; 5225 5226 /* 5227 * check on proper vm_flags and page table alignment 5228 */ 5229 if (vma->vm_flags & VM_MAYSHARE && range_in_vma(vma, base, end)) 5230 return true; 5231 return false; 5232 } 5233 5234 /* 5235 * Determine if start,end range within vma could be mapped by shared pmd. 5236 * If yes, adjust start and end to cover range associated with possible 5237 * shared pmd mappings. 5238 */ 5239 void adjust_range_if_pmd_sharing_possible(struct vm_area_struct *vma, 5240 unsigned long *start, unsigned long *end) 5241 { 5242 unsigned long a_start, a_end; 5243 5244 if (!(vma->vm_flags & VM_MAYSHARE)) 5245 return; 5246 5247 /* Extend the range to be PUD aligned for a worst case scenario */ 5248 a_start = ALIGN_DOWN(*start, PUD_SIZE); 5249 a_end = ALIGN(*end, PUD_SIZE); 5250 5251 /* 5252 * Intersect the range with the vma range, since pmd sharing won't be 5253 * across vma after all 5254 */ 5255 *start = max(vma->vm_start, a_start); 5256 *end = min(vma->vm_end, a_end); 5257 } 5258 5259 /* 5260 * Search for a shareable pmd page for hugetlb. In any case calls pmd_alloc() 5261 * and returns the corresponding pte. While this is not necessary for the 5262 * !shared pmd case because we can allocate the pmd later as well, it makes the 5263 * code much cleaner. 5264 * 5265 * This routine must be called with i_mmap_rwsem held in at least read mode if 5266 * sharing is possible. For hugetlbfs, this prevents removal of any page 5267 * table entries associated with the address space. This is important as we 5268 * are setting up sharing based on existing page table entries (mappings). 5269 * 5270 * NOTE: This routine is only called from huge_pte_alloc. Some callers of 5271 * huge_pte_alloc know that sharing is not possible and do not take 5272 * i_mmap_rwsem as a performance optimization. This is handled by the 5273 * if !vma_shareable check at the beginning of the routine. i_mmap_rwsem is 5274 * only required for subsequent processing. 5275 */ 5276 pte_t *huge_pmd_share(struct mm_struct *mm, unsigned long addr, pud_t *pud) 5277 { 5278 struct vm_area_struct *vma = find_vma(mm, addr); 5279 struct address_space *mapping = vma->vm_file->f_mapping; 5280 pgoff_t idx = ((addr - vma->vm_start) >> PAGE_SHIFT) + 5281 vma->vm_pgoff; 5282 struct vm_area_struct *svma; 5283 unsigned long saddr; 5284 pte_t *spte = NULL; 5285 pte_t *pte; 5286 spinlock_t *ptl; 5287 5288 if (!vma_shareable(vma, addr)) 5289 return (pte_t *)pmd_alloc(mm, pud, addr); 5290 5291 i_mmap_assert_locked(mapping); 5292 vma_interval_tree_foreach(svma, &mapping->i_mmap, idx, idx) { 5293 if (svma == vma) 5294 continue; 5295 5296 saddr = page_table_shareable(svma, vma, addr, idx); 5297 if (saddr) { 5298 spte = huge_pte_offset(svma->vm_mm, saddr, 5299 vma_mmu_pagesize(svma)); 5300 if (spte) { 5301 get_page(virt_to_page(spte)); 5302 break; 5303 } 5304 } 5305 } 5306 5307 if (!spte) 5308 goto out; 5309 5310 ptl = huge_pte_lock(hstate_vma(vma), mm, spte); 5311 if (pud_none(*pud)) { 5312 pud_populate(mm, pud, 5313 (pmd_t *)((unsigned long)spte & PAGE_MASK)); 5314 mm_inc_nr_pmds(mm); 5315 } else { 5316 put_page(virt_to_page(spte)); 5317 } 5318 spin_unlock(ptl); 5319 out: 5320 pte = (pte_t *)pmd_alloc(mm, pud, addr); 5321 return pte; 5322 } 5323 5324 /* 5325 * unmap huge page backed by shared pte. 5326 * 5327 * Hugetlb pte page is ref counted at the time of mapping. If pte is shared 5328 * indicated by page_count > 1, unmap is achieved by clearing pud and 5329 * decrementing the ref count. If count == 1, the pte page is not shared. 5330 * 5331 * Called with page table lock held and i_mmap_rwsem held in write mode. 5332 * 5333 * returns: 1 successfully unmapped a shared pte page 5334 * 0 the underlying pte page is not shared, or it is the last user 5335 */ 5336 int huge_pmd_unshare(struct mm_struct *mm, struct vm_area_struct *vma, 5337 unsigned long *addr, pte_t *ptep) 5338 { 5339 pgd_t *pgd = pgd_offset(mm, *addr); 5340 p4d_t *p4d = p4d_offset(pgd, *addr); 5341 pud_t *pud = pud_offset(p4d, *addr); 5342 5343 i_mmap_assert_write_locked(vma->vm_file->f_mapping); 5344 BUG_ON(page_count(virt_to_page(ptep)) == 0); 5345 if (page_count(virt_to_page(ptep)) == 1) 5346 return 0; 5347 5348 pud_clear(pud); 5349 put_page(virt_to_page(ptep)); 5350 mm_dec_nr_pmds(mm); 5351 *addr = ALIGN(*addr, HPAGE_SIZE * PTRS_PER_PTE) - HPAGE_SIZE; 5352 return 1; 5353 } 5354 #define want_pmd_share() (1) 5355 #else /* !CONFIG_ARCH_WANT_HUGE_PMD_SHARE */ 5356 pte_t *huge_pmd_share(struct mm_struct *mm, unsigned long addr, pud_t *pud) 5357 { 5358 return NULL; 5359 } 5360 5361 int huge_pmd_unshare(struct mm_struct *mm, struct vm_area_struct *vma, 5362 unsigned long *addr, pte_t *ptep) 5363 { 5364 return 0; 5365 } 5366 5367 void adjust_range_if_pmd_sharing_possible(struct vm_area_struct *vma, 5368 unsigned long *start, unsigned long *end) 5369 { 5370 } 5371 #define want_pmd_share() (0) 5372 #endif /* CONFIG_ARCH_WANT_HUGE_PMD_SHARE */ 5373 5374 #ifdef CONFIG_ARCH_WANT_GENERAL_HUGETLB 5375 pte_t *huge_pte_alloc(struct mm_struct *mm, 5376 unsigned long addr, unsigned long sz) 5377 { 5378 pgd_t *pgd; 5379 p4d_t *p4d; 5380 pud_t *pud; 5381 pte_t *pte = NULL; 5382 5383 pgd = pgd_offset(mm, addr); 5384 p4d = p4d_alloc(mm, pgd, addr); 5385 if (!p4d) 5386 return NULL; 5387 pud = pud_alloc(mm, p4d, addr); 5388 if (pud) { 5389 if (sz == PUD_SIZE) { 5390 pte = (pte_t *)pud; 5391 } else { 5392 BUG_ON(sz != PMD_SIZE); 5393 if (want_pmd_share() && pud_none(*pud)) 5394 pte = huge_pmd_share(mm, addr, pud); 5395 else 5396 pte = (pte_t *)pmd_alloc(mm, pud, addr); 5397 } 5398 } 5399 BUG_ON(pte && pte_present(*pte) && !pte_huge(*pte)); 5400 5401 return pte; 5402 } 5403 5404 /* 5405 * huge_pte_offset() - Walk the page table to resolve the hugepage 5406 * entry at address @addr 5407 * 5408 * Return: Pointer to page table entry (PUD or PMD) for 5409 * address @addr, or NULL if a !p*d_present() entry is encountered and the 5410 * size @sz doesn't match the hugepage size at this level of the page 5411 * table. 5412 */ 5413 pte_t *huge_pte_offset(struct mm_struct *mm, 5414 unsigned long addr, unsigned long sz) 5415 { 5416 pgd_t *pgd; 5417 p4d_t *p4d; 5418 pud_t *pud; 5419 pmd_t *pmd; 5420 5421 pgd = pgd_offset(mm, addr); 5422 if (!pgd_present(*pgd)) 5423 return NULL; 5424 p4d = p4d_offset(pgd, addr); 5425 if (!p4d_present(*p4d)) 5426 return NULL; 5427 5428 pud = pud_offset(p4d, addr); 5429 if (sz == PUD_SIZE) 5430 /* must be pud huge, non-present or none */ 5431 return (pte_t *)pud; 5432 if (!pud_present(*pud)) 5433 return NULL; 5434 /* must have a valid entry and size to go further */ 5435 5436 pmd = pmd_offset(pud, addr); 5437 /* must be pmd huge, non-present or none */ 5438 return (pte_t *)pmd; 5439 } 5440 5441 #endif /* CONFIG_ARCH_WANT_GENERAL_HUGETLB */ 5442 5443 /* 5444 * These functions are overwritable if your architecture needs its own 5445 * behavior. 5446 */ 5447 struct page * __weak 5448 follow_huge_addr(struct mm_struct *mm, unsigned long address, 5449 int write) 5450 { 5451 return ERR_PTR(-EINVAL); 5452 } 5453 5454 struct page * __weak 5455 follow_huge_pd(struct vm_area_struct *vma, 5456 unsigned long address, hugepd_t hpd, int flags, int pdshift) 5457 { 5458 WARN(1, "hugepd follow called with no support for hugepage directory format\n"); 5459 return NULL; 5460 } 5461 5462 struct page * __weak 5463 follow_huge_pmd(struct mm_struct *mm, unsigned long address, 5464 pmd_t *pmd, int flags) 5465 { 5466 struct page *page = NULL; 5467 spinlock_t *ptl; 5468 pte_t pte; 5469 5470 /* FOLL_GET and FOLL_PIN are mutually exclusive. */ 5471 if (WARN_ON_ONCE((flags & (FOLL_PIN | FOLL_GET)) == 5472 (FOLL_PIN | FOLL_GET))) 5473 return NULL; 5474 5475 retry: 5476 ptl = pmd_lockptr(mm, pmd); 5477 spin_lock(ptl); 5478 /* 5479 * make sure that the address range covered by this pmd is not 5480 * unmapped from other threads. 5481 */ 5482 if (!pmd_huge(*pmd)) 5483 goto out; 5484 pte = huge_ptep_get((pte_t *)pmd); 5485 if (pte_present(pte)) { 5486 page = pmd_page(*pmd) + ((address & ~PMD_MASK) >> PAGE_SHIFT); 5487 /* 5488 * try_grab_page() should always succeed here, because: a) we 5489 * hold the pmd (ptl) lock, and b) we've just checked that the 5490 * huge pmd (head) page is present in the page tables. The ptl 5491 * prevents the head page and tail pages from being rearranged 5492 * in any way. So this page must be available at this point, 5493 * unless the page refcount overflowed: 5494 */ 5495 if (WARN_ON_ONCE(!try_grab_page(page, flags))) { 5496 page = NULL; 5497 goto out; 5498 } 5499 } else { 5500 if (is_hugetlb_entry_migration(pte)) { 5501 spin_unlock(ptl); 5502 __migration_entry_wait(mm, (pte_t *)pmd, ptl); 5503 goto retry; 5504 } 5505 /* 5506 * hwpoisoned entry is treated as no_page_table in 5507 * follow_page_mask(). 5508 */ 5509 } 5510 out: 5511 spin_unlock(ptl); 5512 return page; 5513 } 5514 5515 struct page * __weak 5516 follow_huge_pud(struct mm_struct *mm, unsigned long address, 5517 pud_t *pud, int flags) 5518 { 5519 if (flags & (FOLL_GET | FOLL_PIN)) 5520 return NULL; 5521 5522 return pte_page(*(pte_t *)pud) + ((address & ~PUD_MASK) >> PAGE_SHIFT); 5523 } 5524 5525 struct page * __weak 5526 follow_huge_pgd(struct mm_struct *mm, unsigned long address, pgd_t *pgd, int flags) 5527 { 5528 if (flags & (FOLL_GET | FOLL_PIN)) 5529 return NULL; 5530 5531 return pte_page(*(pte_t *)pgd) + ((address & ~PGDIR_MASK) >> PAGE_SHIFT); 5532 } 5533 5534 bool isolate_huge_page(struct page *page, struct list_head *list) 5535 { 5536 bool ret = true; 5537 5538 VM_BUG_ON_PAGE(!PageHead(page), page); 5539 spin_lock(&hugetlb_lock); 5540 if (!page_huge_active(page) || !get_page_unless_zero(page)) { 5541 ret = false; 5542 goto unlock; 5543 } 5544 clear_page_huge_active(page); 5545 list_move_tail(&page->lru, list); 5546 unlock: 5547 spin_unlock(&hugetlb_lock); 5548 return ret; 5549 } 5550 5551 void putback_active_hugepage(struct page *page) 5552 { 5553 VM_BUG_ON_PAGE(!PageHead(page), page); 5554 spin_lock(&hugetlb_lock); 5555 set_page_huge_active(page); 5556 list_move_tail(&page->lru, &(page_hstate(page))->hugepage_activelist); 5557 spin_unlock(&hugetlb_lock); 5558 put_page(page); 5559 } 5560 5561 void move_hugetlb_state(struct page *oldpage, struct page *newpage, int reason) 5562 { 5563 struct hstate *h = page_hstate(oldpage); 5564 5565 hugetlb_cgroup_migrate(oldpage, newpage); 5566 set_page_owner_migrate_reason(newpage, reason); 5567 5568 /* 5569 * transfer temporary state of the new huge page. This is 5570 * reverse to other transitions because the newpage is going to 5571 * be final while the old one will be freed so it takes over 5572 * the temporary status. 5573 * 5574 * Also note that we have to transfer the per-node surplus state 5575 * here as well otherwise the global surplus count will not match 5576 * the per-node's. 5577 */ 5578 if (PageHugeTemporary(newpage)) { 5579 int old_nid = page_to_nid(oldpage); 5580 int new_nid = page_to_nid(newpage); 5581 5582 SetPageHugeTemporary(oldpage); 5583 ClearPageHugeTemporary(newpage); 5584 5585 spin_lock(&hugetlb_lock); 5586 if (h->surplus_huge_pages_node[old_nid]) { 5587 h->surplus_huge_pages_node[old_nid]--; 5588 h->surplus_huge_pages_node[new_nid]++; 5589 } 5590 spin_unlock(&hugetlb_lock); 5591 } 5592 } 5593 5594 #ifdef CONFIG_CMA 5595 static bool cma_reserve_called __initdata; 5596 5597 static int __init cmdline_parse_hugetlb_cma(char *p) 5598 { 5599 hugetlb_cma_size = memparse(p, &p); 5600 return 0; 5601 } 5602 5603 early_param("hugetlb_cma", cmdline_parse_hugetlb_cma); 5604 5605 void __init hugetlb_cma_reserve(int order) 5606 { 5607 unsigned long size, reserved, per_node; 5608 int nid; 5609 5610 cma_reserve_called = true; 5611 5612 if (!hugetlb_cma_size) 5613 return; 5614 5615 if (hugetlb_cma_size < (PAGE_SIZE << order)) { 5616 pr_warn("hugetlb_cma: cma area should be at least %lu MiB\n", 5617 (PAGE_SIZE << order) / SZ_1M); 5618 return; 5619 } 5620 5621 /* 5622 * If 3 GB area is requested on a machine with 4 numa nodes, 5623 * let's allocate 1 GB on first three nodes and ignore the last one. 5624 */ 5625 per_node = DIV_ROUND_UP(hugetlb_cma_size, nr_online_nodes); 5626 pr_info("hugetlb_cma: reserve %lu MiB, up to %lu MiB per node\n", 5627 hugetlb_cma_size / SZ_1M, per_node / SZ_1M); 5628 5629 reserved = 0; 5630 for_each_node_state(nid, N_ONLINE) { 5631 int res; 5632 char name[CMA_MAX_NAME]; 5633 5634 size = min(per_node, hugetlb_cma_size - reserved); 5635 size = round_up(size, PAGE_SIZE << order); 5636 5637 snprintf(name, sizeof(name), "hugetlb%d", nid); 5638 res = cma_declare_contiguous_nid(0, size, 0, PAGE_SIZE << order, 5639 0, false, name, 5640 &hugetlb_cma[nid], nid); 5641 if (res) { 5642 pr_warn("hugetlb_cma: reservation failed: err %d, node %d", 5643 res, nid); 5644 continue; 5645 } 5646 5647 reserved += size; 5648 pr_info("hugetlb_cma: reserved %lu MiB on node %d\n", 5649 size / SZ_1M, nid); 5650 5651 if (reserved >= hugetlb_cma_size) 5652 break; 5653 } 5654 } 5655 5656 void __init hugetlb_cma_check(void) 5657 { 5658 if (!hugetlb_cma_size || cma_reserve_called) 5659 return; 5660 5661 pr_warn("hugetlb_cma: the option isn't supported by current arch\n"); 5662 } 5663 5664 #endif /* CONFIG_CMA */ 5665