1 /* 2 * Generic hugetlb support. 3 * (C) Nadia Yvette Chambers, April 2004 4 */ 5 #include <linux/list.h> 6 #include <linux/init.h> 7 #include <linux/mm.h> 8 #include <linux/seq_file.h> 9 #include <linux/sysctl.h> 10 #include <linux/highmem.h> 11 #include <linux/mmu_notifier.h> 12 #include <linux/nodemask.h> 13 #include <linux/pagemap.h> 14 #include <linux/mempolicy.h> 15 #include <linux/compiler.h> 16 #include <linux/cpuset.h> 17 #include <linux/mutex.h> 18 #include <linux/bootmem.h> 19 #include <linux/sysfs.h> 20 #include <linux/slab.h> 21 #include <linux/sched/signal.h> 22 #include <linux/rmap.h> 23 #include <linux/string_helpers.h> 24 #include <linux/swap.h> 25 #include <linux/swapops.h> 26 #include <linux/jhash.h> 27 28 #include <asm/page.h> 29 #include <asm/pgtable.h> 30 #include <asm/tlb.h> 31 32 #include <linux/io.h> 33 #include <linux/hugetlb.h> 34 #include <linux/hugetlb_cgroup.h> 35 #include <linux/node.h> 36 #include <linux/userfaultfd_k.h> 37 #include "internal.h" 38 39 int hugepages_treat_as_movable; 40 41 int hugetlb_max_hstate __read_mostly; 42 unsigned int default_hstate_idx; 43 struct hstate hstates[HUGE_MAX_HSTATE]; 44 /* 45 * Minimum page order among possible hugepage sizes, set to a proper value 46 * at boot time. 47 */ 48 static unsigned int minimum_order __read_mostly = UINT_MAX; 49 50 __initdata LIST_HEAD(huge_boot_pages); 51 52 /* for command line parsing */ 53 static struct hstate * __initdata parsed_hstate; 54 static unsigned long __initdata default_hstate_max_huge_pages; 55 static unsigned long __initdata default_hstate_size; 56 static bool __initdata parsed_valid_hugepagesz = true; 57 58 /* 59 * Protects updates to hugepage_freelists, hugepage_activelist, nr_huge_pages, 60 * free_huge_pages, and surplus_huge_pages. 61 */ 62 DEFINE_SPINLOCK(hugetlb_lock); 63 64 /* 65 * Serializes faults on the same logical page. This is used to 66 * prevent spurious OOMs when the hugepage pool is fully utilized. 67 */ 68 static int num_fault_mutexes; 69 struct mutex *hugetlb_fault_mutex_table ____cacheline_aligned_in_smp; 70 71 /* Forward declaration */ 72 static int hugetlb_acct_memory(struct hstate *h, long delta); 73 74 static inline void unlock_or_release_subpool(struct hugepage_subpool *spool) 75 { 76 bool free = (spool->count == 0) && (spool->used_hpages == 0); 77 78 spin_unlock(&spool->lock); 79 80 /* If no pages are used, and no other handles to the subpool 81 * remain, give up any reservations mased on minimum size and 82 * free the subpool */ 83 if (free) { 84 if (spool->min_hpages != -1) 85 hugetlb_acct_memory(spool->hstate, 86 -spool->min_hpages); 87 kfree(spool); 88 } 89 } 90 91 struct hugepage_subpool *hugepage_new_subpool(struct hstate *h, long max_hpages, 92 long min_hpages) 93 { 94 struct hugepage_subpool *spool; 95 96 spool = kzalloc(sizeof(*spool), GFP_KERNEL); 97 if (!spool) 98 return NULL; 99 100 spin_lock_init(&spool->lock); 101 spool->count = 1; 102 spool->max_hpages = max_hpages; 103 spool->hstate = h; 104 spool->min_hpages = min_hpages; 105 106 if (min_hpages != -1 && hugetlb_acct_memory(h, min_hpages)) { 107 kfree(spool); 108 return NULL; 109 } 110 spool->rsv_hpages = min_hpages; 111 112 return spool; 113 } 114 115 void hugepage_put_subpool(struct hugepage_subpool *spool) 116 { 117 spin_lock(&spool->lock); 118 BUG_ON(!spool->count); 119 spool->count--; 120 unlock_or_release_subpool(spool); 121 } 122 123 /* 124 * Subpool accounting for allocating and reserving pages. 125 * Return -ENOMEM if there are not enough resources to satisfy the 126 * the request. Otherwise, return the number of pages by which the 127 * global pools must be adjusted (upward). The returned value may 128 * only be different than the passed value (delta) in the case where 129 * a subpool minimum size must be manitained. 130 */ 131 static long hugepage_subpool_get_pages(struct hugepage_subpool *spool, 132 long delta) 133 { 134 long ret = delta; 135 136 if (!spool) 137 return ret; 138 139 spin_lock(&spool->lock); 140 141 if (spool->max_hpages != -1) { /* maximum size accounting */ 142 if ((spool->used_hpages + delta) <= spool->max_hpages) 143 spool->used_hpages += delta; 144 else { 145 ret = -ENOMEM; 146 goto unlock_ret; 147 } 148 } 149 150 /* minimum size accounting */ 151 if (spool->min_hpages != -1 && spool->rsv_hpages) { 152 if (delta > spool->rsv_hpages) { 153 /* 154 * Asking for more reserves than those already taken on 155 * behalf of subpool. Return difference. 156 */ 157 ret = delta - spool->rsv_hpages; 158 spool->rsv_hpages = 0; 159 } else { 160 ret = 0; /* reserves already accounted for */ 161 spool->rsv_hpages -= delta; 162 } 163 } 164 165 unlock_ret: 166 spin_unlock(&spool->lock); 167 return ret; 168 } 169 170 /* 171 * Subpool accounting for freeing and unreserving pages. 172 * Return the number of global page reservations that must be dropped. 173 * The return value may only be different than the passed value (delta) 174 * in the case where a subpool minimum size must be maintained. 175 */ 176 static long hugepage_subpool_put_pages(struct hugepage_subpool *spool, 177 long delta) 178 { 179 long ret = delta; 180 181 if (!spool) 182 return delta; 183 184 spin_lock(&spool->lock); 185 186 if (spool->max_hpages != -1) /* maximum size accounting */ 187 spool->used_hpages -= delta; 188 189 /* minimum size accounting */ 190 if (spool->min_hpages != -1 && spool->used_hpages < spool->min_hpages) { 191 if (spool->rsv_hpages + delta <= spool->min_hpages) 192 ret = 0; 193 else 194 ret = spool->rsv_hpages + delta - spool->min_hpages; 195 196 spool->rsv_hpages += delta; 197 if (spool->rsv_hpages > spool->min_hpages) 198 spool->rsv_hpages = spool->min_hpages; 199 } 200 201 /* 202 * If hugetlbfs_put_super couldn't free spool due to an outstanding 203 * quota reference, free it now. 204 */ 205 unlock_or_release_subpool(spool); 206 207 return ret; 208 } 209 210 static inline struct hugepage_subpool *subpool_inode(struct inode *inode) 211 { 212 return HUGETLBFS_SB(inode->i_sb)->spool; 213 } 214 215 static inline struct hugepage_subpool *subpool_vma(struct vm_area_struct *vma) 216 { 217 return subpool_inode(file_inode(vma->vm_file)); 218 } 219 220 /* 221 * Region tracking -- allows tracking of reservations and instantiated pages 222 * across the pages in a mapping. 223 * 224 * The region data structures are embedded into a resv_map and protected 225 * by a resv_map's lock. The set of regions within the resv_map represent 226 * reservations for huge pages, or huge pages that have already been 227 * instantiated within the map. The from and to elements are huge page 228 * indicies into the associated mapping. from indicates the starting index 229 * of the region. to represents the first index past the end of the region. 230 * 231 * For example, a file region structure with from == 0 and to == 4 represents 232 * four huge pages in a mapping. It is important to note that the to element 233 * represents the first element past the end of the region. This is used in 234 * arithmetic as 4(to) - 0(from) = 4 huge pages in the region. 235 * 236 * Interval notation of the form [from, to) will be used to indicate that 237 * the endpoint from is inclusive and to is exclusive. 238 */ 239 struct file_region { 240 struct list_head link; 241 long from; 242 long to; 243 }; 244 245 /* 246 * Add the huge page range represented by [f, t) to the reserve 247 * map. In the normal case, existing regions will be expanded 248 * to accommodate the specified range. Sufficient regions should 249 * exist for expansion due to the previous call to region_chg 250 * with the same range. However, it is possible that region_del 251 * could have been called after region_chg and modifed the map 252 * in such a way that no region exists to be expanded. In this 253 * case, pull a region descriptor from the cache associated with 254 * the map and use that for the new range. 255 * 256 * Return the number of new huge pages added to the map. This 257 * number is greater than or equal to zero. 258 */ 259 static long region_add(struct resv_map *resv, long f, long t) 260 { 261 struct list_head *head = &resv->regions; 262 struct file_region *rg, *nrg, *trg; 263 long add = 0; 264 265 spin_lock(&resv->lock); 266 /* Locate the region we are either in or before. */ 267 list_for_each_entry(rg, head, link) 268 if (f <= rg->to) 269 break; 270 271 /* 272 * If no region exists which can be expanded to include the 273 * specified range, the list must have been modified by an 274 * interleving call to region_del(). Pull a region descriptor 275 * from the cache and use it for this range. 276 */ 277 if (&rg->link == head || t < rg->from) { 278 VM_BUG_ON(resv->region_cache_count <= 0); 279 280 resv->region_cache_count--; 281 nrg = list_first_entry(&resv->region_cache, struct file_region, 282 link); 283 list_del(&nrg->link); 284 285 nrg->from = f; 286 nrg->to = t; 287 list_add(&nrg->link, rg->link.prev); 288 289 add += t - f; 290 goto out_locked; 291 } 292 293 /* Round our left edge to the current segment if it encloses us. */ 294 if (f > rg->from) 295 f = rg->from; 296 297 /* Check for and consume any regions we now overlap with. */ 298 nrg = rg; 299 list_for_each_entry_safe(rg, trg, rg->link.prev, link) { 300 if (&rg->link == head) 301 break; 302 if (rg->from > t) 303 break; 304 305 /* If this area reaches higher then extend our area to 306 * include it completely. If this is not the first area 307 * which we intend to reuse, free it. */ 308 if (rg->to > t) 309 t = rg->to; 310 if (rg != nrg) { 311 /* Decrement return value by the deleted range. 312 * Another range will span this area so that by 313 * end of routine add will be >= zero 314 */ 315 add -= (rg->to - rg->from); 316 list_del(&rg->link); 317 kfree(rg); 318 } 319 } 320 321 add += (nrg->from - f); /* Added to beginning of region */ 322 nrg->from = f; 323 add += t - nrg->to; /* Added to end of region */ 324 nrg->to = t; 325 326 out_locked: 327 resv->adds_in_progress--; 328 spin_unlock(&resv->lock); 329 VM_BUG_ON(add < 0); 330 return add; 331 } 332 333 /* 334 * Examine the existing reserve map and determine how many 335 * huge pages in the specified range [f, t) are NOT currently 336 * represented. This routine is called before a subsequent 337 * call to region_add that will actually modify the reserve 338 * map to add the specified range [f, t). region_chg does 339 * not change the number of huge pages represented by the 340 * map. However, if the existing regions in the map can not 341 * be expanded to represent the new range, a new file_region 342 * structure is added to the map as a placeholder. This is 343 * so that the subsequent region_add call will have all the 344 * regions it needs and will not fail. 345 * 346 * Upon entry, region_chg will also examine the cache of region descriptors 347 * associated with the map. If there are not enough descriptors cached, one 348 * will be allocated for the in progress add operation. 349 * 350 * Returns the number of huge pages that need to be added to the existing 351 * reservation map for the range [f, t). This number is greater or equal to 352 * zero. -ENOMEM is returned if a new file_region structure or cache entry 353 * is needed and can not be allocated. 354 */ 355 static long region_chg(struct resv_map *resv, long f, long t) 356 { 357 struct list_head *head = &resv->regions; 358 struct file_region *rg, *nrg = NULL; 359 long chg = 0; 360 361 retry: 362 spin_lock(&resv->lock); 363 retry_locked: 364 resv->adds_in_progress++; 365 366 /* 367 * Check for sufficient descriptors in the cache to accommodate 368 * the number of in progress add operations. 369 */ 370 if (resv->adds_in_progress > resv->region_cache_count) { 371 struct file_region *trg; 372 373 VM_BUG_ON(resv->adds_in_progress - resv->region_cache_count > 1); 374 /* Must drop lock to allocate a new descriptor. */ 375 resv->adds_in_progress--; 376 spin_unlock(&resv->lock); 377 378 trg = kmalloc(sizeof(*trg), GFP_KERNEL); 379 if (!trg) { 380 kfree(nrg); 381 return -ENOMEM; 382 } 383 384 spin_lock(&resv->lock); 385 list_add(&trg->link, &resv->region_cache); 386 resv->region_cache_count++; 387 goto retry_locked; 388 } 389 390 /* Locate the region we are before or in. */ 391 list_for_each_entry(rg, head, link) 392 if (f <= rg->to) 393 break; 394 395 /* If we are below the current region then a new region is required. 396 * Subtle, allocate a new region at the position but make it zero 397 * size such that we can guarantee to record the reservation. */ 398 if (&rg->link == head || t < rg->from) { 399 if (!nrg) { 400 resv->adds_in_progress--; 401 spin_unlock(&resv->lock); 402 nrg = kmalloc(sizeof(*nrg), GFP_KERNEL); 403 if (!nrg) 404 return -ENOMEM; 405 406 nrg->from = f; 407 nrg->to = f; 408 INIT_LIST_HEAD(&nrg->link); 409 goto retry; 410 } 411 412 list_add(&nrg->link, rg->link.prev); 413 chg = t - f; 414 goto out_nrg; 415 } 416 417 /* Round our left edge to the current segment if it encloses us. */ 418 if (f > rg->from) 419 f = rg->from; 420 chg = t - f; 421 422 /* Check for and consume any regions we now overlap with. */ 423 list_for_each_entry(rg, rg->link.prev, link) { 424 if (&rg->link == head) 425 break; 426 if (rg->from > t) 427 goto out; 428 429 /* We overlap with this area, if it extends further than 430 * us then we must extend ourselves. Account for its 431 * existing reservation. */ 432 if (rg->to > t) { 433 chg += rg->to - t; 434 t = rg->to; 435 } 436 chg -= rg->to - rg->from; 437 } 438 439 out: 440 spin_unlock(&resv->lock); 441 /* We already know we raced and no longer need the new region */ 442 kfree(nrg); 443 return chg; 444 out_nrg: 445 spin_unlock(&resv->lock); 446 return chg; 447 } 448 449 /* 450 * Abort the in progress add operation. The adds_in_progress field 451 * of the resv_map keeps track of the operations in progress between 452 * calls to region_chg and region_add. Operations are sometimes 453 * aborted after the call to region_chg. In such cases, region_abort 454 * is called to decrement the adds_in_progress counter. 455 * 456 * NOTE: The range arguments [f, t) are not needed or used in this 457 * routine. They are kept to make reading the calling code easier as 458 * arguments will match the associated region_chg call. 459 */ 460 static void region_abort(struct resv_map *resv, long f, long t) 461 { 462 spin_lock(&resv->lock); 463 VM_BUG_ON(!resv->region_cache_count); 464 resv->adds_in_progress--; 465 spin_unlock(&resv->lock); 466 } 467 468 /* 469 * Delete the specified range [f, t) from the reserve map. If the 470 * t parameter is LONG_MAX, this indicates that ALL regions after f 471 * should be deleted. Locate the regions which intersect [f, t) 472 * and either trim, delete or split the existing regions. 473 * 474 * Returns the number of huge pages deleted from the reserve map. 475 * In the normal case, the return value is zero or more. In the 476 * case where a region must be split, a new region descriptor must 477 * be allocated. If the allocation fails, -ENOMEM will be returned. 478 * NOTE: If the parameter t == LONG_MAX, then we will never split 479 * a region and possibly return -ENOMEM. Callers specifying 480 * t == LONG_MAX do not need to check for -ENOMEM error. 481 */ 482 static long region_del(struct resv_map *resv, long f, long t) 483 { 484 struct list_head *head = &resv->regions; 485 struct file_region *rg, *trg; 486 struct file_region *nrg = NULL; 487 long del = 0; 488 489 retry: 490 spin_lock(&resv->lock); 491 list_for_each_entry_safe(rg, trg, head, link) { 492 /* 493 * Skip regions before the range to be deleted. file_region 494 * ranges are normally of the form [from, to). However, there 495 * may be a "placeholder" entry in the map which is of the form 496 * (from, to) with from == to. Check for placeholder entries 497 * at the beginning of the range to be deleted. 498 */ 499 if (rg->to <= f && (rg->to != rg->from || rg->to != f)) 500 continue; 501 502 if (rg->from >= t) 503 break; 504 505 if (f > rg->from && t < rg->to) { /* Must split region */ 506 /* 507 * Check for an entry in the cache before dropping 508 * lock and attempting allocation. 509 */ 510 if (!nrg && 511 resv->region_cache_count > resv->adds_in_progress) { 512 nrg = list_first_entry(&resv->region_cache, 513 struct file_region, 514 link); 515 list_del(&nrg->link); 516 resv->region_cache_count--; 517 } 518 519 if (!nrg) { 520 spin_unlock(&resv->lock); 521 nrg = kmalloc(sizeof(*nrg), GFP_KERNEL); 522 if (!nrg) 523 return -ENOMEM; 524 goto retry; 525 } 526 527 del += t - f; 528 529 /* New entry for end of split region */ 530 nrg->from = t; 531 nrg->to = rg->to; 532 INIT_LIST_HEAD(&nrg->link); 533 534 /* Original entry is trimmed */ 535 rg->to = f; 536 537 list_add(&nrg->link, &rg->link); 538 nrg = NULL; 539 break; 540 } 541 542 if (f <= rg->from && t >= rg->to) { /* Remove entire region */ 543 del += rg->to - rg->from; 544 list_del(&rg->link); 545 kfree(rg); 546 continue; 547 } 548 549 if (f <= rg->from) { /* Trim beginning of region */ 550 del += t - rg->from; 551 rg->from = t; 552 } else { /* Trim end of region */ 553 del += rg->to - f; 554 rg->to = f; 555 } 556 } 557 558 spin_unlock(&resv->lock); 559 kfree(nrg); 560 return del; 561 } 562 563 /* 564 * A rare out of memory error was encountered which prevented removal of 565 * the reserve map region for a page. The huge page itself was free'ed 566 * and removed from the page cache. This routine will adjust the subpool 567 * usage count, and the global reserve count if needed. By incrementing 568 * these counts, the reserve map entry which could not be deleted will 569 * appear as a "reserved" entry instead of simply dangling with incorrect 570 * counts. 571 */ 572 void hugetlb_fix_reserve_counts(struct inode *inode) 573 { 574 struct hugepage_subpool *spool = subpool_inode(inode); 575 long rsv_adjust; 576 577 rsv_adjust = hugepage_subpool_get_pages(spool, 1); 578 if (rsv_adjust) { 579 struct hstate *h = hstate_inode(inode); 580 581 hugetlb_acct_memory(h, 1); 582 } 583 } 584 585 /* 586 * Count and return the number of huge pages in the reserve map 587 * that intersect with the range [f, t). 588 */ 589 static long region_count(struct resv_map *resv, long f, long t) 590 { 591 struct list_head *head = &resv->regions; 592 struct file_region *rg; 593 long chg = 0; 594 595 spin_lock(&resv->lock); 596 /* Locate each segment we overlap with, and count that overlap. */ 597 list_for_each_entry(rg, head, link) { 598 long seg_from; 599 long seg_to; 600 601 if (rg->to <= f) 602 continue; 603 if (rg->from >= t) 604 break; 605 606 seg_from = max(rg->from, f); 607 seg_to = min(rg->to, t); 608 609 chg += seg_to - seg_from; 610 } 611 spin_unlock(&resv->lock); 612 613 return chg; 614 } 615 616 /* 617 * Convert the address within this vma to the page offset within 618 * the mapping, in pagecache page units; huge pages here. 619 */ 620 static pgoff_t vma_hugecache_offset(struct hstate *h, 621 struct vm_area_struct *vma, unsigned long address) 622 { 623 return ((address - vma->vm_start) >> huge_page_shift(h)) + 624 (vma->vm_pgoff >> huge_page_order(h)); 625 } 626 627 pgoff_t linear_hugepage_index(struct vm_area_struct *vma, 628 unsigned long address) 629 { 630 return vma_hugecache_offset(hstate_vma(vma), vma, address); 631 } 632 EXPORT_SYMBOL_GPL(linear_hugepage_index); 633 634 /* 635 * Return the size of the pages allocated when backing a VMA. In the majority 636 * cases this will be same size as used by the page table entries. 637 */ 638 unsigned long vma_kernel_pagesize(struct vm_area_struct *vma) 639 { 640 struct hstate *hstate; 641 642 if (!is_vm_hugetlb_page(vma)) 643 return PAGE_SIZE; 644 645 hstate = hstate_vma(vma); 646 647 return 1UL << huge_page_shift(hstate); 648 } 649 EXPORT_SYMBOL_GPL(vma_kernel_pagesize); 650 651 /* 652 * Return the page size being used by the MMU to back a VMA. In the majority 653 * of cases, the page size used by the kernel matches the MMU size. On 654 * architectures where it differs, an architecture-specific version of this 655 * function is required. 656 */ 657 #ifndef vma_mmu_pagesize 658 unsigned long vma_mmu_pagesize(struct vm_area_struct *vma) 659 { 660 return vma_kernel_pagesize(vma); 661 } 662 #endif 663 664 /* 665 * Flags for MAP_PRIVATE reservations. These are stored in the bottom 666 * bits of the reservation map pointer, which are always clear due to 667 * alignment. 668 */ 669 #define HPAGE_RESV_OWNER (1UL << 0) 670 #define HPAGE_RESV_UNMAPPED (1UL << 1) 671 #define HPAGE_RESV_MASK (HPAGE_RESV_OWNER | HPAGE_RESV_UNMAPPED) 672 673 /* 674 * These helpers are used to track how many pages are reserved for 675 * faults in a MAP_PRIVATE mapping. Only the process that called mmap() 676 * is guaranteed to have their future faults succeed. 677 * 678 * With the exception of reset_vma_resv_huge_pages() which is called at fork(), 679 * the reserve counters are updated with the hugetlb_lock held. It is safe 680 * to reset the VMA at fork() time as it is not in use yet and there is no 681 * chance of the global counters getting corrupted as a result of the values. 682 * 683 * The private mapping reservation is represented in a subtly different 684 * manner to a shared mapping. A shared mapping has a region map associated 685 * with the underlying file, this region map represents the backing file 686 * pages which have ever had a reservation assigned which this persists even 687 * after the page is instantiated. A private mapping has a region map 688 * associated with the original mmap which is attached to all VMAs which 689 * reference it, this region map represents those offsets which have consumed 690 * reservation ie. where pages have been instantiated. 691 */ 692 static unsigned long get_vma_private_data(struct vm_area_struct *vma) 693 { 694 return (unsigned long)vma->vm_private_data; 695 } 696 697 static void set_vma_private_data(struct vm_area_struct *vma, 698 unsigned long value) 699 { 700 vma->vm_private_data = (void *)value; 701 } 702 703 struct resv_map *resv_map_alloc(void) 704 { 705 struct resv_map *resv_map = kmalloc(sizeof(*resv_map), GFP_KERNEL); 706 struct file_region *rg = kmalloc(sizeof(*rg), GFP_KERNEL); 707 708 if (!resv_map || !rg) { 709 kfree(resv_map); 710 kfree(rg); 711 return NULL; 712 } 713 714 kref_init(&resv_map->refs); 715 spin_lock_init(&resv_map->lock); 716 INIT_LIST_HEAD(&resv_map->regions); 717 718 resv_map->adds_in_progress = 0; 719 720 INIT_LIST_HEAD(&resv_map->region_cache); 721 list_add(&rg->link, &resv_map->region_cache); 722 resv_map->region_cache_count = 1; 723 724 return resv_map; 725 } 726 727 void resv_map_release(struct kref *ref) 728 { 729 struct resv_map *resv_map = container_of(ref, struct resv_map, refs); 730 struct list_head *head = &resv_map->region_cache; 731 struct file_region *rg, *trg; 732 733 /* Clear out any active regions before we release the map. */ 734 region_del(resv_map, 0, LONG_MAX); 735 736 /* ... and any entries left in the cache */ 737 list_for_each_entry_safe(rg, trg, head, link) { 738 list_del(&rg->link); 739 kfree(rg); 740 } 741 742 VM_BUG_ON(resv_map->adds_in_progress); 743 744 kfree(resv_map); 745 } 746 747 static inline struct resv_map *inode_resv_map(struct inode *inode) 748 { 749 return inode->i_mapping->private_data; 750 } 751 752 static struct resv_map *vma_resv_map(struct vm_area_struct *vma) 753 { 754 VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma); 755 if (vma->vm_flags & VM_MAYSHARE) { 756 struct address_space *mapping = vma->vm_file->f_mapping; 757 struct inode *inode = mapping->host; 758 759 return inode_resv_map(inode); 760 761 } else { 762 return (struct resv_map *)(get_vma_private_data(vma) & 763 ~HPAGE_RESV_MASK); 764 } 765 } 766 767 static void set_vma_resv_map(struct vm_area_struct *vma, struct resv_map *map) 768 { 769 VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma); 770 VM_BUG_ON_VMA(vma->vm_flags & VM_MAYSHARE, vma); 771 772 set_vma_private_data(vma, (get_vma_private_data(vma) & 773 HPAGE_RESV_MASK) | (unsigned long)map); 774 } 775 776 static void set_vma_resv_flags(struct vm_area_struct *vma, unsigned long flags) 777 { 778 VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma); 779 VM_BUG_ON_VMA(vma->vm_flags & VM_MAYSHARE, vma); 780 781 set_vma_private_data(vma, get_vma_private_data(vma) | flags); 782 } 783 784 static int is_vma_resv_set(struct vm_area_struct *vma, unsigned long flag) 785 { 786 VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma); 787 788 return (get_vma_private_data(vma) & flag) != 0; 789 } 790 791 /* Reset counters to 0 and clear all HPAGE_RESV_* flags */ 792 void reset_vma_resv_huge_pages(struct vm_area_struct *vma) 793 { 794 VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma); 795 if (!(vma->vm_flags & VM_MAYSHARE)) 796 vma->vm_private_data = (void *)0; 797 } 798 799 /* Returns true if the VMA has associated reserve pages */ 800 static bool vma_has_reserves(struct vm_area_struct *vma, long chg) 801 { 802 if (vma->vm_flags & VM_NORESERVE) { 803 /* 804 * This address is already reserved by other process(chg == 0), 805 * so, we should decrement reserved count. Without decrementing, 806 * reserve count remains after releasing inode, because this 807 * allocated page will go into page cache and is regarded as 808 * coming from reserved pool in releasing step. Currently, we 809 * don't have any other solution to deal with this situation 810 * properly, so add work-around here. 811 */ 812 if (vma->vm_flags & VM_MAYSHARE && chg == 0) 813 return true; 814 else 815 return false; 816 } 817 818 /* Shared mappings always use reserves */ 819 if (vma->vm_flags & VM_MAYSHARE) { 820 /* 821 * We know VM_NORESERVE is not set. Therefore, there SHOULD 822 * be a region map for all pages. The only situation where 823 * there is no region map is if a hole was punched via 824 * fallocate. In this case, there really are no reverves to 825 * use. This situation is indicated if chg != 0. 826 */ 827 if (chg) 828 return false; 829 else 830 return true; 831 } 832 833 /* 834 * Only the process that called mmap() has reserves for 835 * private mappings. 836 */ 837 if (is_vma_resv_set(vma, HPAGE_RESV_OWNER)) { 838 /* 839 * Like the shared case above, a hole punch or truncate 840 * could have been performed on the private mapping. 841 * Examine the value of chg to determine if reserves 842 * actually exist or were previously consumed. 843 * Very Subtle - The value of chg comes from a previous 844 * call to vma_needs_reserves(). The reserve map for 845 * private mappings has different (opposite) semantics 846 * than that of shared mappings. vma_needs_reserves() 847 * has already taken this difference in semantics into 848 * account. Therefore, the meaning of chg is the same 849 * as in the shared case above. Code could easily be 850 * combined, but keeping it separate draws attention to 851 * subtle differences. 852 */ 853 if (chg) 854 return false; 855 else 856 return true; 857 } 858 859 return false; 860 } 861 862 static void enqueue_huge_page(struct hstate *h, struct page *page) 863 { 864 int nid = page_to_nid(page); 865 list_move(&page->lru, &h->hugepage_freelists[nid]); 866 h->free_huge_pages++; 867 h->free_huge_pages_node[nid]++; 868 } 869 870 static struct page *dequeue_huge_page_node_exact(struct hstate *h, int nid) 871 { 872 struct page *page; 873 874 list_for_each_entry(page, &h->hugepage_freelists[nid], lru) 875 if (!PageHWPoison(page)) 876 break; 877 /* 878 * if 'non-isolated free hugepage' not found on the list, 879 * the allocation fails. 880 */ 881 if (&h->hugepage_freelists[nid] == &page->lru) 882 return NULL; 883 list_move(&page->lru, &h->hugepage_activelist); 884 set_page_refcounted(page); 885 h->free_huge_pages--; 886 h->free_huge_pages_node[nid]--; 887 return page; 888 } 889 890 static struct page *dequeue_huge_page_nodemask(struct hstate *h, gfp_t gfp_mask, int nid, 891 nodemask_t *nmask) 892 { 893 unsigned int cpuset_mems_cookie; 894 struct zonelist *zonelist; 895 struct zone *zone; 896 struct zoneref *z; 897 int node = -1; 898 899 zonelist = node_zonelist(nid, gfp_mask); 900 901 retry_cpuset: 902 cpuset_mems_cookie = read_mems_allowed_begin(); 903 for_each_zone_zonelist_nodemask(zone, z, zonelist, gfp_zone(gfp_mask), nmask) { 904 struct page *page; 905 906 if (!cpuset_zone_allowed(zone, gfp_mask)) 907 continue; 908 /* 909 * no need to ask again on the same node. Pool is node rather than 910 * zone aware 911 */ 912 if (zone_to_nid(zone) == node) 913 continue; 914 node = zone_to_nid(zone); 915 916 page = dequeue_huge_page_node_exact(h, node); 917 if (page) 918 return page; 919 } 920 if (unlikely(read_mems_allowed_retry(cpuset_mems_cookie))) 921 goto retry_cpuset; 922 923 return NULL; 924 } 925 926 /* Movability of hugepages depends on migration support. */ 927 static inline gfp_t htlb_alloc_mask(struct hstate *h) 928 { 929 if (hugepages_treat_as_movable || hugepage_migration_supported(h)) 930 return GFP_HIGHUSER_MOVABLE; 931 else 932 return GFP_HIGHUSER; 933 } 934 935 static struct page *dequeue_huge_page_vma(struct hstate *h, 936 struct vm_area_struct *vma, 937 unsigned long address, int avoid_reserve, 938 long chg) 939 { 940 struct page *page; 941 struct mempolicy *mpol; 942 gfp_t gfp_mask; 943 nodemask_t *nodemask; 944 int nid; 945 946 /* 947 * A child process with MAP_PRIVATE mappings created by their parent 948 * have no page reserves. This check ensures that reservations are 949 * not "stolen". The child may still get SIGKILLed 950 */ 951 if (!vma_has_reserves(vma, chg) && 952 h->free_huge_pages - h->resv_huge_pages == 0) 953 goto err; 954 955 /* If reserves cannot be used, ensure enough pages are in the pool */ 956 if (avoid_reserve && h->free_huge_pages - h->resv_huge_pages == 0) 957 goto err; 958 959 gfp_mask = htlb_alloc_mask(h); 960 nid = huge_node(vma, address, gfp_mask, &mpol, &nodemask); 961 page = dequeue_huge_page_nodemask(h, gfp_mask, nid, nodemask); 962 if (page && !avoid_reserve && vma_has_reserves(vma, chg)) { 963 SetPagePrivate(page); 964 h->resv_huge_pages--; 965 } 966 967 mpol_cond_put(mpol); 968 return page; 969 970 err: 971 return NULL; 972 } 973 974 /* 975 * common helper functions for hstate_next_node_to_{alloc|free}. 976 * We may have allocated or freed a huge page based on a different 977 * nodes_allowed previously, so h->next_node_to_{alloc|free} might 978 * be outside of *nodes_allowed. Ensure that we use an allowed 979 * node for alloc or free. 980 */ 981 static int next_node_allowed(int nid, nodemask_t *nodes_allowed) 982 { 983 nid = next_node_in(nid, *nodes_allowed); 984 VM_BUG_ON(nid >= MAX_NUMNODES); 985 986 return nid; 987 } 988 989 static int get_valid_node_allowed(int nid, nodemask_t *nodes_allowed) 990 { 991 if (!node_isset(nid, *nodes_allowed)) 992 nid = next_node_allowed(nid, nodes_allowed); 993 return nid; 994 } 995 996 /* 997 * returns the previously saved node ["this node"] from which to 998 * allocate a persistent huge page for the pool and advance the 999 * next node from which to allocate, handling wrap at end of node 1000 * mask. 1001 */ 1002 static int hstate_next_node_to_alloc(struct hstate *h, 1003 nodemask_t *nodes_allowed) 1004 { 1005 int nid; 1006 1007 VM_BUG_ON(!nodes_allowed); 1008 1009 nid = get_valid_node_allowed(h->next_nid_to_alloc, nodes_allowed); 1010 h->next_nid_to_alloc = next_node_allowed(nid, nodes_allowed); 1011 1012 return nid; 1013 } 1014 1015 /* 1016 * helper for free_pool_huge_page() - return the previously saved 1017 * node ["this node"] from which to free a huge page. Advance the 1018 * next node id whether or not we find a free huge page to free so 1019 * that the next attempt to free addresses the next node. 1020 */ 1021 static int hstate_next_node_to_free(struct hstate *h, nodemask_t *nodes_allowed) 1022 { 1023 int nid; 1024 1025 VM_BUG_ON(!nodes_allowed); 1026 1027 nid = get_valid_node_allowed(h->next_nid_to_free, nodes_allowed); 1028 h->next_nid_to_free = next_node_allowed(nid, nodes_allowed); 1029 1030 return nid; 1031 } 1032 1033 #define for_each_node_mask_to_alloc(hs, nr_nodes, node, mask) \ 1034 for (nr_nodes = nodes_weight(*mask); \ 1035 nr_nodes > 0 && \ 1036 ((node = hstate_next_node_to_alloc(hs, mask)) || 1); \ 1037 nr_nodes--) 1038 1039 #define for_each_node_mask_to_free(hs, nr_nodes, node, mask) \ 1040 for (nr_nodes = nodes_weight(*mask); \ 1041 nr_nodes > 0 && \ 1042 ((node = hstate_next_node_to_free(hs, mask)) || 1); \ 1043 nr_nodes--) 1044 1045 #ifdef CONFIG_ARCH_HAS_GIGANTIC_PAGE 1046 static void destroy_compound_gigantic_page(struct page *page, 1047 unsigned int order) 1048 { 1049 int i; 1050 int nr_pages = 1 << order; 1051 struct page *p = page + 1; 1052 1053 atomic_set(compound_mapcount_ptr(page), 0); 1054 for (i = 1; i < nr_pages; i++, p = mem_map_next(p, page, i)) { 1055 clear_compound_head(p); 1056 set_page_refcounted(p); 1057 } 1058 1059 set_compound_order(page, 0); 1060 __ClearPageHead(page); 1061 } 1062 1063 static void free_gigantic_page(struct page *page, unsigned int order) 1064 { 1065 free_contig_range(page_to_pfn(page), 1 << order); 1066 } 1067 1068 static int __alloc_gigantic_page(unsigned long start_pfn, 1069 unsigned long nr_pages) 1070 { 1071 unsigned long end_pfn = start_pfn + nr_pages; 1072 return alloc_contig_range(start_pfn, end_pfn, MIGRATE_MOVABLE, 1073 GFP_KERNEL); 1074 } 1075 1076 static bool pfn_range_valid_gigantic(struct zone *z, 1077 unsigned long start_pfn, unsigned long nr_pages) 1078 { 1079 unsigned long i, end_pfn = start_pfn + nr_pages; 1080 struct page *page; 1081 1082 for (i = start_pfn; i < end_pfn; i++) { 1083 if (!pfn_valid(i)) 1084 return false; 1085 1086 page = pfn_to_page(i); 1087 1088 if (page_zone(page) != z) 1089 return false; 1090 1091 if (PageReserved(page)) 1092 return false; 1093 1094 if (page_count(page) > 0) 1095 return false; 1096 1097 if (PageHuge(page)) 1098 return false; 1099 } 1100 1101 return true; 1102 } 1103 1104 static bool zone_spans_last_pfn(const struct zone *zone, 1105 unsigned long start_pfn, unsigned long nr_pages) 1106 { 1107 unsigned long last_pfn = start_pfn + nr_pages - 1; 1108 return zone_spans_pfn(zone, last_pfn); 1109 } 1110 1111 static struct page *alloc_gigantic_page(int nid, unsigned int order) 1112 { 1113 unsigned long nr_pages = 1 << order; 1114 unsigned long ret, pfn, flags; 1115 struct zone *z; 1116 1117 z = NODE_DATA(nid)->node_zones; 1118 for (; z - NODE_DATA(nid)->node_zones < MAX_NR_ZONES; z++) { 1119 spin_lock_irqsave(&z->lock, flags); 1120 1121 pfn = ALIGN(z->zone_start_pfn, nr_pages); 1122 while (zone_spans_last_pfn(z, pfn, nr_pages)) { 1123 if (pfn_range_valid_gigantic(z, pfn, nr_pages)) { 1124 /* 1125 * We release the zone lock here because 1126 * alloc_contig_range() will also lock the zone 1127 * at some point. If there's an allocation 1128 * spinning on this lock, it may win the race 1129 * and cause alloc_contig_range() to fail... 1130 */ 1131 spin_unlock_irqrestore(&z->lock, flags); 1132 ret = __alloc_gigantic_page(pfn, nr_pages); 1133 if (!ret) 1134 return pfn_to_page(pfn); 1135 spin_lock_irqsave(&z->lock, flags); 1136 } 1137 pfn += nr_pages; 1138 } 1139 1140 spin_unlock_irqrestore(&z->lock, flags); 1141 } 1142 1143 return NULL; 1144 } 1145 1146 static void prep_new_huge_page(struct hstate *h, struct page *page, int nid); 1147 static void prep_compound_gigantic_page(struct page *page, unsigned int order); 1148 1149 static struct page *alloc_fresh_gigantic_page_node(struct hstate *h, int nid) 1150 { 1151 struct page *page; 1152 1153 page = alloc_gigantic_page(nid, huge_page_order(h)); 1154 if (page) { 1155 prep_compound_gigantic_page(page, huge_page_order(h)); 1156 prep_new_huge_page(h, page, nid); 1157 } 1158 1159 return page; 1160 } 1161 1162 static int alloc_fresh_gigantic_page(struct hstate *h, 1163 nodemask_t *nodes_allowed) 1164 { 1165 struct page *page = NULL; 1166 int nr_nodes, node; 1167 1168 for_each_node_mask_to_alloc(h, nr_nodes, node, nodes_allowed) { 1169 page = alloc_fresh_gigantic_page_node(h, node); 1170 if (page) 1171 return 1; 1172 } 1173 1174 return 0; 1175 } 1176 1177 #else /* !CONFIG_ARCH_HAS_GIGANTIC_PAGE */ 1178 static inline bool gigantic_page_supported(void) { return false; } 1179 static inline void free_gigantic_page(struct page *page, unsigned int order) { } 1180 static inline void destroy_compound_gigantic_page(struct page *page, 1181 unsigned int order) { } 1182 static inline int alloc_fresh_gigantic_page(struct hstate *h, 1183 nodemask_t *nodes_allowed) { return 0; } 1184 #endif 1185 1186 static void update_and_free_page(struct hstate *h, struct page *page) 1187 { 1188 int i; 1189 1190 if (hstate_is_gigantic(h) && !gigantic_page_supported()) 1191 return; 1192 1193 h->nr_huge_pages--; 1194 h->nr_huge_pages_node[page_to_nid(page)]--; 1195 for (i = 0; i < pages_per_huge_page(h); i++) { 1196 page[i].flags &= ~(1 << PG_locked | 1 << PG_error | 1197 1 << PG_referenced | 1 << PG_dirty | 1198 1 << PG_active | 1 << PG_private | 1199 1 << PG_writeback); 1200 } 1201 VM_BUG_ON_PAGE(hugetlb_cgroup_from_page(page), page); 1202 set_compound_page_dtor(page, NULL_COMPOUND_DTOR); 1203 set_page_refcounted(page); 1204 if (hstate_is_gigantic(h)) { 1205 destroy_compound_gigantic_page(page, huge_page_order(h)); 1206 free_gigantic_page(page, huge_page_order(h)); 1207 } else { 1208 __free_pages(page, huge_page_order(h)); 1209 } 1210 } 1211 1212 struct hstate *size_to_hstate(unsigned long size) 1213 { 1214 struct hstate *h; 1215 1216 for_each_hstate(h) { 1217 if (huge_page_size(h) == size) 1218 return h; 1219 } 1220 return NULL; 1221 } 1222 1223 /* 1224 * Test to determine whether the hugepage is "active/in-use" (i.e. being linked 1225 * to hstate->hugepage_activelist.) 1226 * 1227 * This function can be called for tail pages, but never returns true for them. 1228 */ 1229 bool page_huge_active(struct page *page) 1230 { 1231 VM_BUG_ON_PAGE(!PageHuge(page), page); 1232 return PageHead(page) && PagePrivate(&page[1]); 1233 } 1234 1235 /* never called for tail page */ 1236 static void set_page_huge_active(struct page *page) 1237 { 1238 VM_BUG_ON_PAGE(!PageHeadHuge(page), page); 1239 SetPagePrivate(&page[1]); 1240 } 1241 1242 static void clear_page_huge_active(struct page *page) 1243 { 1244 VM_BUG_ON_PAGE(!PageHeadHuge(page), page); 1245 ClearPagePrivate(&page[1]); 1246 } 1247 1248 void free_huge_page(struct page *page) 1249 { 1250 /* 1251 * Can't pass hstate in here because it is called from the 1252 * compound page destructor. 1253 */ 1254 struct hstate *h = page_hstate(page); 1255 int nid = page_to_nid(page); 1256 struct hugepage_subpool *spool = 1257 (struct hugepage_subpool *)page_private(page); 1258 bool restore_reserve; 1259 1260 set_page_private(page, 0); 1261 page->mapping = NULL; 1262 VM_BUG_ON_PAGE(page_count(page), page); 1263 VM_BUG_ON_PAGE(page_mapcount(page), page); 1264 restore_reserve = PagePrivate(page); 1265 ClearPagePrivate(page); 1266 1267 /* 1268 * A return code of zero implies that the subpool will be under its 1269 * minimum size if the reservation is not restored after page is free. 1270 * Therefore, force restore_reserve operation. 1271 */ 1272 if (hugepage_subpool_put_pages(spool, 1) == 0) 1273 restore_reserve = true; 1274 1275 spin_lock(&hugetlb_lock); 1276 clear_page_huge_active(page); 1277 hugetlb_cgroup_uncharge_page(hstate_index(h), 1278 pages_per_huge_page(h), page); 1279 if (restore_reserve) 1280 h->resv_huge_pages++; 1281 1282 if (h->surplus_huge_pages_node[nid]) { 1283 /* remove the page from active list */ 1284 list_del(&page->lru); 1285 update_and_free_page(h, page); 1286 h->surplus_huge_pages--; 1287 h->surplus_huge_pages_node[nid]--; 1288 } else { 1289 arch_clear_hugepage_flags(page); 1290 enqueue_huge_page(h, page); 1291 } 1292 spin_unlock(&hugetlb_lock); 1293 } 1294 1295 static void prep_new_huge_page(struct hstate *h, struct page *page, int nid) 1296 { 1297 INIT_LIST_HEAD(&page->lru); 1298 set_compound_page_dtor(page, HUGETLB_PAGE_DTOR); 1299 spin_lock(&hugetlb_lock); 1300 set_hugetlb_cgroup(page, NULL); 1301 h->nr_huge_pages++; 1302 h->nr_huge_pages_node[nid]++; 1303 spin_unlock(&hugetlb_lock); 1304 put_page(page); /* free it into the hugepage allocator */ 1305 } 1306 1307 static void prep_compound_gigantic_page(struct page *page, unsigned int order) 1308 { 1309 int i; 1310 int nr_pages = 1 << order; 1311 struct page *p = page + 1; 1312 1313 /* we rely on prep_new_huge_page to set the destructor */ 1314 set_compound_order(page, order); 1315 __ClearPageReserved(page); 1316 __SetPageHead(page); 1317 for (i = 1; i < nr_pages; i++, p = mem_map_next(p, page, i)) { 1318 /* 1319 * For gigantic hugepages allocated through bootmem at 1320 * boot, it's safer to be consistent with the not-gigantic 1321 * hugepages and clear the PG_reserved bit from all tail pages 1322 * too. Otherwse drivers using get_user_pages() to access tail 1323 * pages may get the reference counting wrong if they see 1324 * PG_reserved set on a tail page (despite the head page not 1325 * having PG_reserved set). Enforcing this consistency between 1326 * head and tail pages allows drivers to optimize away a check 1327 * on the head page when they need know if put_page() is needed 1328 * after get_user_pages(). 1329 */ 1330 __ClearPageReserved(p); 1331 set_page_count(p, 0); 1332 set_compound_head(p, page); 1333 } 1334 atomic_set(compound_mapcount_ptr(page), -1); 1335 } 1336 1337 /* 1338 * PageHuge() only returns true for hugetlbfs pages, but not for normal or 1339 * transparent huge pages. See the PageTransHuge() documentation for more 1340 * details. 1341 */ 1342 int PageHuge(struct page *page) 1343 { 1344 if (!PageCompound(page)) 1345 return 0; 1346 1347 page = compound_head(page); 1348 return page[1].compound_dtor == HUGETLB_PAGE_DTOR; 1349 } 1350 EXPORT_SYMBOL_GPL(PageHuge); 1351 1352 /* 1353 * PageHeadHuge() only returns true for hugetlbfs head page, but not for 1354 * normal or transparent huge pages. 1355 */ 1356 int PageHeadHuge(struct page *page_head) 1357 { 1358 if (!PageHead(page_head)) 1359 return 0; 1360 1361 return get_compound_page_dtor(page_head) == free_huge_page; 1362 } 1363 1364 pgoff_t __basepage_index(struct page *page) 1365 { 1366 struct page *page_head = compound_head(page); 1367 pgoff_t index = page_index(page_head); 1368 unsigned long compound_idx; 1369 1370 if (!PageHuge(page_head)) 1371 return page_index(page); 1372 1373 if (compound_order(page_head) >= MAX_ORDER) 1374 compound_idx = page_to_pfn(page) - page_to_pfn(page_head); 1375 else 1376 compound_idx = page - page_head; 1377 1378 return (index << compound_order(page_head)) + compound_idx; 1379 } 1380 1381 static struct page *alloc_fresh_huge_page_node(struct hstate *h, int nid) 1382 { 1383 struct page *page; 1384 1385 page = __alloc_pages_node(nid, 1386 htlb_alloc_mask(h)|__GFP_COMP|__GFP_THISNODE| 1387 __GFP_RETRY_MAYFAIL|__GFP_NOWARN, 1388 huge_page_order(h)); 1389 if (page) { 1390 prep_new_huge_page(h, page, nid); 1391 } 1392 1393 return page; 1394 } 1395 1396 static int alloc_fresh_huge_page(struct hstate *h, nodemask_t *nodes_allowed) 1397 { 1398 struct page *page; 1399 int nr_nodes, node; 1400 int ret = 0; 1401 1402 for_each_node_mask_to_alloc(h, nr_nodes, node, nodes_allowed) { 1403 page = alloc_fresh_huge_page_node(h, node); 1404 if (page) { 1405 ret = 1; 1406 break; 1407 } 1408 } 1409 1410 if (ret) 1411 count_vm_event(HTLB_BUDDY_PGALLOC); 1412 else 1413 count_vm_event(HTLB_BUDDY_PGALLOC_FAIL); 1414 1415 return ret; 1416 } 1417 1418 /* 1419 * Free huge page from pool from next node to free. 1420 * Attempt to keep persistent huge pages more or less 1421 * balanced over allowed nodes. 1422 * Called with hugetlb_lock locked. 1423 */ 1424 static int free_pool_huge_page(struct hstate *h, nodemask_t *nodes_allowed, 1425 bool acct_surplus) 1426 { 1427 int nr_nodes, node; 1428 int ret = 0; 1429 1430 for_each_node_mask_to_free(h, nr_nodes, node, nodes_allowed) { 1431 /* 1432 * If we're returning unused surplus pages, only examine 1433 * nodes with surplus pages. 1434 */ 1435 if ((!acct_surplus || h->surplus_huge_pages_node[node]) && 1436 !list_empty(&h->hugepage_freelists[node])) { 1437 struct page *page = 1438 list_entry(h->hugepage_freelists[node].next, 1439 struct page, lru); 1440 list_del(&page->lru); 1441 h->free_huge_pages--; 1442 h->free_huge_pages_node[node]--; 1443 if (acct_surplus) { 1444 h->surplus_huge_pages--; 1445 h->surplus_huge_pages_node[node]--; 1446 } 1447 update_and_free_page(h, page); 1448 ret = 1; 1449 break; 1450 } 1451 } 1452 1453 return ret; 1454 } 1455 1456 /* 1457 * Dissolve a given free hugepage into free buddy pages. This function does 1458 * nothing for in-use (including surplus) hugepages. Returns -EBUSY if the 1459 * number of free hugepages would be reduced below the number of reserved 1460 * hugepages. 1461 */ 1462 int dissolve_free_huge_page(struct page *page) 1463 { 1464 int rc = 0; 1465 1466 spin_lock(&hugetlb_lock); 1467 if (PageHuge(page) && !page_count(page)) { 1468 struct page *head = compound_head(page); 1469 struct hstate *h = page_hstate(head); 1470 int nid = page_to_nid(head); 1471 if (h->free_huge_pages - h->resv_huge_pages == 0) { 1472 rc = -EBUSY; 1473 goto out; 1474 } 1475 /* 1476 * Move PageHWPoison flag from head page to the raw error page, 1477 * which makes any subpages rather than the error page reusable. 1478 */ 1479 if (PageHWPoison(head) && page != head) { 1480 SetPageHWPoison(page); 1481 ClearPageHWPoison(head); 1482 } 1483 list_del(&head->lru); 1484 h->free_huge_pages--; 1485 h->free_huge_pages_node[nid]--; 1486 h->max_huge_pages--; 1487 update_and_free_page(h, head); 1488 } 1489 out: 1490 spin_unlock(&hugetlb_lock); 1491 return rc; 1492 } 1493 1494 /* 1495 * Dissolve free hugepages in a given pfn range. Used by memory hotplug to 1496 * make specified memory blocks removable from the system. 1497 * Note that this will dissolve a free gigantic hugepage completely, if any 1498 * part of it lies within the given range. 1499 * Also note that if dissolve_free_huge_page() returns with an error, all 1500 * free hugepages that were dissolved before that error are lost. 1501 */ 1502 int dissolve_free_huge_pages(unsigned long start_pfn, unsigned long end_pfn) 1503 { 1504 unsigned long pfn; 1505 struct page *page; 1506 int rc = 0; 1507 1508 if (!hugepages_supported()) 1509 return rc; 1510 1511 for (pfn = start_pfn; pfn < end_pfn; pfn += 1 << minimum_order) { 1512 page = pfn_to_page(pfn); 1513 if (PageHuge(page) && !page_count(page)) { 1514 rc = dissolve_free_huge_page(page); 1515 if (rc) 1516 break; 1517 } 1518 } 1519 1520 return rc; 1521 } 1522 1523 static struct page *__hugetlb_alloc_buddy_huge_page(struct hstate *h, 1524 gfp_t gfp_mask, int nid, nodemask_t *nmask) 1525 { 1526 int order = huge_page_order(h); 1527 1528 gfp_mask |= __GFP_COMP|__GFP_RETRY_MAYFAIL|__GFP_NOWARN; 1529 if (nid == NUMA_NO_NODE) 1530 nid = numa_mem_id(); 1531 return __alloc_pages_nodemask(gfp_mask, order, nid, nmask); 1532 } 1533 1534 static struct page *__alloc_buddy_huge_page(struct hstate *h, gfp_t gfp_mask, 1535 int nid, nodemask_t *nmask) 1536 { 1537 struct page *page; 1538 unsigned int r_nid; 1539 1540 if (hstate_is_gigantic(h)) 1541 return NULL; 1542 1543 /* 1544 * Assume we will successfully allocate the surplus page to 1545 * prevent racing processes from causing the surplus to exceed 1546 * overcommit 1547 * 1548 * This however introduces a different race, where a process B 1549 * tries to grow the static hugepage pool while alloc_pages() is 1550 * called by process A. B will only examine the per-node 1551 * counters in determining if surplus huge pages can be 1552 * converted to normal huge pages in adjust_pool_surplus(). A 1553 * won't be able to increment the per-node counter, until the 1554 * lock is dropped by B, but B doesn't drop hugetlb_lock until 1555 * no more huge pages can be converted from surplus to normal 1556 * state (and doesn't try to convert again). Thus, we have a 1557 * case where a surplus huge page exists, the pool is grown, and 1558 * the surplus huge page still exists after, even though it 1559 * should just have been converted to a normal huge page. This 1560 * does not leak memory, though, as the hugepage will be freed 1561 * once it is out of use. It also does not allow the counters to 1562 * go out of whack in adjust_pool_surplus() as we don't modify 1563 * the node values until we've gotten the hugepage and only the 1564 * per-node value is checked there. 1565 */ 1566 spin_lock(&hugetlb_lock); 1567 if (h->surplus_huge_pages >= h->nr_overcommit_huge_pages) { 1568 spin_unlock(&hugetlb_lock); 1569 return NULL; 1570 } else { 1571 h->nr_huge_pages++; 1572 h->surplus_huge_pages++; 1573 } 1574 spin_unlock(&hugetlb_lock); 1575 1576 page = __hugetlb_alloc_buddy_huge_page(h, gfp_mask, nid, nmask); 1577 1578 spin_lock(&hugetlb_lock); 1579 if (page) { 1580 INIT_LIST_HEAD(&page->lru); 1581 r_nid = page_to_nid(page); 1582 set_compound_page_dtor(page, HUGETLB_PAGE_DTOR); 1583 set_hugetlb_cgroup(page, NULL); 1584 /* 1585 * We incremented the global counters already 1586 */ 1587 h->nr_huge_pages_node[r_nid]++; 1588 h->surplus_huge_pages_node[r_nid]++; 1589 __count_vm_event(HTLB_BUDDY_PGALLOC); 1590 } else { 1591 h->nr_huge_pages--; 1592 h->surplus_huge_pages--; 1593 __count_vm_event(HTLB_BUDDY_PGALLOC_FAIL); 1594 } 1595 spin_unlock(&hugetlb_lock); 1596 1597 return page; 1598 } 1599 1600 /* 1601 * Use the VMA's mpolicy to allocate a huge page from the buddy. 1602 */ 1603 static 1604 struct page *__alloc_buddy_huge_page_with_mpol(struct hstate *h, 1605 struct vm_area_struct *vma, unsigned long addr) 1606 { 1607 struct page *page; 1608 struct mempolicy *mpol; 1609 gfp_t gfp_mask = htlb_alloc_mask(h); 1610 int nid; 1611 nodemask_t *nodemask; 1612 1613 nid = huge_node(vma, addr, gfp_mask, &mpol, &nodemask); 1614 page = __alloc_buddy_huge_page(h, gfp_mask, nid, nodemask); 1615 mpol_cond_put(mpol); 1616 1617 return page; 1618 } 1619 1620 /* 1621 * This allocation function is useful in the context where vma is irrelevant. 1622 * E.g. soft-offlining uses this function because it only cares physical 1623 * address of error page. 1624 */ 1625 struct page *alloc_huge_page_node(struct hstate *h, int nid) 1626 { 1627 gfp_t gfp_mask = htlb_alloc_mask(h); 1628 struct page *page = NULL; 1629 1630 if (nid != NUMA_NO_NODE) 1631 gfp_mask |= __GFP_THISNODE; 1632 1633 spin_lock(&hugetlb_lock); 1634 if (h->free_huge_pages - h->resv_huge_pages > 0) 1635 page = dequeue_huge_page_nodemask(h, gfp_mask, nid, NULL); 1636 spin_unlock(&hugetlb_lock); 1637 1638 if (!page) 1639 page = __alloc_buddy_huge_page(h, gfp_mask, nid, NULL); 1640 1641 return page; 1642 } 1643 1644 1645 struct page *alloc_huge_page_nodemask(struct hstate *h, int preferred_nid, 1646 nodemask_t *nmask) 1647 { 1648 gfp_t gfp_mask = htlb_alloc_mask(h); 1649 1650 spin_lock(&hugetlb_lock); 1651 if (h->free_huge_pages - h->resv_huge_pages > 0) { 1652 struct page *page; 1653 1654 page = dequeue_huge_page_nodemask(h, gfp_mask, preferred_nid, nmask); 1655 if (page) { 1656 spin_unlock(&hugetlb_lock); 1657 return page; 1658 } 1659 } 1660 spin_unlock(&hugetlb_lock); 1661 1662 /* No reservations, try to overcommit */ 1663 1664 return __alloc_buddy_huge_page(h, gfp_mask, preferred_nid, nmask); 1665 } 1666 1667 /* 1668 * Increase the hugetlb pool such that it can accommodate a reservation 1669 * of size 'delta'. 1670 */ 1671 static int gather_surplus_pages(struct hstate *h, int delta) 1672 { 1673 struct list_head surplus_list; 1674 struct page *page, *tmp; 1675 int ret, i; 1676 int needed, allocated; 1677 bool alloc_ok = true; 1678 1679 needed = (h->resv_huge_pages + delta) - h->free_huge_pages; 1680 if (needed <= 0) { 1681 h->resv_huge_pages += delta; 1682 return 0; 1683 } 1684 1685 allocated = 0; 1686 INIT_LIST_HEAD(&surplus_list); 1687 1688 ret = -ENOMEM; 1689 retry: 1690 spin_unlock(&hugetlb_lock); 1691 for (i = 0; i < needed; i++) { 1692 page = __alloc_buddy_huge_page(h, htlb_alloc_mask(h), 1693 NUMA_NO_NODE, NULL); 1694 if (!page) { 1695 alloc_ok = false; 1696 break; 1697 } 1698 list_add(&page->lru, &surplus_list); 1699 cond_resched(); 1700 } 1701 allocated += i; 1702 1703 /* 1704 * After retaking hugetlb_lock, we need to recalculate 'needed' 1705 * because either resv_huge_pages or free_huge_pages may have changed. 1706 */ 1707 spin_lock(&hugetlb_lock); 1708 needed = (h->resv_huge_pages + delta) - 1709 (h->free_huge_pages + allocated); 1710 if (needed > 0) { 1711 if (alloc_ok) 1712 goto retry; 1713 /* 1714 * We were not able to allocate enough pages to 1715 * satisfy the entire reservation so we free what 1716 * we've allocated so far. 1717 */ 1718 goto free; 1719 } 1720 /* 1721 * The surplus_list now contains _at_least_ the number of extra pages 1722 * needed to accommodate the reservation. Add the appropriate number 1723 * of pages to the hugetlb pool and free the extras back to the buddy 1724 * allocator. Commit the entire reservation here to prevent another 1725 * process from stealing the pages as they are added to the pool but 1726 * before they are reserved. 1727 */ 1728 needed += allocated; 1729 h->resv_huge_pages += delta; 1730 ret = 0; 1731 1732 /* Free the needed pages to the hugetlb pool */ 1733 list_for_each_entry_safe(page, tmp, &surplus_list, lru) { 1734 if ((--needed) < 0) 1735 break; 1736 /* 1737 * This page is now managed by the hugetlb allocator and has 1738 * no users -- drop the buddy allocator's reference. 1739 */ 1740 put_page_testzero(page); 1741 VM_BUG_ON_PAGE(page_count(page), page); 1742 enqueue_huge_page(h, page); 1743 } 1744 free: 1745 spin_unlock(&hugetlb_lock); 1746 1747 /* Free unnecessary surplus pages to the buddy allocator */ 1748 list_for_each_entry_safe(page, tmp, &surplus_list, lru) 1749 put_page(page); 1750 spin_lock(&hugetlb_lock); 1751 1752 return ret; 1753 } 1754 1755 /* 1756 * This routine has two main purposes: 1757 * 1) Decrement the reservation count (resv_huge_pages) by the value passed 1758 * in unused_resv_pages. This corresponds to the prior adjustments made 1759 * to the associated reservation map. 1760 * 2) Free any unused surplus pages that may have been allocated to satisfy 1761 * the reservation. As many as unused_resv_pages may be freed. 1762 * 1763 * Called with hugetlb_lock held. However, the lock could be dropped (and 1764 * reacquired) during calls to cond_resched_lock. Whenever dropping the lock, 1765 * we must make sure nobody else can claim pages we are in the process of 1766 * freeing. Do this by ensuring resv_huge_page always is greater than the 1767 * number of huge pages we plan to free when dropping the lock. 1768 */ 1769 static void return_unused_surplus_pages(struct hstate *h, 1770 unsigned long unused_resv_pages) 1771 { 1772 unsigned long nr_pages; 1773 1774 /* Cannot return gigantic pages currently */ 1775 if (hstate_is_gigantic(h)) 1776 goto out; 1777 1778 /* 1779 * Part (or even all) of the reservation could have been backed 1780 * by pre-allocated pages. Only free surplus pages. 1781 */ 1782 nr_pages = min(unused_resv_pages, h->surplus_huge_pages); 1783 1784 /* 1785 * We want to release as many surplus pages as possible, spread 1786 * evenly across all nodes with memory. Iterate across these nodes 1787 * until we can no longer free unreserved surplus pages. This occurs 1788 * when the nodes with surplus pages have no free pages. 1789 * free_pool_huge_page() will balance the the freed pages across the 1790 * on-line nodes with memory and will handle the hstate accounting. 1791 * 1792 * Note that we decrement resv_huge_pages as we free the pages. If 1793 * we drop the lock, resv_huge_pages will still be sufficiently large 1794 * to cover subsequent pages we may free. 1795 */ 1796 while (nr_pages--) { 1797 h->resv_huge_pages--; 1798 unused_resv_pages--; 1799 if (!free_pool_huge_page(h, &node_states[N_MEMORY], 1)) 1800 goto out; 1801 cond_resched_lock(&hugetlb_lock); 1802 } 1803 1804 out: 1805 /* Fully uncommit the reservation */ 1806 h->resv_huge_pages -= unused_resv_pages; 1807 } 1808 1809 1810 /* 1811 * vma_needs_reservation, vma_commit_reservation and vma_end_reservation 1812 * are used by the huge page allocation routines to manage reservations. 1813 * 1814 * vma_needs_reservation is called to determine if the huge page at addr 1815 * within the vma has an associated reservation. If a reservation is 1816 * needed, the value 1 is returned. The caller is then responsible for 1817 * managing the global reservation and subpool usage counts. After 1818 * the huge page has been allocated, vma_commit_reservation is called 1819 * to add the page to the reservation map. If the page allocation fails, 1820 * the reservation must be ended instead of committed. vma_end_reservation 1821 * is called in such cases. 1822 * 1823 * In the normal case, vma_commit_reservation returns the same value 1824 * as the preceding vma_needs_reservation call. The only time this 1825 * is not the case is if a reserve map was changed between calls. It 1826 * is the responsibility of the caller to notice the difference and 1827 * take appropriate action. 1828 * 1829 * vma_add_reservation is used in error paths where a reservation must 1830 * be restored when a newly allocated huge page must be freed. It is 1831 * to be called after calling vma_needs_reservation to determine if a 1832 * reservation exists. 1833 */ 1834 enum vma_resv_mode { 1835 VMA_NEEDS_RESV, 1836 VMA_COMMIT_RESV, 1837 VMA_END_RESV, 1838 VMA_ADD_RESV, 1839 }; 1840 static long __vma_reservation_common(struct hstate *h, 1841 struct vm_area_struct *vma, unsigned long addr, 1842 enum vma_resv_mode mode) 1843 { 1844 struct resv_map *resv; 1845 pgoff_t idx; 1846 long ret; 1847 1848 resv = vma_resv_map(vma); 1849 if (!resv) 1850 return 1; 1851 1852 idx = vma_hugecache_offset(h, vma, addr); 1853 switch (mode) { 1854 case VMA_NEEDS_RESV: 1855 ret = region_chg(resv, idx, idx + 1); 1856 break; 1857 case VMA_COMMIT_RESV: 1858 ret = region_add(resv, idx, idx + 1); 1859 break; 1860 case VMA_END_RESV: 1861 region_abort(resv, idx, idx + 1); 1862 ret = 0; 1863 break; 1864 case VMA_ADD_RESV: 1865 if (vma->vm_flags & VM_MAYSHARE) 1866 ret = region_add(resv, idx, idx + 1); 1867 else { 1868 region_abort(resv, idx, idx + 1); 1869 ret = region_del(resv, idx, idx + 1); 1870 } 1871 break; 1872 default: 1873 BUG(); 1874 } 1875 1876 if (vma->vm_flags & VM_MAYSHARE) 1877 return ret; 1878 else if (is_vma_resv_set(vma, HPAGE_RESV_OWNER) && ret >= 0) { 1879 /* 1880 * In most cases, reserves always exist for private mappings. 1881 * However, a file associated with mapping could have been 1882 * hole punched or truncated after reserves were consumed. 1883 * As subsequent fault on such a range will not use reserves. 1884 * Subtle - The reserve map for private mappings has the 1885 * opposite meaning than that of shared mappings. If NO 1886 * entry is in the reserve map, it means a reservation exists. 1887 * If an entry exists in the reserve map, it means the 1888 * reservation has already been consumed. As a result, the 1889 * return value of this routine is the opposite of the 1890 * value returned from reserve map manipulation routines above. 1891 */ 1892 if (ret) 1893 return 0; 1894 else 1895 return 1; 1896 } 1897 else 1898 return ret < 0 ? ret : 0; 1899 } 1900 1901 static long vma_needs_reservation(struct hstate *h, 1902 struct vm_area_struct *vma, unsigned long addr) 1903 { 1904 return __vma_reservation_common(h, vma, addr, VMA_NEEDS_RESV); 1905 } 1906 1907 static long vma_commit_reservation(struct hstate *h, 1908 struct vm_area_struct *vma, unsigned long addr) 1909 { 1910 return __vma_reservation_common(h, vma, addr, VMA_COMMIT_RESV); 1911 } 1912 1913 static void vma_end_reservation(struct hstate *h, 1914 struct vm_area_struct *vma, unsigned long addr) 1915 { 1916 (void)__vma_reservation_common(h, vma, addr, VMA_END_RESV); 1917 } 1918 1919 static long vma_add_reservation(struct hstate *h, 1920 struct vm_area_struct *vma, unsigned long addr) 1921 { 1922 return __vma_reservation_common(h, vma, addr, VMA_ADD_RESV); 1923 } 1924 1925 /* 1926 * This routine is called to restore a reservation on error paths. In the 1927 * specific error paths, a huge page was allocated (via alloc_huge_page) 1928 * and is about to be freed. If a reservation for the page existed, 1929 * alloc_huge_page would have consumed the reservation and set PagePrivate 1930 * in the newly allocated page. When the page is freed via free_huge_page, 1931 * the global reservation count will be incremented if PagePrivate is set. 1932 * However, free_huge_page can not adjust the reserve map. Adjust the 1933 * reserve map here to be consistent with global reserve count adjustments 1934 * to be made by free_huge_page. 1935 */ 1936 static void restore_reserve_on_error(struct hstate *h, 1937 struct vm_area_struct *vma, unsigned long address, 1938 struct page *page) 1939 { 1940 if (unlikely(PagePrivate(page))) { 1941 long rc = vma_needs_reservation(h, vma, address); 1942 1943 if (unlikely(rc < 0)) { 1944 /* 1945 * Rare out of memory condition in reserve map 1946 * manipulation. Clear PagePrivate so that 1947 * global reserve count will not be incremented 1948 * by free_huge_page. This will make it appear 1949 * as though the reservation for this page was 1950 * consumed. This may prevent the task from 1951 * faulting in the page at a later time. This 1952 * is better than inconsistent global huge page 1953 * accounting of reserve counts. 1954 */ 1955 ClearPagePrivate(page); 1956 } else if (rc) { 1957 rc = vma_add_reservation(h, vma, address); 1958 if (unlikely(rc < 0)) 1959 /* 1960 * See above comment about rare out of 1961 * memory condition. 1962 */ 1963 ClearPagePrivate(page); 1964 } else 1965 vma_end_reservation(h, vma, address); 1966 } 1967 } 1968 1969 struct page *alloc_huge_page(struct vm_area_struct *vma, 1970 unsigned long addr, int avoid_reserve) 1971 { 1972 struct hugepage_subpool *spool = subpool_vma(vma); 1973 struct hstate *h = hstate_vma(vma); 1974 struct page *page; 1975 long map_chg, map_commit; 1976 long gbl_chg; 1977 int ret, idx; 1978 struct hugetlb_cgroup *h_cg; 1979 1980 idx = hstate_index(h); 1981 /* 1982 * Examine the region/reserve map to determine if the process 1983 * has a reservation for the page to be allocated. A return 1984 * code of zero indicates a reservation exists (no change). 1985 */ 1986 map_chg = gbl_chg = vma_needs_reservation(h, vma, addr); 1987 if (map_chg < 0) 1988 return ERR_PTR(-ENOMEM); 1989 1990 /* 1991 * Processes that did not create the mapping will have no 1992 * reserves as indicated by the region/reserve map. Check 1993 * that the allocation will not exceed the subpool limit. 1994 * Allocations for MAP_NORESERVE mappings also need to be 1995 * checked against any subpool limit. 1996 */ 1997 if (map_chg || avoid_reserve) { 1998 gbl_chg = hugepage_subpool_get_pages(spool, 1); 1999 if (gbl_chg < 0) { 2000 vma_end_reservation(h, vma, addr); 2001 return ERR_PTR(-ENOSPC); 2002 } 2003 2004 /* 2005 * Even though there was no reservation in the region/reserve 2006 * map, there could be reservations associated with the 2007 * subpool that can be used. This would be indicated if the 2008 * return value of hugepage_subpool_get_pages() is zero. 2009 * However, if avoid_reserve is specified we still avoid even 2010 * the subpool reservations. 2011 */ 2012 if (avoid_reserve) 2013 gbl_chg = 1; 2014 } 2015 2016 ret = hugetlb_cgroup_charge_cgroup(idx, pages_per_huge_page(h), &h_cg); 2017 if (ret) 2018 goto out_subpool_put; 2019 2020 spin_lock(&hugetlb_lock); 2021 /* 2022 * glb_chg is passed to indicate whether or not a page must be taken 2023 * from the global free pool (global change). gbl_chg == 0 indicates 2024 * a reservation exists for the allocation. 2025 */ 2026 page = dequeue_huge_page_vma(h, vma, addr, avoid_reserve, gbl_chg); 2027 if (!page) { 2028 spin_unlock(&hugetlb_lock); 2029 page = __alloc_buddy_huge_page_with_mpol(h, vma, addr); 2030 if (!page) 2031 goto out_uncharge_cgroup; 2032 if (!avoid_reserve && vma_has_reserves(vma, gbl_chg)) { 2033 SetPagePrivate(page); 2034 h->resv_huge_pages--; 2035 } 2036 spin_lock(&hugetlb_lock); 2037 list_move(&page->lru, &h->hugepage_activelist); 2038 /* Fall through */ 2039 } 2040 hugetlb_cgroup_commit_charge(idx, pages_per_huge_page(h), h_cg, page); 2041 spin_unlock(&hugetlb_lock); 2042 2043 set_page_private(page, (unsigned long)spool); 2044 2045 map_commit = vma_commit_reservation(h, vma, addr); 2046 if (unlikely(map_chg > map_commit)) { 2047 /* 2048 * The page was added to the reservation map between 2049 * vma_needs_reservation and vma_commit_reservation. 2050 * This indicates a race with hugetlb_reserve_pages. 2051 * Adjust for the subpool count incremented above AND 2052 * in hugetlb_reserve_pages for the same page. Also, 2053 * the reservation count added in hugetlb_reserve_pages 2054 * no longer applies. 2055 */ 2056 long rsv_adjust; 2057 2058 rsv_adjust = hugepage_subpool_put_pages(spool, 1); 2059 hugetlb_acct_memory(h, -rsv_adjust); 2060 } 2061 return page; 2062 2063 out_uncharge_cgroup: 2064 hugetlb_cgroup_uncharge_cgroup(idx, pages_per_huge_page(h), h_cg); 2065 out_subpool_put: 2066 if (map_chg || avoid_reserve) 2067 hugepage_subpool_put_pages(spool, 1); 2068 vma_end_reservation(h, vma, addr); 2069 return ERR_PTR(-ENOSPC); 2070 } 2071 2072 /* 2073 * alloc_huge_page()'s wrapper which simply returns the page if allocation 2074 * succeeds, otherwise NULL. This function is called from new_vma_page(), 2075 * where no ERR_VALUE is expected to be returned. 2076 */ 2077 struct page *alloc_huge_page_noerr(struct vm_area_struct *vma, 2078 unsigned long addr, int avoid_reserve) 2079 { 2080 struct page *page = alloc_huge_page(vma, addr, avoid_reserve); 2081 if (IS_ERR(page)) 2082 page = NULL; 2083 return page; 2084 } 2085 2086 int __weak alloc_bootmem_huge_page(struct hstate *h) 2087 { 2088 struct huge_bootmem_page *m; 2089 int nr_nodes, node; 2090 2091 for_each_node_mask_to_alloc(h, nr_nodes, node, &node_states[N_MEMORY]) { 2092 void *addr; 2093 2094 addr = memblock_virt_alloc_try_nid_nopanic( 2095 huge_page_size(h), huge_page_size(h), 2096 0, BOOTMEM_ALLOC_ACCESSIBLE, node); 2097 if (addr) { 2098 /* 2099 * Use the beginning of the huge page to store the 2100 * huge_bootmem_page struct (until gather_bootmem 2101 * puts them into the mem_map). 2102 */ 2103 m = addr; 2104 goto found; 2105 } 2106 } 2107 return 0; 2108 2109 found: 2110 BUG_ON(!IS_ALIGNED(virt_to_phys(m), huge_page_size(h))); 2111 /* Put them into a private list first because mem_map is not up yet */ 2112 list_add(&m->list, &huge_boot_pages); 2113 m->hstate = h; 2114 return 1; 2115 } 2116 2117 static void __init prep_compound_huge_page(struct page *page, 2118 unsigned int order) 2119 { 2120 if (unlikely(order > (MAX_ORDER - 1))) 2121 prep_compound_gigantic_page(page, order); 2122 else 2123 prep_compound_page(page, order); 2124 } 2125 2126 /* Put bootmem huge pages into the standard lists after mem_map is up */ 2127 static void __init gather_bootmem_prealloc(void) 2128 { 2129 struct huge_bootmem_page *m; 2130 2131 list_for_each_entry(m, &huge_boot_pages, list) { 2132 struct hstate *h = m->hstate; 2133 struct page *page; 2134 2135 #ifdef CONFIG_HIGHMEM 2136 page = pfn_to_page(m->phys >> PAGE_SHIFT); 2137 memblock_free_late(__pa(m), 2138 sizeof(struct huge_bootmem_page)); 2139 #else 2140 page = virt_to_page(m); 2141 #endif 2142 WARN_ON(page_count(page) != 1); 2143 prep_compound_huge_page(page, h->order); 2144 WARN_ON(PageReserved(page)); 2145 prep_new_huge_page(h, page, page_to_nid(page)); 2146 /* 2147 * If we had gigantic hugepages allocated at boot time, we need 2148 * to restore the 'stolen' pages to totalram_pages in order to 2149 * fix confusing memory reports from free(1) and another 2150 * side-effects, like CommitLimit going negative. 2151 */ 2152 if (hstate_is_gigantic(h)) 2153 adjust_managed_page_count(page, 1 << h->order); 2154 } 2155 } 2156 2157 static void __init hugetlb_hstate_alloc_pages(struct hstate *h) 2158 { 2159 unsigned long i; 2160 2161 for (i = 0; i < h->max_huge_pages; ++i) { 2162 if (hstate_is_gigantic(h)) { 2163 if (!alloc_bootmem_huge_page(h)) 2164 break; 2165 } else if (!alloc_fresh_huge_page(h, 2166 &node_states[N_MEMORY])) 2167 break; 2168 cond_resched(); 2169 } 2170 if (i < h->max_huge_pages) { 2171 char buf[32]; 2172 2173 string_get_size(huge_page_size(h), 1, STRING_UNITS_2, buf, 32); 2174 pr_warn("HugeTLB: allocating %lu of page size %s failed. Only allocated %lu hugepages.\n", 2175 h->max_huge_pages, buf, i); 2176 h->max_huge_pages = i; 2177 } 2178 } 2179 2180 static void __init hugetlb_init_hstates(void) 2181 { 2182 struct hstate *h; 2183 2184 for_each_hstate(h) { 2185 if (minimum_order > huge_page_order(h)) 2186 minimum_order = huge_page_order(h); 2187 2188 /* oversize hugepages were init'ed in early boot */ 2189 if (!hstate_is_gigantic(h)) 2190 hugetlb_hstate_alloc_pages(h); 2191 } 2192 VM_BUG_ON(minimum_order == UINT_MAX); 2193 } 2194 2195 static void __init report_hugepages(void) 2196 { 2197 struct hstate *h; 2198 2199 for_each_hstate(h) { 2200 char buf[32]; 2201 2202 string_get_size(huge_page_size(h), 1, STRING_UNITS_2, buf, 32); 2203 pr_info("HugeTLB registered %s page size, pre-allocated %ld pages\n", 2204 buf, h->free_huge_pages); 2205 } 2206 } 2207 2208 #ifdef CONFIG_HIGHMEM 2209 static void try_to_free_low(struct hstate *h, unsigned long count, 2210 nodemask_t *nodes_allowed) 2211 { 2212 int i; 2213 2214 if (hstate_is_gigantic(h)) 2215 return; 2216 2217 for_each_node_mask(i, *nodes_allowed) { 2218 struct page *page, *next; 2219 struct list_head *freel = &h->hugepage_freelists[i]; 2220 list_for_each_entry_safe(page, next, freel, lru) { 2221 if (count >= h->nr_huge_pages) 2222 return; 2223 if (PageHighMem(page)) 2224 continue; 2225 list_del(&page->lru); 2226 update_and_free_page(h, page); 2227 h->free_huge_pages--; 2228 h->free_huge_pages_node[page_to_nid(page)]--; 2229 } 2230 } 2231 } 2232 #else 2233 static inline void try_to_free_low(struct hstate *h, unsigned long count, 2234 nodemask_t *nodes_allowed) 2235 { 2236 } 2237 #endif 2238 2239 /* 2240 * Increment or decrement surplus_huge_pages. Keep node-specific counters 2241 * balanced by operating on them in a round-robin fashion. 2242 * Returns 1 if an adjustment was made. 2243 */ 2244 static int adjust_pool_surplus(struct hstate *h, nodemask_t *nodes_allowed, 2245 int delta) 2246 { 2247 int nr_nodes, node; 2248 2249 VM_BUG_ON(delta != -1 && delta != 1); 2250 2251 if (delta < 0) { 2252 for_each_node_mask_to_alloc(h, nr_nodes, node, nodes_allowed) { 2253 if (h->surplus_huge_pages_node[node]) 2254 goto found; 2255 } 2256 } else { 2257 for_each_node_mask_to_free(h, nr_nodes, node, nodes_allowed) { 2258 if (h->surplus_huge_pages_node[node] < 2259 h->nr_huge_pages_node[node]) 2260 goto found; 2261 } 2262 } 2263 return 0; 2264 2265 found: 2266 h->surplus_huge_pages += delta; 2267 h->surplus_huge_pages_node[node] += delta; 2268 return 1; 2269 } 2270 2271 #define persistent_huge_pages(h) (h->nr_huge_pages - h->surplus_huge_pages) 2272 static unsigned long set_max_huge_pages(struct hstate *h, unsigned long count, 2273 nodemask_t *nodes_allowed) 2274 { 2275 unsigned long min_count, ret; 2276 2277 if (hstate_is_gigantic(h) && !gigantic_page_supported()) 2278 return h->max_huge_pages; 2279 2280 /* 2281 * Increase the pool size 2282 * First take pages out of surplus state. Then make up the 2283 * remaining difference by allocating fresh huge pages. 2284 * 2285 * We might race with __alloc_buddy_huge_page() here and be unable 2286 * to convert a surplus huge page to a normal huge page. That is 2287 * not critical, though, it just means the overall size of the 2288 * pool might be one hugepage larger than it needs to be, but 2289 * within all the constraints specified by the sysctls. 2290 */ 2291 spin_lock(&hugetlb_lock); 2292 while (h->surplus_huge_pages && count > persistent_huge_pages(h)) { 2293 if (!adjust_pool_surplus(h, nodes_allowed, -1)) 2294 break; 2295 } 2296 2297 while (count > persistent_huge_pages(h)) { 2298 /* 2299 * If this allocation races such that we no longer need the 2300 * page, free_huge_page will handle it by freeing the page 2301 * and reducing the surplus. 2302 */ 2303 spin_unlock(&hugetlb_lock); 2304 2305 /* yield cpu to avoid soft lockup */ 2306 cond_resched(); 2307 2308 if (hstate_is_gigantic(h)) 2309 ret = alloc_fresh_gigantic_page(h, nodes_allowed); 2310 else 2311 ret = alloc_fresh_huge_page(h, nodes_allowed); 2312 spin_lock(&hugetlb_lock); 2313 if (!ret) 2314 goto out; 2315 2316 /* Bail for signals. Probably ctrl-c from user */ 2317 if (signal_pending(current)) 2318 goto out; 2319 } 2320 2321 /* 2322 * Decrease the pool size 2323 * First return free pages to the buddy allocator (being careful 2324 * to keep enough around to satisfy reservations). Then place 2325 * pages into surplus state as needed so the pool will shrink 2326 * to the desired size as pages become free. 2327 * 2328 * By placing pages into the surplus state independent of the 2329 * overcommit value, we are allowing the surplus pool size to 2330 * exceed overcommit. There are few sane options here. Since 2331 * __alloc_buddy_huge_page() is checking the global counter, 2332 * though, we'll note that we're not allowed to exceed surplus 2333 * and won't grow the pool anywhere else. Not until one of the 2334 * sysctls are changed, or the surplus pages go out of use. 2335 */ 2336 min_count = h->resv_huge_pages + h->nr_huge_pages - h->free_huge_pages; 2337 min_count = max(count, min_count); 2338 try_to_free_low(h, min_count, nodes_allowed); 2339 while (min_count < persistent_huge_pages(h)) { 2340 if (!free_pool_huge_page(h, nodes_allowed, 0)) 2341 break; 2342 cond_resched_lock(&hugetlb_lock); 2343 } 2344 while (count < persistent_huge_pages(h)) { 2345 if (!adjust_pool_surplus(h, nodes_allowed, 1)) 2346 break; 2347 } 2348 out: 2349 ret = persistent_huge_pages(h); 2350 spin_unlock(&hugetlb_lock); 2351 return ret; 2352 } 2353 2354 #define HSTATE_ATTR_RO(_name) \ 2355 static struct kobj_attribute _name##_attr = __ATTR_RO(_name) 2356 2357 #define HSTATE_ATTR(_name) \ 2358 static struct kobj_attribute _name##_attr = \ 2359 __ATTR(_name, 0644, _name##_show, _name##_store) 2360 2361 static struct kobject *hugepages_kobj; 2362 static struct kobject *hstate_kobjs[HUGE_MAX_HSTATE]; 2363 2364 static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp); 2365 2366 static struct hstate *kobj_to_hstate(struct kobject *kobj, int *nidp) 2367 { 2368 int i; 2369 2370 for (i = 0; i < HUGE_MAX_HSTATE; i++) 2371 if (hstate_kobjs[i] == kobj) { 2372 if (nidp) 2373 *nidp = NUMA_NO_NODE; 2374 return &hstates[i]; 2375 } 2376 2377 return kobj_to_node_hstate(kobj, nidp); 2378 } 2379 2380 static ssize_t nr_hugepages_show_common(struct kobject *kobj, 2381 struct kobj_attribute *attr, char *buf) 2382 { 2383 struct hstate *h; 2384 unsigned long nr_huge_pages; 2385 int nid; 2386 2387 h = kobj_to_hstate(kobj, &nid); 2388 if (nid == NUMA_NO_NODE) 2389 nr_huge_pages = h->nr_huge_pages; 2390 else 2391 nr_huge_pages = h->nr_huge_pages_node[nid]; 2392 2393 return sprintf(buf, "%lu\n", nr_huge_pages); 2394 } 2395 2396 static ssize_t __nr_hugepages_store_common(bool obey_mempolicy, 2397 struct hstate *h, int nid, 2398 unsigned long count, size_t len) 2399 { 2400 int err; 2401 NODEMASK_ALLOC(nodemask_t, nodes_allowed, GFP_KERNEL | __GFP_NORETRY); 2402 2403 if (hstate_is_gigantic(h) && !gigantic_page_supported()) { 2404 err = -EINVAL; 2405 goto out; 2406 } 2407 2408 if (nid == NUMA_NO_NODE) { 2409 /* 2410 * global hstate attribute 2411 */ 2412 if (!(obey_mempolicy && 2413 init_nodemask_of_mempolicy(nodes_allowed))) { 2414 NODEMASK_FREE(nodes_allowed); 2415 nodes_allowed = &node_states[N_MEMORY]; 2416 } 2417 } else if (nodes_allowed) { 2418 /* 2419 * per node hstate attribute: adjust count to global, 2420 * but restrict alloc/free to the specified node. 2421 */ 2422 count += h->nr_huge_pages - h->nr_huge_pages_node[nid]; 2423 init_nodemask_of_node(nodes_allowed, nid); 2424 } else 2425 nodes_allowed = &node_states[N_MEMORY]; 2426 2427 h->max_huge_pages = set_max_huge_pages(h, count, nodes_allowed); 2428 2429 if (nodes_allowed != &node_states[N_MEMORY]) 2430 NODEMASK_FREE(nodes_allowed); 2431 2432 return len; 2433 out: 2434 NODEMASK_FREE(nodes_allowed); 2435 return err; 2436 } 2437 2438 static ssize_t nr_hugepages_store_common(bool obey_mempolicy, 2439 struct kobject *kobj, const char *buf, 2440 size_t len) 2441 { 2442 struct hstate *h; 2443 unsigned long count; 2444 int nid; 2445 int err; 2446 2447 err = kstrtoul(buf, 10, &count); 2448 if (err) 2449 return err; 2450 2451 h = kobj_to_hstate(kobj, &nid); 2452 return __nr_hugepages_store_common(obey_mempolicy, h, nid, count, len); 2453 } 2454 2455 static ssize_t nr_hugepages_show(struct kobject *kobj, 2456 struct kobj_attribute *attr, char *buf) 2457 { 2458 return nr_hugepages_show_common(kobj, attr, buf); 2459 } 2460 2461 static ssize_t nr_hugepages_store(struct kobject *kobj, 2462 struct kobj_attribute *attr, const char *buf, size_t len) 2463 { 2464 return nr_hugepages_store_common(false, kobj, buf, len); 2465 } 2466 HSTATE_ATTR(nr_hugepages); 2467 2468 #ifdef CONFIG_NUMA 2469 2470 /* 2471 * hstate attribute for optionally mempolicy-based constraint on persistent 2472 * huge page alloc/free. 2473 */ 2474 static ssize_t nr_hugepages_mempolicy_show(struct kobject *kobj, 2475 struct kobj_attribute *attr, char *buf) 2476 { 2477 return nr_hugepages_show_common(kobj, attr, buf); 2478 } 2479 2480 static ssize_t nr_hugepages_mempolicy_store(struct kobject *kobj, 2481 struct kobj_attribute *attr, const char *buf, size_t len) 2482 { 2483 return nr_hugepages_store_common(true, kobj, buf, len); 2484 } 2485 HSTATE_ATTR(nr_hugepages_mempolicy); 2486 #endif 2487 2488 2489 static ssize_t nr_overcommit_hugepages_show(struct kobject *kobj, 2490 struct kobj_attribute *attr, char *buf) 2491 { 2492 struct hstate *h = kobj_to_hstate(kobj, NULL); 2493 return sprintf(buf, "%lu\n", h->nr_overcommit_huge_pages); 2494 } 2495 2496 static ssize_t nr_overcommit_hugepages_store(struct kobject *kobj, 2497 struct kobj_attribute *attr, const char *buf, size_t count) 2498 { 2499 int err; 2500 unsigned long input; 2501 struct hstate *h = kobj_to_hstate(kobj, NULL); 2502 2503 if (hstate_is_gigantic(h)) 2504 return -EINVAL; 2505 2506 err = kstrtoul(buf, 10, &input); 2507 if (err) 2508 return err; 2509 2510 spin_lock(&hugetlb_lock); 2511 h->nr_overcommit_huge_pages = input; 2512 spin_unlock(&hugetlb_lock); 2513 2514 return count; 2515 } 2516 HSTATE_ATTR(nr_overcommit_hugepages); 2517 2518 static ssize_t free_hugepages_show(struct kobject *kobj, 2519 struct kobj_attribute *attr, char *buf) 2520 { 2521 struct hstate *h; 2522 unsigned long free_huge_pages; 2523 int nid; 2524 2525 h = kobj_to_hstate(kobj, &nid); 2526 if (nid == NUMA_NO_NODE) 2527 free_huge_pages = h->free_huge_pages; 2528 else 2529 free_huge_pages = h->free_huge_pages_node[nid]; 2530 2531 return sprintf(buf, "%lu\n", free_huge_pages); 2532 } 2533 HSTATE_ATTR_RO(free_hugepages); 2534 2535 static ssize_t resv_hugepages_show(struct kobject *kobj, 2536 struct kobj_attribute *attr, char *buf) 2537 { 2538 struct hstate *h = kobj_to_hstate(kobj, NULL); 2539 return sprintf(buf, "%lu\n", h->resv_huge_pages); 2540 } 2541 HSTATE_ATTR_RO(resv_hugepages); 2542 2543 static ssize_t surplus_hugepages_show(struct kobject *kobj, 2544 struct kobj_attribute *attr, char *buf) 2545 { 2546 struct hstate *h; 2547 unsigned long surplus_huge_pages; 2548 int nid; 2549 2550 h = kobj_to_hstate(kobj, &nid); 2551 if (nid == NUMA_NO_NODE) 2552 surplus_huge_pages = h->surplus_huge_pages; 2553 else 2554 surplus_huge_pages = h->surplus_huge_pages_node[nid]; 2555 2556 return sprintf(buf, "%lu\n", surplus_huge_pages); 2557 } 2558 HSTATE_ATTR_RO(surplus_hugepages); 2559 2560 static struct attribute *hstate_attrs[] = { 2561 &nr_hugepages_attr.attr, 2562 &nr_overcommit_hugepages_attr.attr, 2563 &free_hugepages_attr.attr, 2564 &resv_hugepages_attr.attr, 2565 &surplus_hugepages_attr.attr, 2566 #ifdef CONFIG_NUMA 2567 &nr_hugepages_mempolicy_attr.attr, 2568 #endif 2569 NULL, 2570 }; 2571 2572 static struct attribute_group hstate_attr_group = { 2573 .attrs = hstate_attrs, 2574 }; 2575 2576 static int hugetlb_sysfs_add_hstate(struct hstate *h, struct kobject *parent, 2577 struct kobject **hstate_kobjs, 2578 struct attribute_group *hstate_attr_group) 2579 { 2580 int retval; 2581 int hi = hstate_index(h); 2582 2583 hstate_kobjs[hi] = kobject_create_and_add(h->name, parent); 2584 if (!hstate_kobjs[hi]) 2585 return -ENOMEM; 2586 2587 retval = sysfs_create_group(hstate_kobjs[hi], hstate_attr_group); 2588 if (retval) 2589 kobject_put(hstate_kobjs[hi]); 2590 2591 return retval; 2592 } 2593 2594 static void __init hugetlb_sysfs_init(void) 2595 { 2596 struct hstate *h; 2597 int err; 2598 2599 hugepages_kobj = kobject_create_and_add("hugepages", mm_kobj); 2600 if (!hugepages_kobj) 2601 return; 2602 2603 for_each_hstate(h) { 2604 err = hugetlb_sysfs_add_hstate(h, hugepages_kobj, 2605 hstate_kobjs, &hstate_attr_group); 2606 if (err) 2607 pr_err("Hugetlb: Unable to add hstate %s", h->name); 2608 } 2609 } 2610 2611 #ifdef CONFIG_NUMA 2612 2613 /* 2614 * node_hstate/s - associate per node hstate attributes, via their kobjects, 2615 * with node devices in node_devices[] using a parallel array. The array 2616 * index of a node device or _hstate == node id. 2617 * This is here to avoid any static dependency of the node device driver, in 2618 * the base kernel, on the hugetlb module. 2619 */ 2620 struct node_hstate { 2621 struct kobject *hugepages_kobj; 2622 struct kobject *hstate_kobjs[HUGE_MAX_HSTATE]; 2623 }; 2624 static struct node_hstate node_hstates[MAX_NUMNODES]; 2625 2626 /* 2627 * A subset of global hstate attributes for node devices 2628 */ 2629 static struct attribute *per_node_hstate_attrs[] = { 2630 &nr_hugepages_attr.attr, 2631 &free_hugepages_attr.attr, 2632 &surplus_hugepages_attr.attr, 2633 NULL, 2634 }; 2635 2636 static struct attribute_group per_node_hstate_attr_group = { 2637 .attrs = per_node_hstate_attrs, 2638 }; 2639 2640 /* 2641 * kobj_to_node_hstate - lookup global hstate for node device hstate attr kobj. 2642 * Returns node id via non-NULL nidp. 2643 */ 2644 static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp) 2645 { 2646 int nid; 2647 2648 for (nid = 0; nid < nr_node_ids; nid++) { 2649 struct node_hstate *nhs = &node_hstates[nid]; 2650 int i; 2651 for (i = 0; i < HUGE_MAX_HSTATE; i++) 2652 if (nhs->hstate_kobjs[i] == kobj) { 2653 if (nidp) 2654 *nidp = nid; 2655 return &hstates[i]; 2656 } 2657 } 2658 2659 BUG(); 2660 return NULL; 2661 } 2662 2663 /* 2664 * Unregister hstate attributes from a single node device. 2665 * No-op if no hstate attributes attached. 2666 */ 2667 static void hugetlb_unregister_node(struct node *node) 2668 { 2669 struct hstate *h; 2670 struct node_hstate *nhs = &node_hstates[node->dev.id]; 2671 2672 if (!nhs->hugepages_kobj) 2673 return; /* no hstate attributes */ 2674 2675 for_each_hstate(h) { 2676 int idx = hstate_index(h); 2677 if (nhs->hstate_kobjs[idx]) { 2678 kobject_put(nhs->hstate_kobjs[idx]); 2679 nhs->hstate_kobjs[idx] = NULL; 2680 } 2681 } 2682 2683 kobject_put(nhs->hugepages_kobj); 2684 nhs->hugepages_kobj = NULL; 2685 } 2686 2687 2688 /* 2689 * Register hstate attributes for a single node device. 2690 * No-op if attributes already registered. 2691 */ 2692 static void hugetlb_register_node(struct node *node) 2693 { 2694 struct hstate *h; 2695 struct node_hstate *nhs = &node_hstates[node->dev.id]; 2696 int err; 2697 2698 if (nhs->hugepages_kobj) 2699 return; /* already allocated */ 2700 2701 nhs->hugepages_kobj = kobject_create_and_add("hugepages", 2702 &node->dev.kobj); 2703 if (!nhs->hugepages_kobj) 2704 return; 2705 2706 for_each_hstate(h) { 2707 err = hugetlb_sysfs_add_hstate(h, nhs->hugepages_kobj, 2708 nhs->hstate_kobjs, 2709 &per_node_hstate_attr_group); 2710 if (err) { 2711 pr_err("Hugetlb: Unable to add hstate %s for node %d\n", 2712 h->name, node->dev.id); 2713 hugetlb_unregister_node(node); 2714 break; 2715 } 2716 } 2717 } 2718 2719 /* 2720 * hugetlb init time: register hstate attributes for all registered node 2721 * devices of nodes that have memory. All on-line nodes should have 2722 * registered their associated device by this time. 2723 */ 2724 static void __init hugetlb_register_all_nodes(void) 2725 { 2726 int nid; 2727 2728 for_each_node_state(nid, N_MEMORY) { 2729 struct node *node = node_devices[nid]; 2730 if (node->dev.id == nid) 2731 hugetlb_register_node(node); 2732 } 2733 2734 /* 2735 * Let the node device driver know we're here so it can 2736 * [un]register hstate attributes on node hotplug. 2737 */ 2738 register_hugetlbfs_with_node(hugetlb_register_node, 2739 hugetlb_unregister_node); 2740 } 2741 #else /* !CONFIG_NUMA */ 2742 2743 static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp) 2744 { 2745 BUG(); 2746 if (nidp) 2747 *nidp = -1; 2748 return NULL; 2749 } 2750 2751 static void hugetlb_register_all_nodes(void) { } 2752 2753 #endif 2754 2755 static int __init hugetlb_init(void) 2756 { 2757 int i; 2758 2759 if (!hugepages_supported()) 2760 return 0; 2761 2762 if (!size_to_hstate(default_hstate_size)) { 2763 if (default_hstate_size != 0) { 2764 pr_err("HugeTLB: unsupported default_hugepagesz %lu. Reverting to %lu\n", 2765 default_hstate_size, HPAGE_SIZE); 2766 } 2767 2768 default_hstate_size = HPAGE_SIZE; 2769 if (!size_to_hstate(default_hstate_size)) 2770 hugetlb_add_hstate(HUGETLB_PAGE_ORDER); 2771 } 2772 default_hstate_idx = hstate_index(size_to_hstate(default_hstate_size)); 2773 if (default_hstate_max_huge_pages) { 2774 if (!default_hstate.max_huge_pages) 2775 default_hstate.max_huge_pages = default_hstate_max_huge_pages; 2776 } 2777 2778 hugetlb_init_hstates(); 2779 gather_bootmem_prealloc(); 2780 report_hugepages(); 2781 2782 hugetlb_sysfs_init(); 2783 hugetlb_register_all_nodes(); 2784 hugetlb_cgroup_file_init(); 2785 2786 #ifdef CONFIG_SMP 2787 num_fault_mutexes = roundup_pow_of_two(8 * num_possible_cpus()); 2788 #else 2789 num_fault_mutexes = 1; 2790 #endif 2791 hugetlb_fault_mutex_table = 2792 kmalloc(sizeof(struct mutex) * num_fault_mutexes, GFP_KERNEL); 2793 BUG_ON(!hugetlb_fault_mutex_table); 2794 2795 for (i = 0; i < num_fault_mutexes; i++) 2796 mutex_init(&hugetlb_fault_mutex_table[i]); 2797 return 0; 2798 } 2799 subsys_initcall(hugetlb_init); 2800 2801 /* Should be called on processing a hugepagesz=... option */ 2802 void __init hugetlb_bad_size(void) 2803 { 2804 parsed_valid_hugepagesz = false; 2805 } 2806 2807 void __init hugetlb_add_hstate(unsigned int order) 2808 { 2809 struct hstate *h; 2810 unsigned long i; 2811 2812 if (size_to_hstate(PAGE_SIZE << order)) { 2813 pr_warn("hugepagesz= specified twice, ignoring\n"); 2814 return; 2815 } 2816 BUG_ON(hugetlb_max_hstate >= HUGE_MAX_HSTATE); 2817 BUG_ON(order == 0); 2818 h = &hstates[hugetlb_max_hstate++]; 2819 h->order = order; 2820 h->mask = ~((1ULL << (order + PAGE_SHIFT)) - 1); 2821 h->nr_huge_pages = 0; 2822 h->free_huge_pages = 0; 2823 for (i = 0; i < MAX_NUMNODES; ++i) 2824 INIT_LIST_HEAD(&h->hugepage_freelists[i]); 2825 INIT_LIST_HEAD(&h->hugepage_activelist); 2826 h->next_nid_to_alloc = first_memory_node; 2827 h->next_nid_to_free = first_memory_node; 2828 snprintf(h->name, HSTATE_NAME_LEN, "hugepages-%lukB", 2829 huge_page_size(h)/1024); 2830 2831 parsed_hstate = h; 2832 } 2833 2834 static int __init hugetlb_nrpages_setup(char *s) 2835 { 2836 unsigned long *mhp; 2837 static unsigned long *last_mhp; 2838 2839 if (!parsed_valid_hugepagesz) { 2840 pr_warn("hugepages = %s preceded by " 2841 "an unsupported hugepagesz, ignoring\n", s); 2842 parsed_valid_hugepagesz = true; 2843 return 1; 2844 } 2845 /* 2846 * !hugetlb_max_hstate means we haven't parsed a hugepagesz= parameter yet, 2847 * so this hugepages= parameter goes to the "default hstate". 2848 */ 2849 else if (!hugetlb_max_hstate) 2850 mhp = &default_hstate_max_huge_pages; 2851 else 2852 mhp = &parsed_hstate->max_huge_pages; 2853 2854 if (mhp == last_mhp) { 2855 pr_warn("hugepages= specified twice without interleaving hugepagesz=, ignoring\n"); 2856 return 1; 2857 } 2858 2859 if (sscanf(s, "%lu", mhp) <= 0) 2860 *mhp = 0; 2861 2862 /* 2863 * Global state is always initialized later in hugetlb_init. 2864 * But we need to allocate >= MAX_ORDER hstates here early to still 2865 * use the bootmem allocator. 2866 */ 2867 if (hugetlb_max_hstate && parsed_hstate->order >= MAX_ORDER) 2868 hugetlb_hstate_alloc_pages(parsed_hstate); 2869 2870 last_mhp = mhp; 2871 2872 return 1; 2873 } 2874 __setup("hugepages=", hugetlb_nrpages_setup); 2875 2876 static int __init hugetlb_default_setup(char *s) 2877 { 2878 default_hstate_size = memparse(s, &s); 2879 return 1; 2880 } 2881 __setup("default_hugepagesz=", hugetlb_default_setup); 2882 2883 static unsigned int cpuset_mems_nr(unsigned int *array) 2884 { 2885 int node; 2886 unsigned int nr = 0; 2887 2888 for_each_node_mask(node, cpuset_current_mems_allowed) 2889 nr += array[node]; 2890 2891 return nr; 2892 } 2893 2894 #ifdef CONFIG_SYSCTL 2895 static int hugetlb_sysctl_handler_common(bool obey_mempolicy, 2896 struct ctl_table *table, int write, 2897 void __user *buffer, size_t *length, loff_t *ppos) 2898 { 2899 struct hstate *h = &default_hstate; 2900 unsigned long tmp = h->max_huge_pages; 2901 int ret; 2902 2903 if (!hugepages_supported()) 2904 return -EOPNOTSUPP; 2905 2906 table->data = &tmp; 2907 table->maxlen = sizeof(unsigned long); 2908 ret = proc_doulongvec_minmax(table, write, buffer, length, ppos); 2909 if (ret) 2910 goto out; 2911 2912 if (write) 2913 ret = __nr_hugepages_store_common(obey_mempolicy, h, 2914 NUMA_NO_NODE, tmp, *length); 2915 out: 2916 return ret; 2917 } 2918 2919 int hugetlb_sysctl_handler(struct ctl_table *table, int write, 2920 void __user *buffer, size_t *length, loff_t *ppos) 2921 { 2922 2923 return hugetlb_sysctl_handler_common(false, table, write, 2924 buffer, length, ppos); 2925 } 2926 2927 #ifdef CONFIG_NUMA 2928 int hugetlb_mempolicy_sysctl_handler(struct ctl_table *table, int write, 2929 void __user *buffer, size_t *length, loff_t *ppos) 2930 { 2931 return hugetlb_sysctl_handler_common(true, table, write, 2932 buffer, length, ppos); 2933 } 2934 #endif /* CONFIG_NUMA */ 2935 2936 int hugetlb_overcommit_handler(struct ctl_table *table, int write, 2937 void __user *buffer, 2938 size_t *length, loff_t *ppos) 2939 { 2940 struct hstate *h = &default_hstate; 2941 unsigned long tmp; 2942 int ret; 2943 2944 if (!hugepages_supported()) 2945 return -EOPNOTSUPP; 2946 2947 tmp = h->nr_overcommit_huge_pages; 2948 2949 if (write && hstate_is_gigantic(h)) 2950 return -EINVAL; 2951 2952 table->data = &tmp; 2953 table->maxlen = sizeof(unsigned long); 2954 ret = proc_doulongvec_minmax(table, write, buffer, length, ppos); 2955 if (ret) 2956 goto out; 2957 2958 if (write) { 2959 spin_lock(&hugetlb_lock); 2960 h->nr_overcommit_huge_pages = tmp; 2961 spin_unlock(&hugetlb_lock); 2962 } 2963 out: 2964 return ret; 2965 } 2966 2967 #endif /* CONFIG_SYSCTL */ 2968 2969 void hugetlb_report_meminfo(struct seq_file *m) 2970 { 2971 struct hstate *h = &default_hstate; 2972 if (!hugepages_supported()) 2973 return; 2974 seq_printf(m, 2975 "HugePages_Total: %5lu\n" 2976 "HugePages_Free: %5lu\n" 2977 "HugePages_Rsvd: %5lu\n" 2978 "HugePages_Surp: %5lu\n" 2979 "Hugepagesize: %8lu kB\n", 2980 h->nr_huge_pages, 2981 h->free_huge_pages, 2982 h->resv_huge_pages, 2983 h->surplus_huge_pages, 2984 1UL << (huge_page_order(h) + PAGE_SHIFT - 10)); 2985 } 2986 2987 int hugetlb_report_node_meminfo(int nid, char *buf) 2988 { 2989 struct hstate *h = &default_hstate; 2990 if (!hugepages_supported()) 2991 return 0; 2992 return sprintf(buf, 2993 "Node %d HugePages_Total: %5u\n" 2994 "Node %d HugePages_Free: %5u\n" 2995 "Node %d HugePages_Surp: %5u\n", 2996 nid, h->nr_huge_pages_node[nid], 2997 nid, h->free_huge_pages_node[nid], 2998 nid, h->surplus_huge_pages_node[nid]); 2999 } 3000 3001 void hugetlb_show_meminfo(void) 3002 { 3003 struct hstate *h; 3004 int nid; 3005 3006 if (!hugepages_supported()) 3007 return; 3008 3009 for_each_node_state(nid, N_MEMORY) 3010 for_each_hstate(h) 3011 pr_info("Node %d hugepages_total=%u hugepages_free=%u hugepages_surp=%u hugepages_size=%lukB\n", 3012 nid, 3013 h->nr_huge_pages_node[nid], 3014 h->free_huge_pages_node[nid], 3015 h->surplus_huge_pages_node[nid], 3016 1UL << (huge_page_order(h) + PAGE_SHIFT - 10)); 3017 } 3018 3019 void hugetlb_report_usage(struct seq_file *m, struct mm_struct *mm) 3020 { 3021 seq_printf(m, "HugetlbPages:\t%8lu kB\n", 3022 atomic_long_read(&mm->hugetlb_usage) << (PAGE_SHIFT - 10)); 3023 } 3024 3025 /* Return the number pages of memory we physically have, in PAGE_SIZE units. */ 3026 unsigned long hugetlb_total_pages(void) 3027 { 3028 struct hstate *h; 3029 unsigned long nr_total_pages = 0; 3030 3031 for_each_hstate(h) 3032 nr_total_pages += h->nr_huge_pages * pages_per_huge_page(h); 3033 return nr_total_pages; 3034 } 3035 3036 static int hugetlb_acct_memory(struct hstate *h, long delta) 3037 { 3038 int ret = -ENOMEM; 3039 3040 spin_lock(&hugetlb_lock); 3041 /* 3042 * When cpuset is configured, it breaks the strict hugetlb page 3043 * reservation as the accounting is done on a global variable. Such 3044 * reservation is completely rubbish in the presence of cpuset because 3045 * the reservation is not checked against page availability for the 3046 * current cpuset. Application can still potentially OOM'ed by kernel 3047 * with lack of free htlb page in cpuset that the task is in. 3048 * Attempt to enforce strict accounting with cpuset is almost 3049 * impossible (or too ugly) because cpuset is too fluid that 3050 * task or memory node can be dynamically moved between cpusets. 3051 * 3052 * The change of semantics for shared hugetlb mapping with cpuset is 3053 * undesirable. However, in order to preserve some of the semantics, 3054 * we fall back to check against current free page availability as 3055 * a best attempt and hopefully to minimize the impact of changing 3056 * semantics that cpuset has. 3057 */ 3058 if (delta > 0) { 3059 if (gather_surplus_pages(h, delta) < 0) 3060 goto out; 3061 3062 if (delta > cpuset_mems_nr(h->free_huge_pages_node)) { 3063 return_unused_surplus_pages(h, delta); 3064 goto out; 3065 } 3066 } 3067 3068 ret = 0; 3069 if (delta < 0) 3070 return_unused_surplus_pages(h, (unsigned long) -delta); 3071 3072 out: 3073 spin_unlock(&hugetlb_lock); 3074 return ret; 3075 } 3076 3077 static void hugetlb_vm_op_open(struct vm_area_struct *vma) 3078 { 3079 struct resv_map *resv = vma_resv_map(vma); 3080 3081 /* 3082 * This new VMA should share its siblings reservation map if present. 3083 * The VMA will only ever have a valid reservation map pointer where 3084 * it is being copied for another still existing VMA. As that VMA 3085 * has a reference to the reservation map it cannot disappear until 3086 * after this open call completes. It is therefore safe to take a 3087 * new reference here without additional locking. 3088 */ 3089 if (resv && is_vma_resv_set(vma, HPAGE_RESV_OWNER)) 3090 kref_get(&resv->refs); 3091 } 3092 3093 static void hugetlb_vm_op_close(struct vm_area_struct *vma) 3094 { 3095 struct hstate *h = hstate_vma(vma); 3096 struct resv_map *resv = vma_resv_map(vma); 3097 struct hugepage_subpool *spool = subpool_vma(vma); 3098 unsigned long reserve, start, end; 3099 long gbl_reserve; 3100 3101 if (!resv || !is_vma_resv_set(vma, HPAGE_RESV_OWNER)) 3102 return; 3103 3104 start = vma_hugecache_offset(h, vma, vma->vm_start); 3105 end = vma_hugecache_offset(h, vma, vma->vm_end); 3106 3107 reserve = (end - start) - region_count(resv, start, end); 3108 3109 kref_put(&resv->refs, resv_map_release); 3110 3111 if (reserve) { 3112 /* 3113 * Decrement reserve counts. The global reserve count may be 3114 * adjusted if the subpool has a minimum size. 3115 */ 3116 gbl_reserve = hugepage_subpool_put_pages(spool, reserve); 3117 hugetlb_acct_memory(h, -gbl_reserve); 3118 } 3119 } 3120 3121 /* 3122 * We cannot handle pagefaults against hugetlb pages at all. They cause 3123 * handle_mm_fault() to try to instantiate regular-sized pages in the 3124 * hugegpage VMA. do_page_fault() is supposed to trap this, so BUG is we get 3125 * this far. 3126 */ 3127 static int hugetlb_vm_op_fault(struct vm_fault *vmf) 3128 { 3129 BUG(); 3130 return 0; 3131 } 3132 3133 const struct vm_operations_struct hugetlb_vm_ops = { 3134 .fault = hugetlb_vm_op_fault, 3135 .open = hugetlb_vm_op_open, 3136 .close = hugetlb_vm_op_close, 3137 }; 3138 3139 static pte_t make_huge_pte(struct vm_area_struct *vma, struct page *page, 3140 int writable) 3141 { 3142 pte_t entry; 3143 3144 if (writable) { 3145 entry = huge_pte_mkwrite(huge_pte_mkdirty(mk_huge_pte(page, 3146 vma->vm_page_prot))); 3147 } else { 3148 entry = huge_pte_wrprotect(mk_huge_pte(page, 3149 vma->vm_page_prot)); 3150 } 3151 entry = pte_mkyoung(entry); 3152 entry = pte_mkhuge(entry); 3153 entry = arch_make_huge_pte(entry, vma, page, writable); 3154 3155 return entry; 3156 } 3157 3158 static void set_huge_ptep_writable(struct vm_area_struct *vma, 3159 unsigned long address, pte_t *ptep) 3160 { 3161 pte_t entry; 3162 3163 entry = huge_pte_mkwrite(huge_pte_mkdirty(huge_ptep_get(ptep))); 3164 if (huge_ptep_set_access_flags(vma, address, ptep, entry, 1)) 3165 update_mmu_cache(vma, address, ptep); 3166 } 3167 3168 bool is_hugetlb_entry_migration(pte_t pte) 3169 { 3170 swp_entry_t swp; 3171 3172 if (huge_pte_none(pte) || pte_present(pte)) 3173 return false; 3174 swp = pte_to_swp_entry(pte); 3175 if (non_swap_entry(swp) && is_migration_entry(swp)) 3176 return true; 3177 else 3178 return false; 3179 } 3180 3181 static int is_hugetlb_entry_hwpoisoned(pte_t pte) 3182 { 3183 swp_entry_t swp; 3184 3185 if (huge_pte_none(pte) || pte_present(pte)) 3186 return 0; 3187 swp = pte_to_swp_entry(pte); 3188 if (non_swap_entry(swp) && is_hwpoison_entry(swp)) 3189 return 1; 3190 else 3191 return 0; 3192 } 3193 3194 int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src, 3195 struct vm_area_struct *vma) 3196 { 3197 pte_t *src_pte, *dst_pte, entry; 3198 struct page *ptepage; 3199 unsigned long addr; 3200 int cow; 3201 struct hstate *h = hstate_vma(vma); 3202 unsigned long sz = huge_page_size(h); 3203 unsigned long mmun_start; /* For mmu_notifiers */ 3204 unsigned long mmun_end; /* For mmu_notifiers */ 3205 int ret = 0; 3206 3207 cow = (vma->vm_flags & (VM_SHARED | VM_MAYWRITE)) == VM_MAYWRITE; 3208 3209 mmun_start = vma->vm_start; 3210 mmun_end = vma->vm_end; 3211 if (cow) 3212 mmu_notifier_invalidate_range_start(src, mmun_start, mmun_end); 3213 3214 for (addr = vma->vm_start; addr < vma->vm_end; addr += sz) { 3215 spinlock_t *src_ptl, *dst_ptl; 3216 src_pte = huge_pte_offset(src, addr, sz); 3217 if (!src_pte) 3218 continue; 3219 dst_pte = huge_pte_alloc(dst, addr, sz); 3220 if (!dst_pte) { 3221 ret = -ENOMEM; 3222 break; 3223 } 3224 3225 /* If the pagetables are shared don't copy or take references */ 3226 if (dst_pte == src_pte) 3227 continue; 3228 3229 dst_ptl = huge_pte_lock(h, dst, dst_pte); 3230 src_ptl = huge_pte_lockptr(h, src, src_pte); 3231 spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING); 3232 entry = huge_ptep_get(src_pte); 3233 if (huge_pte_none(entry)) { /* skip none entry */ 3234 ; 3235 } else if (unlikely(is_hugetlb_entry_migration(entry) || 3236 is_hugetlb_entry_hwpoisoned(entry))) { 3237 swp_entry_t swp_entry = pte_to_swp_entry(entry); 3238 3239 if (is_write_migration_entry(swp_entry) && cow) { 3240 /* 3241 * COW mappings require pages in both 3242 * parent and child to be set to read. 3243 */ 3244 make_migration_entry_read(&swp_entry); 3245 entry = swp_entry_to_pte(swp_entry); 3246 set_huge_swap_pte_at(src, addr, src_pte, 3247 entry, sz); 3248 } 3249 set_huge_swap_pte_at(dst, addr, dst_pte, entry, sz); 3250 } else { 3251 if (cow) { 3252 huge_ptep_set_wrprotect(src, addr, src_pte); 3253 mmu_notifier_invalidate_range(src, mmun_start, 3254 mmun_end); 3255 } 3256 entry = huge_ptep_get(src_pte); 3257 ptepage = pte_page(entry); 3258 get_page(ptepage); 3259 page_dup_rmap(ptepage, true); 3260 set_huge_pte_at(dst, addr, dst_pte, entry); 3261 hugetlb_count_add(pages_per_huge_page(h), dst); 3262 } 3263 spin_unlock(src_ptl); 3264 spin_unlock(dst_ptl); 3265 } 3266 3267 if (cow) 3268 mmu_notifier_invalidate_range_end(src, mmun_start, mmun_end); 3269 3270 return ret; 3271 } 3272 3273 void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct *vma, 3274 unsigned long start, unsigned long end, 3275 struct page *ref_page) 3276 { 3277 struct mm_struct *mm = vma->vm_mm; 3278 unsigned long address; 3279 pte_t *ptep; 3280 pte_t pte; 3281 spinlock_t *ptl; 3282 struct page *page; 3283 struct hstate *h = hstate_vma(vma); 3284 unsigned long sz = huge_page_size(h); 3285 const unsigned long mmun_start = start; /* For mmu_notifiers */ 3286 const unsigned long mmun_end = end; /* For mmu_notifiers */ 3287 3288 WARN_ON(!is_vm_hugetlb_page(vma)); 3289 BUG_ON(start & ~huge_page_mask(h)); 3290 BUG_ON(end & ~huge_page_mask(h)); 3291 3292 /* 3293 * This is a hugetlb vma, all the pte entries should point 3294 * to huge page. 3295 */ 3296 tlb_remove_check_page_size_change(tlb, sz); 3297 tlb_start_vma(tlb, vma); 3298 mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end); 3299 address = start; 3300 for (; address < end; address += sz) { 3301 ptep = huge_pte_offset(mm, address, sz); 3302 if (!ptep) 3303 continue; 3304 3305 ptl = huge_pte_lock(h, mm, ptep); 3306 if (huge_pmd_unshare(mm, &address, ptep)) { 3307 spin_unlock(ptl); 3308 continue; 3309 } 3310 3311 pte = huge_ptep_get(ptep); 3312 if (huge_pte_none(pte)) { 3313 spin_unlock(ptl); 3314 continue; 3315 } 3316 3317 /* 3318 * Migrating hugepage or HWPoisoned hugepage is already 3319 * unmapped and its refcount is dropped, so just clear pte here. 3320 */ 3321 if (unlikely(!pte_present(pte))) { 3322 huge_pte_clear(mm, address, ptep, sz); 3323 spin_unlock(ptl); 3324 continue; 3325 } 3326 3327 page = pte_page(pte); 3328 /* 3329 * If a reference page is supplied, it is because a specific 3330 * page is being unmapped, not a range. Ensure the page we 3331 * are about to unmap is the actual page of interest. 3332 */ 3333 if (ref_page) { 3334 if (page != ref_page) { 3335 spin_unlock(ptl); 3336 continue; 3337 } 3338 /* 3339 * Mark the VMA as having unmapped its page so that 3340 * future faults in this VMA will fail rather than 3341 * looking like data was lost 3342 */ 3343 set_vma_resv_flags(vma, HPAGE_RESV_UNMAPPED); 3344 } 3345 3346 pte = huge_ptep_get_and_clear(mm, address, ptep); 3347 tlb_remove_huge_tlb_entry(h, tlb, ptep, address); 3348 if (huge_pte_dirty(pte)) 3349 set_page_dirty(page); 3350 3351 hugetlb_count_sub(pages_per_huge_page(h), mm); 3352 page_remove_rmap(page, true); 3353 3354 spin_unlock(ptl); 3355 tlb_remove_page_size(tlb, page, huge_page_size(h)); 3356 /* 3357 * Bail out after unmapping reference page if supplied 3358 */ 3359 if (ref_page) 3360 break; 3361 } 3362 mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end); 3363 tlb_end_vma(tlb, vma); 3364 } 3365 3366 void __unmap_hugepage_range_final(struct mmu_gather *tlb, 3367 struct vm_area_struct *vma, unsigned long start, 3368 unsigned long end, struct page *ref_page) 3369 { 3370 __unmap_hugepage_range(tlb, vma, start, end, ref_page); 3371 3372 /* 3373 * Clear this flag so that x86's huge_pmd_share page_table_shareable 3374 * test will fail on a vma being torn down, and not grab a page table 3375 * on its way out. We're lucky that the flag has such an appropriate 3376 * name, and can in fact be safely cleared here. We could clear it 3377 * before the __unmap_hugepage_range above, but all that's necessary 3378 * is to clear it before releasing the i_mmap_rwsem. This works 3379 * because in the context this is called, the VMA is about to be 3380 * destroyed and the i_mmap_rwsem is held. 3381 */ 3382 vma->vm_flags &= ~VM_MAYSHARE; 3383 } 3384 3385 void unmap_hugepage_range(struct vm_area_struct *vma, unsigned long start, 3386 unsigned long end, struct page *ref_page) 3387 { 3388 struct mm_struct *mm; 3389 struct mmu_gather tlb; 3390 3391 mm = vma->vm_mm; 3392 3393 tlb_gather_mmu(&tlb, mm, start, end); 3394 __unmap_hugepage_range(&tlb, vma, start, end, ref_page); 3395 tlb_finish_mmu(&tlb, start, end); 3396 } 3397 3398 /* 3399 * This is called when the original mapper is failing to COW a MAP_PRIVATE 3400 * mappping it owns the reserve page for. The intention is to unmap the page 3401 * from other VMAs and let the children be SIGKILLed if they are faulting the 3402 * same region. 3403 */ 3404 static void unmap_ref_private(struct mm_struct *mm, struct vm_area_struct *vma, 3405 struct page *page, unsigned long address) 3406 { 3407 struct hstate *h = hstate_vma(vma); 3408 struct vm_area_struct *iter_vma; 3409 struct address_space *mapping; 3410 pgoff_t pgoff; 3411 3412 /* 3413 * vm_pgoff is in PAGE_SIZE units, hence the different calculation 3414 * from page cache lookup which is in HPAGE_SIZE units. 3415 */ 3416 address = address & huge_page_mask(h); 3417 pgoff = ((address - vma->vm_start) >> PAGE_SHIFT) + 3418 vma->vm_pgoff; 3419 mapping = vma->vm_file->f_mapping; 3420 3421 /* 3422 * Take the mapping lock for the duration of the table walk. As 3423 * this mapping should be shared between all the VMAs, 3424 * __unmap_hugepage_range() is called as the lock is already held 3425 */ 3426 i_mmap_lock_write(mapping); 3427 vma_interval_tree_foreach(iter_vma, &mapping->i_mmap, pgoff, pgoff) { 3428 /* Do not unmap the current VMA */ 3429 if (iter_vma == vma) 3430 continue; 3431 3432 /* 3433 * Shared VMAs have their own reserves and do not affect 3434 * MAP_PRIVATE accounting but it is possible that a shared 3435 * VMA is using the same page so check and skip such VMAs. 3436 */ 3437 if (iter_vma->vm_flags & VM_MAYSHARE) 3438 continue; 3439 3440 /* 3441 * Unmap the page from other VMAs without their own reserves. 3442 * They get marked to be SIGKILLed if they fault in these 3443 * areas. This is because a future no-page fault on this VMA 3444 * could insert a zeroed page instead of the data existing 3445 * from the time of fork. This would look like data corruption 3446 */ 3447 if (!is_vma_resv_set(iter_vma, HPAGE_RESV_OWNER)) 3448 unmap_hugepage_range(iter_vma, address, 3449 address + huge_page_size(h), page); 3450 } 3451 i_mmap_unlock_write(mapping); 3452 } 3453 3454 /* 3455 * Hugetlb_cow() should be called with page lock of the original hugepage held. 3456 * Called with hugetlb_instantiation_mutex held and pte_page locked so we 3457 * cannot race with other handlers or page migration. 3458 * Keep the pte_same checks anyway to make transition from the mutex easier. 3459 */ 3460 static int hugetlb_cow(struct mm_struct *mm, struct vm_area_struct *vma, 3461 unsigned long address, pte_t *ptep, 3462 struct page *pagecache_page, spinlock_t *ptl) 3463 { 3464 pte_t pte; 3465 struct hstate *h = hstate_vma(vma); 3466 struct page *old_page, *new_page; 3467 int ret = 0, outside_reserve = 0; 3468 unsigned long mmun_start; /* For mmu_notifiers */ 3469 unsigned long mmun_end; /* For mmu_notifiers */ 3470 3471 pte = huge_ptep_get(ptep); 3472 old_page = pte_page(pte); 3473 3474 retry_avoidcopy: 3475 /* If no-one else is actually using this page, avoid the copy 3476 * and just make the page writable */ 3477 if (page_mapcount(old_page) == 1 && PageAnon(old_page)) { 3478 page_move_anon_rmap(old_page, vma); 3479 set_huge_ptep_writable(vma, address, ptep); 3480 return 0; 3481 } 3482 3483 /* 3484 * If the process that created a MAP_PRIVATE mapping is about to 3485 * perform a COW due to a shared page count, attempt to satisfy 3486 * the allocation without using the existing reserves. The pagecache 3487 * page is used to determine if the reserve at this address was 3488 * consumed or not. If reserves were used, a partial faulted mapping 3489 * at the time of fork() could consume its reserves on COW instead 3490 * of the full address range. 3491 */ 3492 if (is_vma_resv_set(vma, HPAGE_RESV_OWNER) && 3493 old_page != pagecache_page) 3494 outside_reserve = 1; 3495 3496 get_page(old_page); 3497 3498 /* 3499 * Drop page table lock as buddy allocator may be called. It will 3500 * be acquired again before returning to the caller, as expected. 3501 */ 3502 spin_unlock(ptl); 3503 new_page = alloc_huge_page(vma, address, outside_reserve); 3504 3505 if (IS_ERR(new_page)) { 3506 /* 3507 * If a process owning a MAP_PRIVATE mapping fails to COW, 3508 * it is due to references held by a child and an insufficient 3509 * huge page pool. To guarantee the original mappers 3510 * reliability, unmap the page from child processes. The child 3511 * may get SIGKILLed if it later faults. 3512 */ 3513 if (outside_reserve) { 3514 put_page(old_page); 3515 BUG_ON(huge_pte_none(pte)); 3516 unmap_ref_private(mm, vma, old_page, address); 3517 BUG_ON(huge_pte_none(pte)); 3518 spin_lock(ptl); 3519 ptep = huge_pte_offset(mm, address & huge_page_mask(h), 3520 huge_page_size(h)); 3521 if (likely(ptep && 3522 pte_same(huge_ptep_get(ptep), pte))) 3523 goto retry_avoidcopy; 3524 /* 3525 * race occurs while re-acquiring page table 3526 * lock, and our job is done. 3527 */ 3528 return 0; 3529 } 3530 3531 ret = (PTR_ERR(new_page) == -ENOMEM) ? 3532 VM_FAULT_OOM : VM_FAULT_SIGBUS; 3533 goto out_release_old; 3534 } 3535 3536 /* 3537 * When the original hugepage is shared one, it does not have 3538 * anon_vma prepared. 3539 */ 3540 if (unlikely(anon_vma_prepare(vma))) { 3541 ret = VM_FAULT_OOM; 3542 goto out_release_all; 3543 } 3544 3545 copy_user_huge_page(new_page, old_page, address, vma, 3546 pages_per_huge_page(h)); 3547 __SetPageUptodate(new_page); 3548 set_page_huge_active(new_page); 3549 3550 mmun_start = address & huge_page_mask(h); 3551 mmun_end = mmun_start + huge_page_size(h); 3552 mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end); 3553 3554 /* 3555 * Retake the page table lock to check for racing updates 3556 * before the page tables are altered 3557 */ 3558 spin_lock(ptl); 3559 ptep = huge_pte_offset(mm, address & huge_page_mask(h), 3560 huge_page_size(h)); 3561 if (likely(ptep && pte_same(huge_ptep_get(ptep), pte))) { 3562 ClearPagePrivate(new_page); 3563 3564 /* Break COW */ 3565 huge_ptep_clear_flush(vma, address, ptep); 3566 mmu_notifier_invalidate_range(mm, mmun_start, mmun_end); 3567 set_huge_pte_at(mm, address, ptep, 3568 make_huge_pte(vma, new_page, 1)); 3569 page_remove_rmap(old_page, true); 3570 hugepage_add_new_anon_rmap(new_page, vma, address); 3571 /* Make the old page be freed below */ 3572 new_page = old_page; 3573 } 3574 spin_unlock(ptl); 3575 mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end); 3576 out_release_all: 3577 restore_reserve_on_error(h, vma, address, new_page); 3578 put_page(new_page); 3579 out_release_old: 3580 put_page(old_page); 3581 3582 spin_lock(ptl); /* Caller expects lock to be held */ 3583 return ret; 3584 } 3585 3586 /* Return the pagecache page at a given address within a VMA */ 3587 static struct page *hugetlbfs_pagecache_page(struct hstate *h, 3588 struct vm_area_struct *vma, unsigned long address) 3589 { 3590 struct address_space *mapping; 3591 pgoff_t idx; 3592 3593 mapping = vma->vm_file->f_mapping; 3594 idx = vma_hugecache_offset(h, vma, address); 3595 3596 return find_lock_page(mapping, idx); 3597 } 3598 3599 /* 3600 * Return whether there is a pagecache page to back given address within VMA. 3601 * Caller follow_hugetlb_page() holds page_table_lock so we cannot lock_page. 3602 */ 3603 static bool hugetlbfs_pagecache_present(struct hstate *h, 3604 struct vm_area_struct *vma, unsigned long address) 3605 { 3606 struct address_space *mapping; 3607 pgoff_t idx; 3608 struct page *page; 3609 3610 mapping = vma->vm_file->f_mapping; 3611 idx = vma_hugecache_offset(h, vma, address); 3612 3613 page = find_get_page(mapping, idx); 3614 if (page) 3615 put_page(page); 3616 return page != NULL; 3617 } 3618 3619 int huge_add_to_page_cache(struct page *page, struct address_space *mapping, 3620 pgoff_t idx) 3621 { 3622 struct inode *inode = mapping->host; 3623 struct hstate *h = hstate_inode(inode); 3624 int err = add_to_page_cache(page, mapping, idx, GFP_KERNEL); 3625 3626 if (err) 3627 return err; 3628 ClearPagePrivate(page); 3629 3630 spin_lock(&inode->i_lock); 3631 inode->i_blocks += blocks_per_huge_page(h); 3632 spin_unlock(&inode->i_lock); 3633 return 0; 3634 } 3635 3636 static int hugetlb_no_page(struct mm_struct *mm, struct vm_area_struct *vma, 3637 struct address_space *mapping, pgoff_t idx, 3638 unsigned long address, pte_t *ptep, unsigned int flags) 3639 { 3640 struct hstate *h = hstate_vma(vma); 3641 int ret = VM_FAULT_SIGBUS; 3642 int anon_rmap = 0; 3643 unsigned long size; 3644 struct page *page; 3645 pte_t new_pte; 3646 spinlock_t *ptl; 3647 3648 /* 3649 * Currently, we are forced to kill the process in the event the 3650 * original mapper has unmapped pages from the child due to a failed 3651 * COW. Warn that such a situation has occurred as it may not be obvious 3652 */ 3653 if (is_vma_resv_set(vma, HPAGE_RESV_UNMAPPED)) { 3654 pr_warn_ratelimited("PID %d killed due to inadequate hugepage pool\n", 3655 current->pid); 3656 return ret; 3657 } 3658 3659 /* 3660 * Use page lock to guard against racing truncation 3661 * before we get page_table_lock. 3662 */ 3663 retry: 3664 page = find_lock_page(mapping, idx); 3665 if (!page) { 3666 size = i_size_read(mapping->host) >> huge_page_shift(h); 3667 if (idx >= size) 3668 goto out; 3669 3670 /* 3671 * Check for page in userfault range 3672 */ 3673 if (userfaultfd_missing(vma)) { 3674 u32 hash; 3675 struct vm_fault vmf = { 3676 .vma = vma, 3677 .address = address, 3678 .flags = flags, 3679 /* 3680 * Hard to debug if it ends up being 3681 * used by a callee that assumes 3682 * something about the other 3683 * uninitialized fields... same as in 3684 * memory.c 3685 */ 3686 }; 3687 3688 /* 3689 * hugetlb_fault_mutex must be dropped before 3690 * handling userfault. Reacquire after handling 3691 * fault to make calling code simpler. 3692 */ 3693 hash = hugetlb_fault_mutex_hash(h, mm, vma, mapping, 3694 idx, address); 3695 mutex_unlock(&hugetlb_fault_mutex_table[hash]); 3696 ret = handle_userfault(&vmf, VM_UFFD_MISSING); 3697 mutex_lock(&hugetlb_fault_mutex_table[hash]); 3698 goto out; 3699 } 3700 3701 page = alloc_huge_page(vma, address, 0); 3702 if (IS_ERR(page)) { 3703 ret = PTR_ERR(page); 3704 if (ret == -ENOMEM) 3705 ret = VM_FAULT_OOM; 3706 else 3707 ret = VM_FAULT_SIGBUS; 3708 goto out; 3709 } 3710 clear_huge_page(page, address, pages_per_huge_page(h)); 3711 __SetPageUptodate(page); 3712 set_page_huge_active(page); 3713 3714 if (vma->vm_flags & VM_MAYSHARE) { 3715 int err = huge_add_to_page_cache(page, mapping, idx); 3716 if (err) { 3717 put_page(page); 3718 if (err == -EEXIST) 3719 goto retry; 3720 goto out; 3721 } 3722 } else { 3723 lock_page(page); 3724 if (unlikely(anon_vma_prepare(vma))) { 3725 ret = VM_FAULT_OOM; 3726 goto backout_unlocked; 3727 } 3728 anon_rmap = 1; 3729 } 3730 } else { 3731 /* 3732 * If memory error occurs between mmap() and fault, some process 3733 * don't have hwpoisoned swap entry for errored virtual address. 3734 * So we need to block hugepage fault by PG_hwpoison bit check. 3735 */ 3736 if (unlikely(PageHWPoison(page))) { 3737 ret = VM_FAULT_HWPOISON | 3738 VM_FAULT_SET_HINDEX(hstate_index(h)); 3739 goto backout_unlocked; 3740 } 3741 } 3742 3743 /* 3744 * If we are going to COW a private mapping later, we examine the 3745 * pending reservations for this page now. This will ensure that 3746 * any allocations necessary to record that reservation occur outside 3747 * the spinlock. 3748 */ 3749 if ((flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED)) { 3750 if (vma_needs_reservation(h, vma, address) < 0) { 3751 ret = VM_FAULT_OOM; 3752 goto backout_unlocked; 3753 } 3754 /* Just decrements count, does not deallocate */ 3755 vma_end_reservation(h, vma, address); 3756 } 3757 3758 ptl = huge_pte_lock(h, mm, ptep); 3759 size = i_size_read(mapping->host) >> huge_page_shift(h); 3760 if (idx >= size) 3761 goto backout; 3762 3763 ret = 0; 3764 if (!huge_pte_none(huge_ptep_get(ptep))) 3765 goto backout; 3766 3767 if (anon_rmap) { 3768 ClearPagePrivate(page); 3769 hugepage_add_new_anon_rmap(page, vma, address); 3770 } else 3771 page_dup_rmap(page, true); 3772 new_pte = make_huge_pte(vma, page, ((vma->vm_flags & VM_WRITE) 3773 && (vma->vm_flags & VM_SHARED))); 3774 set_huge_pte_at(mm, address, ptep, new_pte); 3775 3776 hugetlb_count_add(pages_per_huge_page(h), mm); 3777 if ((flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED)) { 3778 /* Optimization, do the COW without a second fault */ 3779 ret = hugetlb_cow(mm, vma, address, ptep, page, ptl); 3780 } 3781 3782 spin_unlock(ptl); 3783 unlock_page(page); 3784 out: 3785 return ret; 3786 3787 backout: 3788 spin_unlock(ptl); 3789 backout_unlocked: 3790 unlock_page(page); 3791 restore_reserve_on_error(h, vma, address, page); 3792 put_page(page); 3793 goto out; 3794 } 3795 3796 #ifdef CONFIG_SMP 3797 u32 hugetlb_fault_mutex_hash(struct hstate *h, struct mm_struct *mm, 3798 struct vm_area_struct *vma, 3799 struct address_space *mapping, 3800 pgoff_t idx, unsigned long address) 3801 { 3802 unsigned long key[2]; 3803 u32 hash; 3804 3805 if (vma->vm_flags & VM_SHARED) { 3806 key[0] = (unsigned long) mapping; 3807 key[1] = idx; 3808 } else { 3809 key[0] = (unsigned long) mm; 3810 key[1] = address >> huge_page_shift(h); 3811 } 3812 3813 hash = jhash2((u32 *)&key, sizeof(key)/sizeof(u32), 0); 3814 3815 return hash & (num_fault_mutexes - 1); 3816 } 3817 #else 3818 /* 3819 * For uniprocesor systems we always use a single mutex, so just 3820 * return 0 and avoid the hashing overhead. 3821 */ 3822 u32 hugetlb_fault_mutex_hash(struct hstate *h, struct mm_struct *mm, 3823 struct vm_area_struct *vma, 3824 struct address_space *mapping, 3825 pgoff_t idx, unsigned long address) 3826 { 3827 return 0; 3828 } 3829 #endif 3830 3831 int hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma, 3832 unsigned long address, unsigned int flags) 3833 { 3834 pte_t *ptep, entry; 3835 spinlock_t *ptl; 3836 int ret; 3837 u32 hash; 3838 pgoff_t idx; 3839 struct page *page = NULL; 3840 struct page *pagecache_page = NULL; 3841 struct hstate *h = hstate_vma(vma); 3842 struct address_space *mapping; 3843 int need_wait_lock = 0; 3844 3845 address &= huge_page_mask(h); 3846 3847 ptep = huge_pte_offset(mm, address, huge_page_size(h)); 3848 if (ptep) { 3849 entry = huge_ptep_get(ptep); 3850 if (unlikely(is_hugetlb_entry_migration(entry))) { 3851 migration_entry_wait_huge(vma, mm, ptep); 3852 return 0; 3853 } else if (unlikely(is_hugetlb_entry_hwpoisoned(entry))) 3854 return VM_FAULT_HWPOISON_LARGE | 3855 VM_FAULT_SET_HINDEX(hstate_index(h)); 3856 } else { 3857 ptep = huge_pte_alloc(mm, address, huge_page_size(h)); 3858 if (!ptep) 3859 return VM_FAULT_OOM; 3860 } 3861 3862 mapping = vma->vm_file->f_mapping; 3863 idx = vma_hugecache_offset(h, vma, address); 3864 3865 /* 3866 * Serialize hugepage allocation and instantiation, so that we don't 3867 * get spurious allocation failures if two CPUs race to instantiate 3868 * the same page in the page cache. 3869 */ 3870 hash = hugetlb_fault_mutex_hash(h, mm, vma, mapping, idx, address); 3871 mutex_lock(&hugetlb_fault_mutex_table[hash]); 3872 3873 entry = huge_ptep_get(ptep); 3874 if (huge_pte_none(entry)) { 3875 ret = hugetlb_no_page(mm, vma, mapping, idx, address, ptep, flags); 3876 goto out_mutex; 3877 } 3878 3879 ret = 0; 3880 3881 /* 3882 * entry could be a migration/hwpoison entry at this point, so this 3883 * check prevents the kernel from going below assuming that we have 3884 * a active hugepage in pagecache. This goto expects the 2nd page fault, 3885 * and is_hugetlb_entry_(migration|hwpoisoned) check will properly 3886 * handle it. 3887 */ 3888 if (!pte_present(entry)) 3889 goto out_mutex; 3890 3891 /* 3892 * If we are going to COW the mapping later, we examine the pending 3893 * reservations for this page now. This will ensure that any 3894 * allocations necessary to record that reservation occur outside the 3895 * spinlock. For private mappings, we also lookup the pagecache 3896 * page now as it is used to determine if a reservation has been 3897 * consumed. 3898 */ 3899 if ((flags & FAULT_FLAG_WRITE) && !huge_pte_write(entry)) { 3900 if (vma_needs_reservation(h, vma, address) < 0) { 3901 ret = VM_FAULT_OOM; 3902 goto out_mutex; 3903 } 3904 /* Just decrements count, does not deallocate */ 3905 vma_end_reservation(h, vma, address); 3906 3907 if (!(vma->vm_flags & VM_MAYSHARE)) 3908 pagecache_page = hugetlbfs_pagecache_page(h, 3909 vma, address); 3910 } 3911 3912 ptl = huge_pte_lock(h, mm, ptep); 3913 3914 /* Check for a racing update before calling hugetlb_cow */ 3915 if (unlikely(!pte_same(entry, huge_ptep_get(ptep)))) 3916 goto out_ptl; 3917 3918 /* 3919 * hugetlb_cow() requires page locks of pte_page(entry) and 3920 * pagecache_page, so here we need take the former one 3921 * when page != pagecache_page or !pagecache_page. 3922 */ 3923 page = pte_page(entry); 3924 if (page != pagecache_page) 3925 if (!trylock_page(page)) { 3926 need_wait_lock = 1; 3927 goto out_ptl; 3928 } 3929 3930 get_page(page); 3931 3932 if (flags & FAULT_FLAG_WRITE) { 3933 if (!huge_pte_write(entry)) { 3934 ret = hugetlb_cow(mm, vma, address, ptep, 3935 pagecache_page, ptl); 3936 goto out_put_page; 3937 } 3938 entry = huge_pte_mkdirty(entry); 3939 } 3940 entry = pte_mkyoung(entry); 3941 if (huge_ptep_set_access_flags(vma, address, ptep, entry, 3942 flags & FAULT_FLAG_WRITE)) 3943 update_mmu_cache(vma, address, ptep); 3944 out_put_page: 3945 if (page != pagecache_page) 3946 unlock_page(page); 3947 put_page(page); 3948 out_ptl: 3949 spin_unlock(ptl); 3950 3951 if (pagecache_page) { 3952 unlock_page(pagecache_page); 3953 put_page(pagecache_page); 3954 } 3955 out_mutex: 3956 mutex_unlock(&hugetlb_fault_mutex_table[hash]); 3957 /* 3958 * Generally it's safe to hold refcount during waiting page lock. But 3959 * here we just wait to defer the next page fault to avoid busy loop and 3960 * the page is not used after unlocked before returning from the current 3961 * page fault. So we are safe from accessing freed page, even if we wait 3962 * here without taking refcount. 3963 */ 3964 if (need_wait_lock) 3965 wait_on_page_locked(page); 3966 return ret; 3967 } 3968 3969 /* 3970 * Used by userfaultfd UFFDIO_COPY. Based on mcopy_atomic_pte with 3971 * modifications for huge pages. 3972 */ 3973 int hugetlb_mcopy_atomic_pte(struct mm_struct *dst_mm, 3974 pte_t *dst_pte, 3975 struct vm_area_struct *dst_vma, 3976 unsigned long dst_addr, 3977 unsigned long src_addr, 3978 struct page **pagep) 3979 { 3980 int vm_shared = dst_vma->vm_flags & VM_SHARED; 3981 struct hstate *h = hstate_vma(dst_vma); 3982 pte_t _dst_pte; 3983 spinlock_t *ptl; 3984 int ret; 3985 struct page *page; 3986 3987 if (!*pagep) { 3988 ret = -ENOMEM; 3989 page = alloc_huge_page(dst_vma, dst_addr, 0); 3990 if (IS_ERR(page)) 3991 goto out; 3992 3993 ret = copy_huge_page_from_user(page, 3994 (const void __user *) src_addr, 3995 pages_per_huge_page(h), false); 3996 3997 /* fallback to copy_from_user outside mmap_sem */ 3998 if (unlikely(ret)) { 3999 ret = -EFAULT; 4000 *pagep = page; 4001 /* don't free the page */ 4002 goto out; 4003 } 4004 } else { 4005 page = *pagep; 4006 *pagep = NULL; 4007 } 4008 4009 /* 4010 * The memory barrier inside __SetPageUptodate makes sure that 4011 * preceding stores to the page contents become visible before 4012 * the set_pte_at() write. 4013 */ 4014 __SetPageUptodate(page); 4015 set_page_huge_active(page); 4016 4017 /* 4018 * If shared, add to page cache 4019 */ 4020 if (vm_shared) { 4021 struct address_space *mapping = dst_vma->vm_file->f_mapping; 4022 pgoff_t idx = vma_hugecache_offset(h, dst_vma, dst_addr); 4023 4024 ret = huge_add_to_page_cache(page, mapping, idx); 4025 if (ret) 4026 goto out_release_nounlock; 4027 } 4028 4029 ptl = huge_pte_lockptr(h, dst_mm, dst_pte); 4030 spin_lock(ptl); 4031 4032 ret = -EEXIST; 4033 if (!huge_pte_none(huge_ptep_get(dst_pte))) 4034 goto out_release_unlock; 4035 4036 if (vm_shared) { 4037 page_dup_rmap(page, true); 4038 } else { 4039 ClearPagePrivate(page); 4040 hugepage_add_new_anon_rmap(page, dst_vma, dst_addr); 4041 } 4042 4043 _dst_pte = make_huge_pte(dst_vma, page, dst_vma->vm_flags & VM_WRITE); 4044 if (dst_vma->vm_flags & VM_WRITE) 4045 _dst_pte = huge_pte_mkdirty(_dst_pte); 4046 _dst_pte = pte_mkyoung(_dst_pte); 4047 4048 set_huge_pte_at(dst_mm, dst_addr, dst_pte, _dst_pte); 4049 4050 (void)huge_ptep_set_access_flags(dst_vma, dst_addr, dst_pte, _dst_pte, 4051 dst_vma->vm_flags & VM_WRITE); 4052 hugetlb_count_add(pages_per_huge_page(h), dst_mm); 4053 4054 /* No need to invalidate - it was non-present before */ 4055 update_mmu_cache(dst_vma, dst_addr, dst_pte); 4056 4057 spin_unlock(ptl); 4058 if (vm_shared) 4059 unlock_page(page); 4060 ret = 0; 4061 out: 4062 return ret; 4063 out_release_unlock: 4064 spin_unlock(ptl); 4065 out_release_nounlock: 4066 if (vm_shared) 4067 unlock_page(page); 4068 put_page(page); 4069 goto out; 4070 } 4071 4072 long follow_hugetlb_page(struct mm_struct *mm, struct vm_area_struct *vma, 4073 struct page **pages, struct vm_area_struct **vmas, 4074 unsigned long *position, unsigned long *nr_pages, 4075 long i, unsigned int flags, int *nonblocking) 4076 { 4077 unsigned long pfn_offset; 4078 unsigned long vaddr = *position; 4079 unsigned long remainder = *nr_pages; 4080 struct hstate *h = hstate_vma(vma); 4081 4082 while (vaddr < vma->vm_end && remainder) { 4083 pte_t *pte; 4084 spinlock_t *ptl = NULL; 4085 int absent; 4086 struct page *page; 4087 4088 /* 4089 * If we have a pending SIGKILL, don't keep faulting pages and 4090 * potentially allocating memory. 4091 */ 4092 if (unlikely(fatal_signal_pending(current))) { 4093 remainder = 0; 4094 break; 4095 } 4096 4097 /* 4098 * Some archs (sparc64, sh*) have multiple pte_ts to 4099 * each hugepage. We have to make sure we get the 4100 * first, for the page indexing below to work. 4101 * 4102 * Note that page table lock is not held when pte is null. 4103 */ 4104 pte = huge_pte_offset(mm, vaddr & huge_page_mask(h), 4105 huge_page_size(h)); 4106 if (pte) 4107 ptl = huge_pte_lock(h, mm, pte); 4108 absent = !pte || huge_pte_none(huge_ptep_get(pte)); 4109 4110 /* 4111 * When coredumping, it suits get_dump_page if we just return 4112 * an error where there's an empty slot with no huge pagecache 4113 * to back it. This way, we avoid allocating a hugepage, and 4114 * the sparse dumpfile avoids allocating disk blocks, but its 4115 * huge holes still show up with zeroes where they need to be. 4116 */ 4117 if (absent && (flags & FOLL_DUMP) && 4118 !hugetlbfs_pagecache_present(h, vma, vaddr)) { 4119 if (pte) 4120 spin_unlock(ptl); 4121 remainder = 0; 4122 break; 4123 } 4124 4125 /* 4126 * We need call hugetlb_fault for both hugepages under migration 4127 * (in which case hugetlb_fault waits for the migration,) and 4128 * hwpoisoned hugepages (in which case we need to prevent the 4129 * caller from accessing to them.) In order to do this, we use 4130 * here is_swap_pte instead of is_hugetlb_entry_migration and 4131 * is_hugetlb_entry_hwpoisoned. This is because it simply covers 4132 * both cases, and because we can't follow correct pages 4133 * directly from any kind of swap entries. 4134 */ 4135 if (absent || is_swap_pte(huge_ptep_get(pte)) || 4136 ((flags & FOLL_WRITE) && 4137 !huge_pte_write(huge_ptep_get(pte)))) { 4138 int ret; 4139 unsigned int fault_flags = 0; 4140 4141 if (pte) 4142 spin_unlock(ptl); 4143 if (flags & FOLL_WRITE) 4144 fault_flags |= FAULT_FLAG_WRITE; 4145 if (nonblocking) 4146 fault_flags |= FAULT_FLAG_ALLOW_RETRY; 4147 if (flags & FOLL_NOWAIT) 4148 fault_flags |= FAULT_FLAG_ALLOW_RETRY | 4149 FAULT_FLAG_RETRY_NOWAIT; 4150 if (flags & FOLL_TRIED) { 4151 VM_WARN_ON_ONCE(fault_flags & 4152 FAULT_FLAG_ALLOW_RETRY); 4153 fault_flags |= FAULT_FLAG_TRIED; 4154 } 4155 ret = hugetlb_fault(mm, vma, vaddr, fault_flags); 4156 if (ret & VM_FAULT_ERROR) { 4157 int err = vm_fault_to_errno(ret, flags); 4158 4159 if (err) 4160 return err; 4161 4162 remainder = 0; 4163 break; 4164 } 4165 if (ret & VM_FAULT_RETRY) { 4166 if (nonblocking) 4167 *nonblocking = 0; 4168 *nr_pages = 0; 4169 /* 4170 * VM_FAULT_RETRY must not return an 4171 * error, it will return zero 4172 * instead. 4173 * 4174 * No need to update "position" as the 4175 * caller will not check it after 4176 * *nr_pages is set to 0. 4177 */ 4178 return i; 4179 } 4180 continue; 4181 } 4182 4183 pfn_offset = (vaddr & ~huge_page_mask(h)) >> PAGE_SHIFT; 4184 page = pte_page(huge_ptep_get(pte)); 4185 same_page: 4186 if (pages) { 4187 pages[i] = mem_map_offset(page, pfn_offset); 4188 get_page(pages[i]); 4189 } 4190 4191 if (vmas) 4192 vmas[i] = vma; 4193 4194 vaddr += PAGE_SIZE; 4195 ++pfn_offset; 4196 --remainder; 4197 ++i; 4198 if (vaddr < vma->vm_end && remainder && 4199 pfn_offset < pages_per_huge_page(h)) { 4200 /* 4201 * We use pfn_offset to avoid touching the pageframes 4202 * of this compound page. 4203 */ 4204 goto same_page; 4205 } 4206 spin_unlock(ptl); 4207 } 4208 *nr_pages = remainder; 4209 /* 4210 * setting position is actually required only if remainder is 4211 * not zero but it's faster not to add a "if (remainder)" 4212 * branch. 4213 */ 4214 *position = vaddr; 4215 4216 return i ? i : -EFAULT; 4217 } 4218 4219 #ifndef __HAVE_ARCH_FLUSH_HUGETLB_TLB_RANGE 4220 /* 4221 * ARCHes with special requirements for evicting HUGETLB backing TLB entries can 4222 * implement this. 4223 */ 4224 #define flush_hugetlb_tlb_range(vma, addr, end) flush_tlb_range(vma, addr, end) 4225 #endif 4226 4227 unsigned long hugetlb_change_protection(struct vm_area_struct *vma, 4228 unsigned long address, unsigned long end, pgprot_t newprot) 4229 { 4230 struct mm_struct *mm = vma->vm_mm; 4231 unsigned long start = address; 4232 pte_t *ptep; 4233 pte_t pte; 4234 struct hstate *h = hstate_vma(vma); 4235 unsigned long pages = 0; 4236 4237 BUG_ON(address >= end); 4238 flush_cache_range(vma, address, end); 4239 4240 mmu_notifier_invalidate_range_start(mm, start, end); 4241 i_mmap_lock_write(vma->vm_file->f_mapping); 4242 for (; address < end; address += huge_page_size(h)) { 4243 spinlock_t *ptl; 4244 ptep = huge_pte_offset(mm, address, huge_page_size(h)); 4245 if (!ptep) 4246 continue; 4247 ptl = huge_pte_lock(h, mm, ptep); 4248 if (huge_pmd_unshare(mm, &address, ptep)) { 4249 pages++; 4250 spin_unlock(ptl); 4251 continue; 4252 } 4253 pte = huge_ptep_get(ptep); 4254 if (unlikely(is_hugetlb_entry_hwpoisoned(pte))) { 4255 spin_unlock(ptl); 4256 continue; 4257 } 4258 if (unlikely(is_hugetlb_entry_migration(pte))) { 4259 swp_entry_t entry = pte_to_swp_entry(pte); 4260 4261 if (is_write_migration_entry(entry)) { 4262 pte_t newpte; 4263 4264 make_migration_entry_read(&entry); 4265 newpte = swp_entry_to_pte(entry); 4266 set_huge_swap_pte_at(mm, address, ptep, 4267 newpte, huge_page_size(h)); 4268 pages++; 4269 } 4270 spin_unlock(ptl); 4271 continue; 4272 } 4273 if (!huge_pte_none(pte)) { 4274 pte = huge_ptep_get_and_clear(mm, address, ptep); 4275 pte = pte_mkhuge(huge_pte_modify(pte, newprot)); 4276 pte = arch_make_huge_pte(pte, vma, NULL, 0); 4277 set_huge_pte_at(mm, address, ptep, pte); 4278 pages++; 4279 } 4280 spin_unlock(ptl); 4281 } 4282 /* 4283 * Must flush TLB before releasing i_mmap_rwsem: x86's huge_pmd_unshare 4284 * may have cleared our pud entry and done put_page on the page table: 4285 * once we release i_mmap_rwsem, another task can do the final put_page 4286 * and that page table be reused and filled with junk. 4287 */ 4288 flush_hugetlb_tlb_range(vma, start, end); 4289 mmu_notifier_invalidate_range(mm, start, end); 4290 i_mmap_unlock_write(vma->vm_file->f_mapping); 4291 mmu_notifier_invalidate_range_end(mm, start, end); 4292 4293 return pages << h->order; 4294 } 4295 4296 int hugetlb_reserve_pages(struct inode *inode, 4297 long from, long to, 4298 struct vm_area_struct *vma, 4299 vm_flags_t vm_flags) 4300 { 4301 long ret, chg; 4302 struct hstate *h = hstate_inode(inode); 4303 struct hugepage_subpool *spool = subpool_inode(inode); 4304 struct resv_map *resv_map; 4305 long gbl_reserve; 4306 4307 /* 4308 * Only apply hugepage reservation if asked. At fault time, an 4309 * attempt will be made for VM_NORESERVE to allocate a page 4310 * without using reserves 4311 */ 4312 if (vm_flags & VM_NORESERVE) 4313 return 0; 4314 4315 /* 4316 * Shared mappings base their reservation on the number of pages that 4317 * are already allocated on behalf of the file. Private mappings need 4318 * to reserve the full area even if read-only as mprotect() may be 4319 * called to make the mapping read-write. Assume !vma is a shm mapping 4320 */ 4321 if (!vma || vma->vm_flags & VM_MAYSHARE) { 4322 resv_map = inode_resv_map(inode); 4323 4324 chg = region_chg(resv_map, from, to); 4325 4326 } else { 4327 resv_map = resv_map_alloc(); 4328 if (!resv_map) 4329 return -ENOMEM; 4330 4331 chg = to - from; 4332 4333 set_vma_resv_map(vma, resv_map); 4334 set_vma_resv_flags(vma, HPAGE_RESV_OWNER); 4335 } 4336 4337 if (chg < 0) { 4338 ret = chg; 4339 goto out_err; 4340 } 4341 4342 /* 4343 * There must be enough pages in the subpool for the mapping. If 4344 * the subpool has a minimum size, there may be some global 4345 * reservations already in place (gbl_reserve). 4346 */ 4347 gbl_reserve = hugepage_subpool_get_pages(spool, chg); 4348 if (gbl_reserve < 0) { 4349 ret = -ENOSPC; 4350 goto out_err; 4351 } 4352 4353 /* 4354 * Check enough hugepages are available for the reservation. 4355 * Hand the pages back to the subpool if there are not 4356 */ 4357 ret = hugetlb_acct_memory(h, gbl_reserve); 4358 if (ret < 0) { 4359 /* put back original number of pages, chg */ 4360 (void)hugepage_subpool_put_pages(spool, chg); 4361 goto out_err; 4362 } 4363 4364 /* 4365 * Account for the reservations made. Shared mappings record regions 4366 * that have reservations as they are shared by multiple VMAs. 4367 * When the last VMA disappears, the region map says how much 4368 * the reservation was and the page cache tells how much of 4369 * the reservation was consumed. Private mappings are per-VMA and 4370 * only the consumed reservations are tracked. When the VMA 4371 * disappears, the original reservation is the VMA size and the 4372 * consumed reservations are stored in the map. Hence, nothing 4373 * else has to be done for private mappings here 4374 */ 4375 if (!vma || vma->vm_flags & VM_MAYSHARE) { 4376 long add = region_add(resv_map, from, to); 4377 4378 if (unlikely(chg > add)) { 4379 /* 4380 * pages in this range were added to the reserve 4381 * map between region_chg and region_add. This 4382 * indicates a race with alloc_huge_page. Adjust 4383 * the subpool and reserve counts modified above 4384 * based on the difference. 4385 */ 4386 long rsv_adjust; 4387 4388 rsv_adjust = hugepage_subpool_put_pages(spool, 4389 chg - add); 4390 hugetlb_acct_memory(h, -rsv_adjust); 4391 } 4392 } 4393 return 0; 4394 out_err: 4395 if (!vma || vma->vm_flags & VM_MAYSHARE) 4396 /* Don't call region_abort if region_chg failed */ 4397 if (chg >= 0) 4398 region_abort(resv_map, from, to); 4399 if (vma && is_vma_resv_set(vma, HPAGE_RESV_OWNER)) 4400 kref_put(&resv_map->refs, resv_map_release); 4401 return ret; 4402 } 4403 4404 long hugetlb_unreserve_pages(struct inode *inode, long start, long end, 4405 long freed) 4406 { 4407 struct hstate *h = hstate_inode(inode); 4408 struct resv_map *resv_map = inode_resv_map(inode); 4409 long chg = 0; 4410 struct hugepage_subpool *spool = subpool_inode(inode); 4411 long gbl_reserve; 4412 4413 if (resv_map) { 4414 chg = region_del(resv_map, start, end); 4415 /* 4416 * region_del() can fail in the rare case where a region 4417 * must be split and another region descriptor can not be 4418 * allocated. If end == LONG_MAX, it will not fail. 4419 */ 4420 if (chg < 0) 4421 return chg; 4422 } 4423 4424 spin_lock(&inode->i_lock); 4425 inode->i_blocks -= (blocks_per_huge_page(h) * freed); 4426 spin_unlock(&inode->i_lock); 4427 4428 /* 4429 * If the subpool has a minimum size, the number of global 4430 * reservations to be released may be adjusted. 4431 */ 4432 gbl_reserve = hugepage_subpool_put_pages(spool, (chg - freed)); 4433 hugetlb_acct_memory(h, -gbl_reserve); 4434 4435 return 0; 4436 } 4437 4438 #ifdef CONFIG_ARCH_WANT_HUGE_PMD_SHARE 4439 static unsigned long page_table_shareable(struct vm_area_struct *svma, 4440 struct vm_area_struct *vma, 4441 unsigned long addr, pgoff_t idx) 4442 { 4443 unsigned long saddr = ((idx - svma->vm_pgoff) << PAGE_SHIFT) + 4444 svma->vm_start; 4445 unsigned long sbase = saddr & PUD_MASK; 4446 unsigned long s_end = sbase + PUD_SIZE; 4447 4448 /* Allow segments to share if only one is marked locked */ 4449 unsigned long vm_flags = vma->vm_flags & VM_LOCKED_CLEAR_MASK; 4450 unsigned long svm_flags = svma->vm_flags & VM_LOCKED_CLEAR_MASK; 4451 4452 /* 4453 * match the virtual addresses, permission and the alignment of the 4454 * page table page. 4455 */ 4456 if (pmd_index(addr) != pmd_index(saddr) || 4457 vm_flags != svm_flags || 4458 sbase < svma->vm_start || svma->vm_end < s_end) 4459 return 0; 4460 4461 return saddr; 4462 } 4463 4464 static bool vma_shareable(struct vm_area_struct *vma, unsigned long addr) 4465 { 4466 unsigned long base = addr & PUD_MASK; 4467 unsigned long end = base + PUD_SIZE; 4468 4469 /* 4470 * check on proper vm_flags and page table alignment 4471 */ 4472 if (vma->vm_flags & VM_MAYSHARE && 4473 vma->vm_start <= base && end <= vma->vm_end) 4474 return true; 4475 return false; 4476 } 4477 4478 /* 4479 * Search for a shareable pmd page for hugetlb. In any case calls pmd_alloc() 4480 * and returns the corresponding pte. While this is not necessary for the 4481 * !shared pmd case because we can allocate the pmd later as well, it makes the 4482 * code much cleaner. pmd allocation is essential for the shared case because 4483 * pud has to be populated inside the same i_mmap_rwsem section - otherwise 4484 * racing tasks could either miss the sharing (see huge_pte_offset) or select a 4485 * bad pmd for sharing. 4486 */ 4487 pte_t *huge_pmd_share(struct mm_struct *mm, unsigned long addr, pud_t *pud) 4488 { 4489 struct vm_area_struct *vma = find_vma(mm, addr); 4490 struct address_space *mapping = vma->vm_file->f_mapping; 4491 pgoff_t idx = ((addr - vma->vm_start) >> PAGE_SHIFT) + 4492 vma->vm_pgoff; 4493 struct vm_area_struct *svma; 4494 unsigned long saddr; 4495 pte_t *spte = NULL; 4496 pte_t *pte; 4497 spinlock_t *ptl; 4498 4499 if (!vma_shareable(vma, addr)) 4500 return (pte_t *)pmd_alloc(mm, pud, addr); 4501 4502 i_mmap_lock_write(mapping); 4503 vma_interval_tree_foreach(svma, &mapping->i_mmap, idx, idx) { 4504 if (svma == vma) 4505 continue; 4506 4507 saddr = page_table_shareable(svma, vma, addr, idx); 4508 if (saddr) { 4509 spte = huge_pte_offset(svma->vm_mm, saddr, 4510 vma_mmu_pagesize(svma)); 4511 if (spte) { 4512 get_page(virt_to_page(spte)); 4513 break; 4514 } 4515 } 4516 } 4517 4518 if (!spte) 4519 goto out; 4520 4521 ptl = huge_pte_lock(hstate_vma(vma), mm, spte); 4522 if (pud_none(*pud)) { 4523 pud_populate(mm, pud, 4524 (pmd_t *)((unsigned long)spte & PAGE_MASK)); 4525 mm_inc_nr_pmds(mm); 4526 } else { 4527 put_page(virt_to_page(spte)); 4528 } 4529 spin_unlock(ptl); 4530 out: 4531 pte = (pte_t *)pmd_alloc(mm, pud, addr); 4532 i_mmap_unlock_write(mapping); 4533 return pte; 4534 } 4535 4536 /* 4537 * unmap huge page backed by shared pte. 4538 * 4539 * Hugetlb pte page is ref counted at the time of mapping. If pte is shared 4540 * indicated by page_count > 1, unmap is achieved by clearing pud and 4541 * decrementing the ref count. If count == 1, the pte page is not shared. 4542 * 4543 * called with page table lock held. 4544 * 4545 * returns: 1 successfully unmapped a shared pte page 4546 * 0 the underlying pte page is not shared, or it is the last user 4547 */ 4548 int huge_pmd_unshare(struct mm_struct *mm, unsigned long *addr, pte_t *ptep) 4549 { 4550 pgd_t *pgd = pgd_offset(mm, *addr); 4551 p4d_t *p4d = p4d_offset(pgd, *addr); 4552 pud_t *pud = pud_offset(p4d, *addr); 4553 4554 BUG_ON(page_count(virt_to_page(ptep)) == 0); 4555 if (page_count(virt_to_page(ptep)) == 1) 4556 return 0; 4557 4558 pud_clear(pud); 4559 put_page(virt_to_page(ptep)); 4560 mm_dec_nr_pmds(mm); 4561 *addr = ALIGN(*addr, HPAGE_SIZE * PTRS_PER_PTE) - HPAGE_SIZE; 4562 return 1; 4563 } 4564 #define want_pmd_share() (1) 4565 #else /* !CONFIG_ARCH_WANT_HUGE_PMD_SHARE */ 4566 pte_t *huge_pmd_share(struct mm_struct *mm, unsigned long addr, pud_t *pud) 4567 { 4568 return NULL; 4569 } 4570 4571 int huge_pmd_unshare(struct mm_struct *mm, unsigned long *addr, pte_t *ptep) 4572 { 4573 return 0; 4574 } 4575 #define want_pmd_share() (0) 4576 #endif /* CONFIG_ARCH_WANT_HUGE_PMD_SHARE */ 4577 4578 #ifdef CONFIG_ARCH_WANT_GENERAL_HUGETLB 4579 pte_t *huge_pte_alloc(struct mm_struct *mm, 4580 unsigned long addr, unsigned long sz) 4581 { 4582 pgd_t *pgd; 4583 p4d_t *p4d; 4584 pud_t *pud; 4585 pte_t *pte = NULL; 4586 4587 pgd = pgd_offset(mm, addr); 4588 p4d = p4d_offset(pgd, addr); 4589 pud = pud_alloc(mm, p4d, addr); 4590 if (pud) { 4591 if (sz == PUD_SIZE) { 4592 pte = (pte_t *)pud; 4593 } else { 4594 BUG_ON(sz != PMD_SIZE); 4595 if (want_pmd_share() && pud_none(*pud)) 4596 pte = huge_pmd_share(mm, addr, pud); 4597 else 4598 pte = (pte_t *)pmd_alloc(mm, pud, addr); 4599 } 4600 } 4601 BUG_ON(pte && pte_present(*pte) && !pte_huge(*pte)); 4602 4603 return pte; 4604 } 4605 4606 pte_t *huge_pte_offset(struct mm_struct *mm, 4607 unsigned long addr, unsigned long sz) 4608 { 4609 pgd_t *pgd; 4610 p4d_t *p4d; 4611 pud_t *pud; 4612 pmd_t *pmd; 4613 4614 pgd = pgd_offset(mm, addr); 4615 if (!pgd_present(*pgd)) 4616 return NULL; 4617 p4d = p4d_offset(pgd, addr); 4618 if (!p4d_present(*p4d)) 4619 return NULL; 4620 pud = pud_offset(p4d, addr); 4621 if (!pud_present(*pud)) 4622 return NULL; 4623 if (pud_huge(*pud)) 4624 return (pte_t *)pud; 4625 pmd = pmd_offset(pud, addr); 4626 return (pte_t *) pmd; 4627 } 4628 4629 #endif /* CONFIG_ARCH_WANT_GENERAL_HUGETLB */ 4630 4631 /* 4632 * These functions are overwritable if your architecture needs its own 4633 * behavior. 4634 */ 4635 struct page * __weak 4636 follow_huge_addr(struct mm_struct *mm, unsigned long address, 4637 int write) 4638 { 4639 return ERR_PTR(-EINVAL); 4640 } 4641 4642 struct page * __weak 4643 follow_huge_pd(struct vm_area_struct *vma, 4644 unsigned long address, hugepd_t hpd, int flags, int pdshift) 4645 { 4646 WARN(1, "hugepd follow called with no support for hugepage directory format\n"); 4647 return NULL; 4648 } 4649 4650 struct page * __weak 4651 follow_huge_pmd(struct mm_struct *mm, unsigned long address, 4652 pmd_t *pmd, int flags) 4653 { 4654 struct page *page = NULL; 4655 spinlock_t *ptl; 4656 pte_t pte; 4657 retry: 4658 ptl = pmd_lockptr(mm, pmd); 4659 spin_lock(ptl); 4660 /* 4661 * make sure that the address range covered by this pmd is not 4662 * unmapped from other threads. 4663 */ 4664 if (!pmd_huge(*pmd)) 4665 goto out; 4666 pte = huge_ptep_get((pte_t *)pmd); 4667 if (pte_present(pte)) { 4668 page = pmd_page(*pmd) + ((address & ~PMD_MASK) >> PAGE_SHIFT); 4669 if (flags & FOLL_GET) 4670 get_page(page); 4671 } else { 4672 if (is_hugetlb_entry_migration(pte)) { 4673 spin_unlock(ptl); 4674 __migration_entry_wait(mm, (pte_t *)pmd, ptl); 4675 goto retry; 4676 } 4677 /* 4678 * hwpoisoned entry is treated as no_page_table in 4679 * follow_page_mask(). 4680 */ 4681 } 4682 out: 4683 spin_unlock(ptl); 4684 return page; 4685 } 4686 4687 struct page * __weak 4688 follow_huge_pud(struct mm_struct *mm, unsigned long address, 4689 pud_t *pud, int flags) 4690 { 4691 if (flags & FOLL_GET) 4692 return NULL; 4693 4694 return pte_page(*(pte_t *)pud) + ((address & ~PUD_MASK) >> PAGE_SHIFT); 4695 } 4696 4697 struct page * __weak 4698 follow_huge_pgd(struct mm_struct *mm, unsigned long address, pgd_t *pgd, int flags) 4699 { 4700 if (flags & FOLL_GET) 4701 return NULL; 4702 4703 return pte_page(*(pte_t *)pgd) + ((address & ~PGDIR_MASK) >> PAGE_SHIFT); 4704 } 4705 4706 bool isolate_huge_page(struct page *page, struct list_head *list) 4707 { 4708 bool ret = true; 4709 4710 VM_BUG_ON_PAGE(!PageHead(page), page); 4711 spin_lock(&hugetlb_lock); 4712 if (!page_huge_active(page) || !get_page_unless_zero(page)) { 4713 ret = false; 4714 goto unlock; 4715 } 4716 clear_page_huge_active(page); 4717 list_move_tail(&page->lru, list); 4718 unlock: 4719 spin_unlock(&hugetlb_lock); 4720 return ret; 4721 } 4722 4723 void putback_active_hugepage(struct page *page) 4724 { 4725 VM_BUG_ON_PAGE(!PageHead(page), page); 4726 spin_lock(&hugetlb_lock); 4727 set_page_huge_active(page); 4728 list_move_tail(&page->lru, &(page_hstate(page))->hugepage_activelist); 4729 spin_unlock(&hugetlb_lock); 4730 put_page(page); 4731 } 4732