1 /* 2 * Generic hugetlb support. 3 * (C) Nadia Yvette Chambers, April 2004 4 */ 5 #include <linux/list.h> 6 #include <linux/init.h> 7 #include <linux/mm.h> 8 #include <linux/seq_file.h> 9 #include <linux/sysctl.h> 10 #include <linux/highmem.h> 11 #include <linux/mmu_notifier.h> 12 #include <linux/nodemask.h> 13 #include <linux/pagemap.h> 14 #include <linux/mempolicy.h> 15 #include <linux/compiler.h> 16 #include <linux/cpuset.h> 17 #include <linux/mutex.h> 18 #include <linux/bootmem.h> 19 #include <linux/sysfs.h> 20 #include <linux/slab.h> 21 #include <linux/rmap.h> 22 #include <linux/swap.h> 23 #include <linux/swapops.h> 24 #include <linux/page-isolation.h> 25 #include <linux/jhash.h> 26 27 #include <asm/page.h> 28 #include <asm/pgtable.h> 29 #include <asm/tlb.h> 30 31 #include <linux/io.h> 32 #include <linux/hugetlb.h> 33 #include <linux/hugetlb_cgroup.h> 34 #include <linux/node.h> 35 #include "internal.h" 36 37 int hugepages_treat_as_movable; 38 39 int hugetlb_max_hstate __read_mostly; 40 unsigned int default_hstate_idx; 41 struct hstate hstates[HUGE_MAX_HSTATE]; 42 /* 43 * Minimum page order among possible hugepage sizes, set to a proper value 44 * at boot time. 45 */ 46 static unsigned int minimum_order __read_mostly = UINT_MAX; 47 48 __initdata LIST_HEAD(huge_boot_pages); 49 50 /* for command line parsing */ 51 static struct hstate * __initdata parsed_hstate; 52 static unsigned long __initdata default_hstate_max_huge_pages; 53 static unsigned long __initdata default_hstate_size; 54 static bool __initdata parsed_valid_hugepagesz = true; 55 56 /* 57 * Protects updates to hugepage_freelists, hugepage_activelist, nr_huge_pages, 58 * free_huge_pages, and surplus_huge_pages. 59 */ 60 DEFINE_SPINLOCK(hugetlb_lock); 61 62 /* 63 * Serializes faults on the same logical page. This is used to 64 * prevent spurious OOMs when the hugepage pool is fully utilized. 65 */ 66 static int num_fault_mutexes; 67 struct mutex *hugetlb_fault_mutex_table ____cacheline_aligned_in_smp; 68 69 /* Forward declaration */ 70 static int hugetlb_acct_memory(struct hstate *h, long delta); 71 72 static inline void unlock_or_release_subpool(struct hugepage_subpool *spool) 73 { 74 bool free = (spool->count == 0) && (spool->used_hpages == 0); 75 76 spin_unlock(&spool->lock); 77 78 /* If no pages are used, and no other handles to the subpool 79 * remain, give up any reservations mased on minimum size and 80 * free the subpool */ 81 if (free) { 82 if (spool->min_hpages != -1) 83 hugetlb_acct_memory(spool->hstate, 84 -spool->min_hpages); 85 kfree(spool); 86 } 87 } 88 89 struct hugepage_subpool *hugepage_new_subpool(struct hstate *h, long max_hpages, 90 long min_hpages) 91 { 92 struct hugepage_subpool *spool; 93 94 spool = kzalloc(sizeof(*spool), GFP_KERNEL); 95 if (!spool) 96 return NULL; 97 98 spin_lock_init(&spool->lock); 99 spool->count = 1; 100 spool->max_hpages = max_hpages; 101 spool->hstate = h; 102 spool->min_hpages = min_hpages; 103 104 if (min_hpages != -1 && hugetlb_acct_memory(h, min_hpages)) { 105 kfree(spool); 106 return NULL; 107 } 108 spool->rsv_hpages = min_hpages; 109 110 return spool; 111 } 112 113 void hugepage_put_subpool(struct hugepage_subpool *spool) 114 { 115 spin_lock(&spool->lock); 116 BUG_ON(!spool->count); 117 spool->count--; 118 unlock_or_release_subpool(spool); 119 } 120 121 /* 122 * Subpool accounting for allocating and reserving pages. 123 * Return -ENOMEM if there are not enough resources to satisfy the 124 * the request. Otherwise, return the number of pages by which the 125 * global pools must be adjusted (upward). The returned value may 126 * only be different than the passed value (delta) in the case where 127 * a subpool minimum size must be manitained. 128 */ 129 static long hugepage_subpool_get_pages(struct hugepage_subpool *spool, 130 long delta) 131 { 132 long ret = delta; 133 134 if (!spool) 135 return ret; 136 137 spin_lock(&spool->lock); 138 139 if (spool->max_hpages != -1) { /* maximum size accounting */ 140 if ((spool->used_hpages + delta) <= spool->max_hpages) 141 spool->used_hpages += delta; 142 else { 143 ret = -ENOMEM; 144 goto unlock_ret; 145 } 146 } 147 148 /* minimum size accounting */ 149 if (spool->min_hpages != -1 && spool->rsv_hpages) { 150 if (delta > spool->rsv_hpages) { 151 /* 152 * Asking for more reserves than those already taken on 153 * behalf of subpool. Return difference. 154 */ 155 ret = delta - spool->rsv_hpages; 156 spool->rsv_hpages = 0; 157 } else { 158 ret = 0; /* reserves already accounted for */ 159 spool->rsv_hpages -= delta; 160 } 161 } 162 163 unlock_ret: 164 spin_unlock(&spool->lock); 165 return ret; 166 } 167 168 /* 169 * Subpool accounting for freeing and unreserving pages. 170 * Return the number of global page reservations that must be dropped. 171 * The return value may only be different than the passed value (delta) 172 * in the case where a subpool minimum size must be maintained. 173 */ 174 static long hugepage_subpool_put_pages(struct hugepage_subpool *spool, 175 long delta) 176 { 177 long ret = delta; 178 179 if (!spool) 180 return delta; 181 182 spin_lock(&spool->lock); 183 184 if (spool->max_hpages != -1) /* maximum size accounting */ 185 spool->used_hpages -= delta; 186 187 /* minimum size accounting */ 188 if (spool->min_hpages != -1 && spool->used_hpages < spool->min_hpages) { 189 if (spool->rsv_hpages + delta <= spool->min_hpages) 190 ret = 0; 191 else 192 ret = spool->rsv_hpages + delta - spool->min_hpages; 193 194 spool->rsv_hpages += delta; 195 if (spool->rsv_hpages > spool->min_hpages) 196 spool->rsv_hpages = spool->min_hpages; 197 } 198 199 /* 200 * If hugetlbfs_put_super couldn't free spool due to an outstanding 201 * quota reference, free it now. 202 */ 203 unlock_or_release_subpool(spool); 204 205 return ret; 206 } 207 208 static inline struct hugepage_subpool *subpool_inode(struct inode *inode) 209 { 210 return HUGETLBFS_SB(inode->i_sb)->spool; 211 } 212 213 static inline struct hugepage_subpool *subpool_vma(struct vm_area_struct *vma) 214 { 215 return subpool_inode(file_inode(vma->vm_file)); 216 } 217 218 /* 219 * Region tracking -- allows tracking of reservations and instantiated pages 220 * across the pages in a mapping. 221 * 222 * The region data structures are embedded into a resv_map and protected 223 * by a resv_map's lock. The set of regions within the resv_map represent 224 * reservations for huge pages, or huge pages that have already been 225 * instantiated within the map. The from and to elements are huge page 226 * indicies into the associated mapping. from indicates the starting index 227 * of the region. to represents the first index past the end of the region. 228 * 229 * For example, a file region structure with from == 0 and to == 4 represents 230 * four huge pages in a mapping. It is important to note that the to element 231 * represents the first element past the end of the region. This is used in 232 * arithmetic as 4(to) - 0(from) = 4 huge pages in the region. 233 * 234 * Interval notation of the form [from, to) will be used to indicate that 235 * the endpoint from is inclusive and to is exclusive. 236 */ 237 struct file_region { 238 struct list_head link; 239 long from; 240 long to; 241 }; 242 243 /* 244 * Add the huge page range represented by [f, t) to the reserve 245 * map. In the normal case, existing regions will be expanded 246 * to accommodate the specified range. Sufficient regions should 247 * exist for expansion due to the previous call to region_chg 248 * with the same range. However, it is possible that region_del 249 * could have been called after region_chg and modifed the map 250 * in such a way that no region exists to be expanded. In this 251 * case, pull a region descriptor from the cache associated with 252 * the map and use that for the new range. 253 * 254 * Return the number of new huge pages added to the map. This 255 * number is greater than or equal to zero. 256 */ 257 static long region_add(struct resv_map *resv, long f, long t) 258 { 259 struct list_head *head = &resv->regions; 260 struct file_region *rg, *nrg, *trg; 261 long add = 0; 262 263 spin_lock(&resv->lock); 264 /* Locate the region we are either in or before. */ 265 list_for_each_entry(rg, head, link) 266 if (f <= rg->to) 267 break; 268 269 /* 270 * If no region exists which can be expanded to include the 271 * specified range, the list must have been modified by an 272 * interleving call to region_del(). Pull a region descriptor 273 * from the cache and use it for this range. 274 */ 275 if (&rg->link == head || t < rg->from) { 276 VM_BUG_ON(resv->region_cache_count <= 0); 277 278 resv->region_cache_count--; 279 nrg = list_first_entry(&resv->region_cache, struct file_region, 280 link); 281 list_del(&nrg->link); 282 283 nrg->from = f; 284 nrg->to = t; 285 list_add(&nrg->link, rg->link.prev); 286 287 add += t - f; 288 goto out_locked; 289 } 290 291 /* Round our left edge to the current segment if it encloses us. */ 292 if (f > rg->from) 293 f = rg->from; 294 295 /* Check for and consume any regions we now overlap with. */ 296 nrg = rg; 297 list_for_each_entry_safe(rg, trg, rg->link.prev, link) { 298 if (&rg->link == head) 299 break; 300 if (rg->from > t) 301 break; 302 303 /* If this area reaches higher then extend our area to 304 * include it completely. If this is not the first area 305 * which we intend to reuse, free it. */ 306 if (rg->to > t) 307 t = rg->to; 308 if (rg != nrg) { 309 /* Decrement return value by the deleted range. 310 * Another range will span this area so that by 311 * end of routine add will be >= zero 312 */ 313 add -= (rg->to - rg->from); 314 list_del(&rg->link); 315 kfree(rg); 316 } 317 } 318 319 add += (nrg->from - f); /* Added to beginning of region */ 320 nrg->from = f; 321 add += t - nrg->to; /* Added to end of region */ 322 nrg->to = t; 323 324 out_locked: 325 resv->adds_in_progress--; 326 spin_unlock(&resv->lock); 327 VM_BUG_ON(add < 0); 328 return add; 329 } 330 331 /* 332 * Examine the existing reserve map and determine how many 333 * huge pages in the specified range [f, t) are NOT currently 334 * represented. This routine is called before a subsequent 335 * call to region_add that will actually modify the reserve 336 * map to add the specified range [f, t). region_chg does 337 * not change the number of huge pages represented by the 338 * map. However, if the existing regions in the map can not 339 * be expanded to represent the new range, a new file_region 340 * structure is added to the map as a placeholder. This is 341 * so that the subsequent region_add call will have all the 342 * regions it needs and will not fail. 343 * 344 * Upon entry, region_chg will also examine the cache of region descriptors 345 * associated with the map. If there are not enough descriptors cached, one 346 * will be allocated for the in progress add operation. 347 * 348 * Returns the number of huge pages that need to be added to the existing 349 * reservation map for the range [f, t). This number is greater or equal to 350 * zero. -ENOMEM is returned if a new file_region structure or cache entry 351 * is needed and can not be allocated. 352 */ 353 static long region_chg(struct resv_map *resv, long f, long t) 354 { 355 struct list_head *head = &resv->regions; 356 struct file_region *rg, *nrg = NULL; 357 long chg = 0; 358 359 retry: 360 spin_lock(&resv->lock); 361 retry_locked: 362 resv->adds_in_progress++; 363 364 /* 365 * Check for sufficient descriptors in the cache to accommodate 366 * the number of in progress add operations. 367 */ 368 if (resv->adds_in_progress > resv->region_cache_count) { 369 struct file_region *trg; 370 371 VM_BUG_ON(resv->adds_in_progress - resv->region_cache_count > 1); 372 /* Must drop lock to allocate a new descriptor. */ 373 resv->adds_in_progress--; 374 spin_unlock(&resv->lock); 375 376 trg = kmalloc(sizeof(*trg), GFP_KERNEL); 377 if (!trg) { 378 kfree(nrg); 379 return -ENOMEM; 380 } 381 382 spin_lock(&resv->lock); 383 list_add(&trg->link, &resv->region_cache); 384 resv->region_cache_count++; 385 goto retry_locked; 386 } 387 388 /* Locate the region we are before or in. */ 389 list_for_each_entry(rg, head, link) 390 if (f <= rg->to) 391 break; 392 393 /* If we are below the current region then a new region is required. 394 * Subtle, allocate a new region at the position but make it zero 395 * size such that we can guarantee to record the reservation. */ 396 if (&rg->link == head || t < rg->from) { 397 if (!nrg) { 398 resv->adds_in_progress--; 399 spin_unlock(&resv->lock); 400 nrg = kmalloc(sizeof(*nrg), GFP_KERNEL); 401 if (!nrg) 402 return -ENOMEM; 403 404 nrg->from = f; 405 nrg->to = f; 406 INIT_LIST_HEAD(&nrg->link); 407 goto retry; 408 } 409 410 list_add(&nrg->link, rg->link.prev); 411 chg = t - f; 412 goto out_nrg; 413 } 414 415 /* Round our left edge to the current segment if it encloses us. */ 416 if (f > rg->from) 417 f = rg->from; 418 chg = t - f; 419 420 /* Check for and consume any regions we now overlap with. */ 421 list_for_each_entry(rg, rg->link.prev, link) { 422 if (&rg->link == head) 423 break; 424 if (rg->from > t) 425 goto out; 426 427 /* We overlap with this area, if it extends further than 428 * us then we must extend ourselves. Account for its 429 * existing reservation. */ 430 if (rg->to > t) { 431 chg += rg->to - t; 432 t = rg->to; 433 } 434 chg -= rg->to - rg->from; 435 } 436 437 out: 438 spin_unlock(&resv->lock); 439 /* We already know we raced and no longer need the new region */ 440 kfree(nrg); 441 return chg; 442 out_nrg: 443 spin_unlock(&resv->lock); 444 return chg; 445 } 446 447 /* 448 * Abort the in progress add operation. The adds_in_progress field 449 * of the resv_map keeps track of the operations in progress between 450 * calls to region_chg and region_add. Operations are sometimes 451 * aborted after the call to region_chg. In such cases, region_abort 452 * is called to decrement the adds_in_progress counter. 453 * 454 * NOTE: The range arguments [f, t) are not needed or used in this 455 * routine. They are kept to make reading the calling code easier as 456 * arguments will match the associated region_chg call. 457 */ 458 static void region_abort(struct resv_map *resv, long f, long t) 459 { 460 spin_lock(&resv->lock); 461 VM_BUG_ON(!resv->region_cache_count); 462 resv->adds_in_progress--; 463 spin_unlock(&resv->lock); 464 } 465 466 /* 467 * Delete the specified range [f, t) from the reserve map. If the 468 * t parameter is LONG_MAX, this indicates that ALL regions after f 469 * should be deleted. Locate the regions which intersect [f, t) 470 * and either trim, delete or split the existing regions. 471 * 472 * Returns the number of huge pages deleted from the reserve map. 473 * In the normal case, the return value is zero or more. In the 474 * case where a region must be split, a new region descriptor must 475 * be allocated. If the allocation fails, -ENOMEM will be returned. 476 * NOTE: If the parameter t == LONG_MAX, then we will never split 477 * a region and possibly return -ENOMEM. Callers specifying 478 * t == LONG_MAX do not need to check for -ENOMEM error. 479 */ 480 static long region_del(struct resv_map *resv, long f, long t) 481 { 482 struct list_head *head = &resv->regions; 483 struct file_region *rg, *trg; 484 struct file_region *nrg = NULL; 485 long del = 0; 486 487 retry: 488 spin_lock(&resv->lock); 489 list_for_each_entry_safe(rg, trg, head, link) { 490 /* 491 * Skip regions before the range to be deleted. file_region 492 * ranges are normally of the form [from, to). However, there 493 * may be a "placeholder" entry in the map which is of the form 494 * (from, to) with from == to. Check for placeholder entries 495 * at the beginning of the range to be deleted. 496 */ 497 if (rg->to <= f && (rg->to != rg->from || rg->to != f)) 498 continue; 499 500 if (rg->from >= t) 501 break; 502 503 if (f > rg->from && t < rg->to) { /* Must split region */ 504 /* 505 * Check for an entry in the cache before dropping 506 * lock and attempting allocation. 507 */ 508 if (!nrg && 509 resv->region_cache_count > resv->adds_in_progress) { 510 nrg = list_first_entry(&resv->region_cache, 511 struct file_region, 512 link); 513 list_del(&nrg->link); 514 resv->region_cache_count--; 515 } 516 517 if (!nrg) { 518 spin_unlock(&resv->lock); 519 nrg = kmalloc(sizeof(*nrg), GFP_KERNEL); 520 if (!nrg) 521 return -ENOMEM; 522 goto retry; 523 } 524 525 del += t - f; 526 527 /* New entry for end of split region */ 528 nrg->from = t; 529 nrg->to = rg->to; 530 INIT_LIST_HEAD(&nrg->link); 531 532 /* Original entry is trimmed */ 533 rg->to = f; 534 535 list_add(&nrg->link, &rg->link); 536 nrg = NULL; 537 break; 538 } 539 540 if (f <= rg->from && t >= rg->to) { /* Remove entire region */ 541 del += rg->to - rg->from; 542 list_del(&rg->link); 543 kfree(rg); 544 continue; 545 } 546 547 if (f <= rg->from) { /* Trim beginning of region */ 548 del += t - rg->from; 549 rg->from = t; 550 } else { /* Trim end of region */ 551 del += rg->to - f; 552 rg->to = f; 553 } 554 } 555 556 spin_unlock(&resv->lock); 557 kfree(nrg); 558 return del; 559 } 560 561 /* 562 * A rare out of memory error was encountered which prevented removal of 563 * the reserve map region for a page. The huge page itself was free'ed 564 * and removed from the page cache. This routine will adjust the subpool 565 * usage count, and the global reserve count if needed. By incrementing 566 * these counts, the reserve map entry which could not be deleted will 567 * appear as a "reserved" entry instead of simply dangling with incorrect 568 * counts. 569 */ 570 void hugetlb_fix_reserve_counts(struct inode *inode, bool restore_reserve) 571 { 572 struct hugepage_subpool *spool = subpool_inode(inode); 573 long rsv_adjust; 574 575 rsv_adjust = hugepage_subpool_get_pages(spool, 1); 576 if (restore_reserve && rsv_adjust) { 577 struct hstate *h = hstate_inode(inode); 578 579 hugetlb_acct_memory(h, 1); 580 } 581 } 582 583 /* 584 * Count and return the number of huge pages in the reserve map 585 * that intersect with the range [f, t). 586 */ 587 static long region_count(struct resv_map *resv, long f, long t) 588 { 589 struct list_head *head = &resv->regions; 590 struct file_region *rg; 591 long chg = 0; 592 593 spin_lock(&resv->lock); 594 /* Locate each segment we overlap with, and count that overlap. */ 595 list_for_each_entry(rg, head, link) { 596 long seg_from; 597 long seg_to; 598 599 if (rg->to <= f) 600 continue; 601 if (rg->from >= t) 602 break; 603 604 seg_from = max(rg->from, f); 605 seg_to = min(rg->to, t); 606 607 chg += seg_to - seg_from; 608 } 609 spin_unlock(&resv->lock); 610 611 return chg; 612 } 613 614 /* 615 * Convert the address within this vma to the page offset within 616 * the mapping, in pagecache page units; huge pages here. 617 */ 618 static pgoff_t vma_hugecache_offset(struct hstate *h, 619 struct vm_area_struct *vma, unsigned long address) 620 { 621 return ((address - vma->vm_start) >> huge_page_shift(h)) + 622 (vma->vm_pgoff >> huge_page_order(h)); 623 } 624 625 pgoff_t linear_hugepage_index(struct vm_area_struct *vma, 626 unsigned long address) 627 { 628 return vma_hugecache_offset(hstate_vma(vma), vma, address); 629 } 630 EXPORT_SYMBOL_GPL(linear_hugepage_index); 631 632 /* 633 * Return the size of the pages allocated when backing a VMA. In the majority 634 * cases this will be same size as used by the page table entries. 635 */ 636 unsigned long vma_kernel_pagesize(struct vm_area_struct *vma) 637 { 638 struct hstate *hstate; 639 640 if (!is_vm_hugetlb_page(vma)) 641 return PAGE_SIZE; 642 643 hstate = hstate_vma(vma); 644 645 return 1UL << huge_page_shift(hstate); 646 } 647 EXPORT_SYMBOL_GPL(vma_kernel_pagesize); 648 649 /* 650 * Return the page size being used by the MMU to back a VMA. In the majority 651 * of cases, the page size used by the kernel matches the MMU size. On 652 * architectures where it differs, an architecture-specific version of this 653 * function is required. 654 */ 655 #ifndef vma_mmu_pagesize 656 unsigned long vma_mmu_pagesize(struct vm_area_struct *vma) 657 { 658 return vma_kernel_pagesize(vma); 659 } 660 #endif 661 662 /* 663 * Flags for MAP_PRIVATE reservations. These are stored in the bottom 664 * bits of the reservation map pointer, which are always clear due to 665 * alignment. 666 */ 667 #define HPAGE_RESV_OWNER (1UL << 0) 668 #define HPAGE_RESV_UNMAPPED (1UL << 1) 669 #define HPAGE_RESV_MASK (HPAGE_RESV_OWNER | HPAGE_RESV_UNMAPPED) 670 671 /* 672 * These helpers are used to track how many pages are reserved for 673 * faults in a MAP_PRIVATE mapping. Only the process that called mmap() 674 * is guaranteed to have their future faults succeed. 675 * 676 * With the exception of reset_vma_resv_huge_pages() which is called at fork(), 677 * the reserve counters are updated with the hugetlb_lock held. It is safe 678 * to reset the VMA at fork() time as it is not in use yet and there is no 679 * chance of the global counters getting corrupted as a result of the values. 680 * 681 * The private mapping reservation is represented in a subtly different 682 * manner to a shared mapping. A shared mapping has a region map associated 683 * with the underlying file, this region map represents the backing file 684 * pages which have ever had a reservation assigned which this persists even 685 * after the page is instantiated. A private mapping has a region map 686 * associated with the original mmap which is attached to all VMAs which 687 * reference it, this region map represents those offsets which have consumed 688 * reservation ie. where pages have been instantiated. 689 */ 690 static unsigned long get_vma_private_data(struct vm_area_struct *vma) 691 { 692 return (unsigned long)vma->vm_private_data; 693 } 694 695 static void set_vma_private_data(struct vm_area_struct *vma, 696 unsigned long value) 697 { 698 vma->vm_private_data = (void *)value; 699 } 700 701 struct resv_map *resv_map_alloc(void) 702 { 703 struct resv_map *resv_map = kmalloc(sizeof(*resv_map), GFP_KERNEL); 704 struct file_region *rg = kmalloc(sizeof(*rg), GFP_KERNEL); 705 706 if (!resv_map || !rg) { 707 kfree(resv_map); 708 kfree(rg); 709 return NULL; 710 } 711 712 kref_init(&resv_map->refs); 713 spin_lock_init(&resv_map->lock); 714 INIT_LIST_HEAD(&resv_map->regions); 715 716 resv_map->adds_in_progress = 0; 717 718 INIT_LIST_HEAD(&resv_map->region_cache); 719 list_add(&rg->link, &resv_map->region_cache); 720 resv_map->region_cache_count = 1; 721 722 return resv_map; 723 } 724 725 void resv_map_release(struct kref *ref) 726 { 727 struct resv_map *resv_map = container_of(ref, struct resv_map, refs); 728 struct list_head *head = &resv_map->region_cache; 729 struct file_region *rg, *trg; 730 731 /* Clear out any active regions before we release the map. */ 732 region_del(resv_map, 0, LONG_MAX); 733 734 /* ... and any entries left in the cache */ 735 list_for_each_entry_safe(rg, trg, head, link) { 736 list_del(&rg->link); 737 kfree(rg); 738 } 739 740 VM_BUG_ON(resv_map->adds_in_progress); 741 742 kfree(resv_map); 743 } 744 745 static inline struct resv_map *inode_resv_map(struct inode *inode) 746 { 747 return inode->i_mapping->private_data; 748 } 749 750 static struct resv_map *vma_resv_map(struct vm_area_struct *vma) 751 { 752 VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma); 753 if (vma->vm_flags & VM_MAYSHARE) { 754 struct address_space *mapping = vma->vm_file->f_mapping; 755 struct inode *inode = mapping->host; 756 757 return inode_resv_map(inode); 758 759 } else { 760 return (struct resv_map *)(get_vma_private_data(vma) & 761 ~HPAGE_RESV_MASK); 762 } 763 } 764 765 static void set_vma_resv_map(struct vm_area_struct *vma, struct resv_map *map) 766 { 767 VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma); 768 VM_BUG_ON_VMA(vma->vm_flags & VM_MAYSHARE, vma); 769 770 set_vma_private_data(vma, (get_vma_private_data(vma) & 771 HPAGE_RESV_MASK) | (unsigned long)map); 772 } 773 774 static void set_vma_resv_flags(struct vm_area_struct *vma, unsigned long flags) 775 { 776 VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma); 777 VM_BUG_ON_VMA(vma->vm_flags & VM_MAYSHARE, vma); 778 779 set_vma_private_data(vma, get_vma_private_data(vma) | flags); 780 } 781 782 static int is_vma_resv_set(struct vm_area_struct *vma, unsigned long flag) 783 { 784 VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma); 785 786 return (get_vma_private_data(vma) & flag) != 0; 787 } 788 789 /* Reset counters to 0 and clear all HPAGE_RESV_* flags */ 790 void reset_vma_resv_huge_pages(struct vm_area_struct *vma) 791 { 792 VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma); 793 if (!(vma->vm_flags & VM_MAYSHARE)) 794 vma->vm_private_data = (void *)0; 795 } 796 797 /* Returns true if the VMA has associated reserve pages */ 798 static bool vma_has_reserves(struct vm_area_struct *vma, long chg) 799 { 800 if (vma->vm_flags & VM_NORESERVE) { 801 /* 802 * This address is already reserved by other process(chg == 0), 803 * so, we should decrement reserved count. Without decrementing, 804 * reserve count remains after releasing inode, because this 805 * allocated page will go into page cache and is regarded as 806 * coming from reserved pool in releasing step. Currently, we 807 * don't have any other solution to deal with this situation 808 * properly, so add work-around here. 809 */ 810 if (vma->vm_flags & VM_MAYSHARE && chg == 0) 811 return true; 812 else 813 return false; 814 } 815 816 /* Shared mappings always use reserves */ 817 if (vma->vm_flags & VM_MAYSHARE) { 818 /* 819 * We know VM_NORESERVE is not set. Therefore, there SHOULD 820 * be a region map for all pages. The only situation where 821 * there is no region map is if a hole was punched via 822 * fallocate. In this case, there really are no reverves to 823 * use. This situation is indicated if chg != 0. 824 */ 825 if (chg) 826 return false; 827 else 828 return true; 829 } 830 831 /* 832 * Only the process that called mmap() has reserves for 833 * private mappings. 834 */ 835 if (is_vma_resv_set(vma, HPAGE_RESV_OWNER)) { 836 /* 837 * Like the shared case above, a hole punch or truncate 838 * could have been performed on the private mapping. 839 * Examine the value of chg to determine if reserves 840 * actually exist or were previously consumed. 841 * Very Subtle - The value of chg comes from a previous 842 * call to vma_needs_reserves(). The reserve map for 843 * private mappings has different (opposite) semantics 844 * than that of shared mappings. vma_needs_reserves() 845 * has already taken this difference in semantics into 846 * account. Therefore, the meaning of chg is the same 847 * as in the shared case above. Code could easily be 848 * combined, but keeping it separate draws attention to 849 * subtle differences. 850 */ 851 if (chg) 852 return false; 853 else 854 return true; 855 } 856 857 return false; 858 } 859 860 static void enqueue_huge_page(struct hstate *h, struct page *page) 861 { 862 int nid = page_to_nid(page); 863 list_move(&page->lru, &h->hugepage_freelists[nid]); 864 h->free_huge_pages++; 865 h->free_huge_pages_node[nid]++; 866 } 867 868 static struct page *dequeue_huge_page_node(struct hstate *h, int nid) 869 { 870 struct page *page; 871 872 list_for_each_entry(page, &h->hugepage_freelists[nid], lru) 873 if (!is_migrate_isolate_page(page)) 874 break; 875 /* 876 * if 'non-isolated free hugepage' not found on the list, 877 * the allocation fails. 878 */ 879 if (&h->hugepage_freelists[nid] == &page->lru) 880 return NULL; 881 list_move(&page->lru, &h->hugepage_activelist); 882 set_page_refcounted(page); 883 h->free_huge_pages--; 884 h->free_huge_pages_node[nid]--; 885 return page; 886 } 887 888 /* Movability of hugepages depends on migration support. */ 889 static inline gfp_t htlb_alloc_mask(struct hstate *h) 890 { 891 if (hugepages_treat_as_movable || hugepage_migration_supported(h)) 892 return GFP_HIGHUSER_MOVABLE; 893 else 894 return GFP_HIGHUSER; 895 } 896 897 static struct page *dequeue_huge_page_vma(struct hstate *h, 898 struct vm_area_struct *vma, 899 unsigned long address, int avoid_reserve, 900 long chg) 901 { 902 struct page *page = NULL; 903 struct mempolicy *mpol; 904 nodemask_t *nodemask; 905 struct zonelist *zonelist; 906 struct zone *zone; 907 struct zoneref *z; 908 unsigned int cpuset_mems_cookie; 909 910 /* 911 * A child process with MAP_PRIVATE mappings created by their parent 912 * have no page reserves. This check ensures that reservations are 913 * not "stolen". The child may still get SIGKILLed 914 */ 915 if (!vma_has_reserves(vma, chg) && 916 h->free_huge_pages - h->resv_huge_pages == 0) 917 goto err; 918 919 /* If reserves cannot be used, ensure enough pages are in the pool */ 920 if (avoid_reserve && h->free_huge_pages - h->resv_huge_pages == 0) 921 goto err; 922 923 retry_cpuset: 924 cpuset_mems_cookie = read_mems_allowed_begin(); 925 zonelist = huge_zonelist(vma, address, 926 htlb_alloc_mask(h), &mpol, &nodemask); 927 928 for_each_zone_zonelist_nodemask(zone, z, zonelist, 929 MAX_NR_ZONES - 1, nodemask) { 930 if (cpuset_zone_allowed(zone, htlb_alloc_mask(h))) { 931 page = dequeue_huge_page_node(h, zone_to_nid(zone)); 932 if (page) { 933 if (avoid_reserve) 934 break; 935 if (!vma_has_reserves(vma, chg)) 936 break; 937 938 SetPagePrivate(page); 939 h->resv_huge_pages--; 940 break; 941 } 942 } 943 } 944 945 mpol_cond_put(mpol); 946 if (unlikely(!page && read_mems_allowed_retry(cpuset_mems_cookie))) 947 goto retry_cpuset; 948 return page; 949 950 err: 951 return NULL; 952 } 953 954 /* 955 * common helper functions for hstate_next_node_to_{alloc|free}. 956 * We may have allocated or freed a huge page based on a different 957 * nodes_allowed previously, so h->next_node_to_{alloc|free} might 958 * be outside of *nodes_allowed. Ensure that we use an allowed 959 * node for alloc or free. 960 */ 961 static int next_node_allowed(int nid, nodemask_t *nodes_allowed) 962 { 963 nid = next_node_in(nid, *nodes_allowed); 964 VM_BUG_ON(nid >= MAX_NUMNODES); 965 966 return nid; 967 } 968 969 static int get_valid_node_allowed(int nid, nodemask_t *nodes_allowed) 970 { 971 if (!node_isset(nid, *nodes_allowed)) 972 nid = next_node_allowed(nid, nodes_allowed); 973 return nid; 974 } 975 976 /* 977 * returns the previously saved node ["this node"] from which to 978 * allocate a persistent huge page for the pool and advance the 979 * next node from which to allocate, handling wrap at end of node 980 * mask. 981 */ 982 static int hstate_next_node_to_alloc(struct hstate *h, 983 nodemask_t *nodes_allowed) 984 { 985 int nid; 986 987 VM_BUG_ON(!nodes_allowed); 988 989 nid = get_valid_node_allowed(h->next_nid_to_alloc, nodes_allowed); 990 h->next_nid_to_alloc = next_node_allowed(nid, nodes_allowed); 991 992 return nid; 993 } 994 995 /* 996 * helper for free_pool_huge_page() - return the previously saved 997 * node ["this node"] from which to free a huge page. Advance the 998 * next node id whether or not we find a free huge page to free so 999 * that the next attempt to free addresses the next node. 1000 */ 1001 static int hstate_next_node_to_free(struct hstate *h, nodemask_t *nodes_allowed) 1002 { 1003 int nid; 1004 1005 VM_BUG_ON(!nodes_allowed); 1006 1007 nid = get_valid_node_allowed(h->next_nid_to_free, nodes_allowed); 1008 h->next_nid_to_free = next_node_allowed(nid, nodes_allowed); 1009 1010 return nid; 1011 } 1012 1013 #define for_each_node_mask_to_alloc(hs, nr_nodes, node, mask) \ 1014 for (nr_nodes = nodes_weight(*mask); \ 1015 nr_nodes > 0 && \ 1016 ((node = hstate_next_node_to_alloc(hs, mask)) || 1); \ 1017 nr_nodes--) 1018 1019 #define for_each_node_mask_to_free(hs, nr_nodes, node, mask) \ 1020 for (nr_nodes = nodes_weight(*mask); \ 1021 nr_nodes > 0 && \ 1022 ((node = hstate_next_node_to_free(hs, mask)) || 1); \ 1023 nr_nodes--) 1024 1025 #if (defined(CONFIG_X86_64) || defined(CONFIG_S390)) && \ 1026 ((defined(CONFIG_MEMORY_ISOLATION) && defined(CONFIG_COMPACTION)) || \ 1027 defined(CONFIG_CMA)) 1028 static void destroy_compound_gigantic_page(struct page *page, 1029 unsigned int order) 1030 { 1031 int i; 1032 int nr_pages = 1 << order; 1033 struct page *p = page + 1; 1034 1035 atomic_set(compound_mapcount_ptr(page), 0); 1036 for (i = 1; i < nr_pages; i++, p = mem_map_next(p, page, i)) { 1037 clear_compound_head(p); 1038 set_page_refcounted(p); 1039 } 1040 1041 set_compound_order(page, 0); 1042 __ClearPageHead(page); 1043 } 1044 1045 static void free_gigantic_page(struct page *page, unsigned int order) 1046 { 1047 free_contig_range(page_to_pfn(page), 1 << order); 1048 } 1049 1050 static int __alloc_gigantic_page(unsigned long start_pfn, 1051 unsigned long nr_pages) 1052 { 1053 unsigned long end_pfn = start_pfn + nr_pages; 1054 return alloc_contig_range(start_pfn, end_pfn, MIGRATE_MOVABLE); 1055 } 1056 1057 static bool pfn_range_valid_gigantic(struct zone *z, 1058 unsigned long start_pfn, unsigned long nr_pages) 1059 { 1060 unsigned long i, end_pfn = start_pfn + nr_pages; 1061 struct page *page; 1062 1063 for (i = start_pfn; i < end_pfn; i++) { 1064 if (!pfn_valid(i)) 1065 return false; 1066 1067 page = pfn_to_page(i); 1068 1069 if (page_zone(page) != z) 1070 return false; 1071 1072 if (PageReserved(page)) 1073 return false; 1074 1075 if (page_count(page) > 0) 1076 return false; 1077 1078 if (PageHuge(page)) 1079 return false; 1080 } 1081 1082 return true; 1083 } 1084 1085 static bool zone_spans_last_pfn(const struct zone *zone, 1086 unsigned long start_pfn, unsigned long nr_pages) 1087 { 1088 unsigned long last_pfn = start_pfn + nr_pages - 1; 1089 return zone_spans_pfn(zone, last_pfn); 1090 } 1091 1092 static struct page *alloc_gigantic_page(int nid, unsigned int order) 1093 { 1094 unsigned long nr_pages = 1 << order; 1095 unsigned long ret, pfn, flags; 1096 struct zone *z; 1097 1098 z = NODE_DATA(nid)->node_zones; 1099 for (; z - NODE_DATA(nid)->node_zones < MAX_NR_ZONES; z++) { 1100 spin_lock_irqsave(&z->lock, flags); 1101 1102 pfn = ALIGN(z->zone_start_pfn, nr_pages); 1103 while (zone_spans_last_pfn(z, pfn, nr_pages)) { 1104 if (pfn_range_valid_gigantic(z, pfn, nr_pages)) { 1105 /* 1106 * We release the zone lock here because 1107 * alloc_contig_range() will also lock the zone 1108 * at some point. If there's an allocation 1109 * spinning on this lock, it may win the race 1110 * and cause alloc_contig_range() to fail... 1111 */ 1112 spin_unlock_irqrestore(&z->lock, flags); 1113 ret = __alloc_gigantic_page(pfn, nr_pages); 1114 if (!ret) 1115 return pfn_to_page(pfn); 1116 spin_lock_irqsave(&z->lock, flags); 1117 } 1118 pfn += nr_pages; 1119 } 1120 1121 spin_unlock_irqrestore(&z->lock, flags); 1122 } 1123 1124 return NULL; 1125 } 1126 1127 static void prep_new_huge_page(struct hstate *h, struct page *page, int nid); 1128 static void prep_compound_gigantic_page(struct page *page, unsigned int order); 1129 1130 static struct page *alloc_fresh_gigantic_page_node(struct hstate *h, int nid) 1131 { 1132 struct page *page; 1133 1134 page = alloc_gigantic_page(nid, huge_page_order(h)); 1135 if (page) { 1136 prep_compound_gigantic_page(page, huge_page_order(h)); 1137 prep_new_huge_page(h, page, nid); 1138 } 1139 1140 return page; 1141 } 1142 1143 static int alloc_fresh_gigantic_page(struct hstate *h, 1144 nodemask_t *nodes_allowed) 1145 { 1146 struct page *page = NULL; 1147 int nr_nodes, node; 1148 1149 for_each_node_mask_to_alloc(h, nr_nodes, node, nodes_allowed) { 1150 page = alloc_fresh_gigantic_page_node(h, node); 1151 if (page) 1152 return 1; 1153 } 1154 1155 return 0; 1156 } 1157 1158 static inline bool gigantic_page_supported(void) { return true; } 1159 #else 1160 static inline bool gigantic_page_supported(void) { return false; } 1161 static inline void free_gigantic_page(struct page *page, unsigned int order) { } 1162 static inline void destroy_compound_gigantic_page(struct page *page, 1163 unsigned int order) { } 1164 static inline int alloc_fresh_gigantic_page(struct hstate *h, 1165 nodemask_t *nodes_allowed) { return 0; } 1166 #endif 1167 1168 static void update_and_free_page(struct hstate *h, struct page *page) 1169 { 1170 int i; 1171 1172 if (hstate_is_gigantic(h) && !gigantic_page_supported()) 1173 return; 1174 1175 h->nr_huge_pages--; 1176 h->nr_huge_pages_node[page_to_nid(page)]--; 1177 for (i = 0; i < pages_per_huge_page(h); i++) { 1178 page[i].flags &= ~(1 << PG_locked | 1 << PG_error | 1179 1 << PG_referenced | 1 << PG_dirty | 1180 1 << PG_active | 1 << PG_private | 1181 1 << PG_writeback); 1182 } 1183 VM_BUG_ON_PAGE(hugetlb_cgroup_from_page(page), page); 1184 set_compound_page_dtor(page, NULL_COMPOUND_DTOR); 1185 set_page_refcounted(page); 1186 if (hstate_is_gigantic(h)) { 1187 destroy_compound_gigantic_page(page, huge_page_order(h)); 1188 free_gigantic_page(page, huge_page_order(h)); 1189 } else { 1190 __free_pages(page, huge_page_order(h)); 1191 } 1192 } 1193 1194 struct hstate *size_to_hstate(unsigned long size) 1195 { 1196 struct hstate *h; 1197 1198 for_each_hstate(h) { 1199 if (huge_page_size(h) == size) 1200 return h; 1201 } 1202 return NULL; 1203 } 1204 1205 /* 1206 * Test to determine whether the hugepage is "active/in-use" (i.e. being linked 1207 * to hstate->hugepage_activelist.) 1208 * 1209 * This function can be called for tail pages, but never returns true for them. 1210 */ 1211 bool page_huge_active(struct page *page) 1212 { 1213 VM_BUG_ON_PAGE(!PageHuge(page), page); 1214 return PageHead(page) && PagePrivate(&page[1]); 1215 } 1216 1217 /* never called for tail page */ 1218 static void set_page_huge_active(struct page *page) 1219 { 1220 VM_BUG_ON_PAGE(!PageHeadHuge(page), page); 1221 SetPagePrivate(&page[1]); 1222 } 1223 1224 static void clear_page_huge_active(struct page *page) 1225 { 1226 VM_BUG_ON_PAGE(!PageHeadHuge(page), page); 1227 ClearPagePrivate(&page[1]); 1228 } 1229 1230 void free_huge_page(struct page *page) 1231 { 1232 /* 1233 * Can't pass hstate in here because it is called from the 1234 * compound page destructor. 1235 */ 1236 struct hstate *h = page_hstate(page); 1237 int nid = page_to_nid(page); 1238 struct hugepage_subpool *spool = 1239 (struct hugepage_subpool *)page_private(page); 1240 bool restore_reserve; 1241 1242 set_page_private(page, 0); 1243 page->mapping = NULL; 1244 VM_BUG_ON_PAGE(page_count(page), page); 1245 VM_BUG_ON_PAGE(page_mapcount(page), page); 1246 restore_reserve = PagePrivate(page); 1247 ClearPagePrivate(page); 1248 1249 /* 1250 * A return code of zero implies that the subpool will be under its 1251 * minimum size if the reservation is not restored after page is free. 1252 * Therefore, force restore_reserve operation. 1253 */ 1254 if (hugepage_subpool_put_pages(spool, 1) == 0) 1255 restore_reserve = true; 1256 1257 spin_lock(&hugetlb_lock); 1258 clear_page_huge_active(page); 1259 hugetlb_cgroup_uncharge_page(hstate_index(h), 1260 pages_per_huge_page(h), page); 1261 if (restore_reserve) 1262 h->resv_huge_pages++; 1263 1264 if (h->surplus_huge_pages_node[nid]) { 1265 /* remove the page from active list */ 1266 list_del(&page->lru); 1267 update_and_free_page(h, page); 1268 h->surplus_huge_pages--; 1269 h->surplus_huge_pages_node[nid]--; 1270 } else { 1271 arch_clear_hugepage_flags(page); 1272 enqueue_huge_page(h, page); 1273 } 1274 spin_unlock(&hugetlb_lock); 1275 } 1276 1277 static void prep_new_huge_page(struct hstate *h, struct page *page, int nid) 1278 { 1279 INIT_LIST_HEAD(&page->lru); 1280 set_compound_page_dtor(page, HUGETLB_PAGE_DTOR); 1281 spin_lock(&hugetlb_lock); 1282 set_hugetlb_cgroup(page, NULL); 1283 h->nr_huge_pages++; 1284 h->nr_huge_pages_node[nid]++; 1285 spin_unlock(&hugetlb_lock); 1286 put_page(page); /* free it into the hugepage allocator */ 1287 } 1288 1289 static void prep_compound_gigantic_page(struct page *page, unsigned int order) 1290 { 1291 int i; 1292 int nr_pages = 1 << order; 1293 struct page *p = page + 1; 1294 1295 /* we rely on prep_new_huge_page to set the destructor */ 1296 set_compound_order(page, order); 1297 __ClearPageReserved(page); 1298 __SetPageHead(page); 1299 for (i = 1; i < nr_pages; i++, p = mem_map_next(p, page, i)) { 1300 /* 1301 * For gigantic hugepages allocated through bootmem at 1302 * boot, it's safer to be consistent with the not-gigantic 1303 * hugepages and clear the PG_reserved bit from all tail pages 1304 * too. Otherwse drivers using get_user_pages() to access tail 1305 * pages may get the reference counting wrong if they see 1306 * PG_reserved set on a tail page (despite the head page not 1307 * having PG_reserved set). Enforcing this consistency between 1308 * head and tail pages allows drivers to optimize away a check 1309 * on the head page when they need know if put_page() is needed 1310 * after get_user_pages(). 1311 */ 1312 __ClearPageReserved(p); 1313 set_page_count(p, 0); 1314 set_compound_head(p, page); 1315 } 1316 atomic_set(compound_mapcount_ptr(page), -1); 1317 } 1318 1319 /* 1320 * PageHuge() only returns true for hugetlbfs pages, but not for normal or 1321 * transparent huge pages. See the PageTransHuge() documentation for more 1322 * details. 1323 */ 1324 int PageHuge(struct page *page) 1325 { 1326 if (!PageCompound(page)) 1327 return 0; 1328 1329 page = compound_head(page); 1330 return page[1].compound_dtor == HUGETLB_PAGE_DTOR; 1331 } 1332 EXPORT_SYMBOL_GPL(PageHuge); 1333 1334 /* 1335 * PageHeadHuge() only returns true for hugetlbfs head page, but not for 1336 * normal or transparent huge pages. 1337 */ 1338 int PageHeadHuge(struct page *page_head) 1339 { 1340 if (!PageHead(page_head)) 1341 return 0; 1342 1343 return get_compound_page_dtor(page_head) == free_huge_page; 1344 } 1345 1346 pgoff_t __basepage_index(struct page *page) 1347 { 1348 struct page *page_head = compound_head(page); 1349 pgoff_t index = page_index(page_head); 1350 unsigned long compound_idx; 1351 1352 if (!PageHuge(page_head)) 1353 return page_index(page); 1354 1355 if (compound_order(page_head) >= MAX_ORDER) 1356 compound_idx = page_to_pfn(page) - page_to_pfn(page_head); 1357 else 1358 compound_idx = page - page_head; 1359 1360 return (index << compound_order(page_head)) + compound_idx; 1361 } 1362 1363 static struct page *alloc_fresh_huge_page_node(struct hstate *h, int nid) 1364 { 1365 struct page *page; 1366 1367 page = __alloc_pages_node(nid, 1368 htlb_alloc_mask(h)|__GFP_COMP|__GFP_THISNODE| 1369 __GFP_REPEAT|__GFP_NOWARN, 1370 huge_page_order(h)); 1371 if (page) { 1372 prep_new_huge_page(h, page, nid); 1373 } 1374 1375 return page; 1376 } 1377 1378 static int alloc_fresh_huge_page(struct hstate *h, nodemask_t *nodes_allowed) 1379 { 1380 struct page *page; 1381 int nr_nodes, node; 1382 int ret = 0; 1383 1384 for_each_node_mask_to_alloc(h, nr_nodes, node, nodes_allowed) { 1385 page = alloc_fresh_huge_page_node(h, node); 1386 if (page) { 1387 ret = 1; 1388 break; 1389 } 1390 } 1391 1392 if (ret) 1393 count_vm_event(HTLB_BUDDY_PGALLOC); 1394 else 1395 count_vm_event(HTLB_BUDDY_PGALLOC_FAIL); 1396 1397 return ret; 1398 } 1399 1400 /* 1401 * Free huge page from pool from next node to free. 1402 * Attempt to keep persistent huge pages more or less 1403 * balanced over allowed nodes. 1404 * Called with hugetlb_lock locked. 1405 */ 1406 static int free_pool_huge_page(struct hstate *h, nodemask_t *nodes_allowed, 1407 bool acct_surplus) 1408 { 1409 int nr_nodes, node; 1410 int ret = 0; 1411 1412 for_each_node_mask_to_free(h, nr_nodes, node, nodes_allowed) { 1413 /* 1414 * If we're returning unused surplus pages, only examine 1415 * nodes with surplus pages. 1416 */ 1417 if ((!acct_surplus || h->surplus_huge_pages_node[node]) && 1418 !list_empty(&h->hugepage_freelists[node])) { 1419 struct page *page = 1420 list_entry(h->hugepage_freelists[node].next, 1421 struct page, lru); 1422 list_del(&page->lru); 1423 h->free_huge_pages--; 1424 h->free_huge_pages_node[node]--; 1425 if (acct_surplus) { 1426 h->surplus_huge_pages--; 1427 h->surplus_huge_pages_node[node]--; 1428 } 1429 update_and_free_page(h, page); 1430 ret = 1; 1431 break; 1432 } 1433 } 1434 1435 return ret; 1436 } 1437 1438 /* 1439 * Dissolve a given free hugepage into free buddy pages. This function does 1440 * nothing for in-use (including surplus) hugepages. 1441 */ 1442 static void dissolve_free_huge_page(struct page *page) 1443 { 1444 spin_lock(&hugetlb_lock); 1445 if (PageHuge(page) && !page_count(page)) { 1446 struct hstate *h = page_hstate(page); 1447 int nid = page_to_nid(page); 1448 list_del(&page->lru); 1449 h->free_huge_pages--; 1450 h->free_huge_pages_node[nid]--; 1451 h->max_huge_pages--; 1452 update_and_free_page(h, page); 1453 } 1454 spin_unlock(&hugetlb_lock); 1455 } 1456 1457 /* 1458 * Dissolve free hugepages in a given pfn range. Used by memory hotplug to 1459 * make specified memory blocks removable from the system. 1460 * Note that start_pfn should aligned with (minimum) hugepage size. 1461 */ 1462 void dissolve_free_huge_pages(unsigned long start_pfn, unsigned long end_pfn) 1463 { 1464 unsigned long pfn; 1465 1466 if (!hugepages_supported()) 1467 return; 1468 1469 VM_BUG_ON(!IS_ALIGNED(start_pfn, 1 << minimum_order)); 1470 for (pfn = start_pfn; pfn < end_pfn; pfn += 1 << minimum_order) 1471 dissolve_free_huge_page(pfn_to_page(pfn)); 1472 } 1473 1474 /* 1475 * There are 3 ways this can get called: 1476 * 1. With vma+addr: we use the VMA's memory policy 1477 * 2. With !vma, but nid=NUMA_NO_NODE: We try to allocate a huge 1478 * page from any node, and let the buddy allocator itself figure 1479 * it out. 1480 * 3. With !vma, but nid!=NUMA_NO_NODE. We allocate a huge page 1481 * strictly from 'nid' 1482 */ 1483 static struct page *__hugetlb_alloc_buddy_huge_page(struct hstate *h, 1484 struct vm_area_struct *vma, unsigned long addr, int nid) 1485 { 1486 int order = huge_page_order(h); 1487 gfp_t gfp = htlb_alloc_mask(h)|__GFP_COMP|__GFP_REPEAT|__GFP_NOWARN; 1488 unsigned int cpuset_mems_cookie; 1489 1490 /* 1491 * We need a VMA to get a memory policy. If we do not 1492 * have one, we use the 'nid' argument. 1493 * 1494 * The mempolicy stuff below has some non-inlined bits 1495 * and calls ->vm_ops. That makes it hard to optimize at 1496 * compile-time, even when NUMA is off and it does 1497 * nothing. This helps the compiler optimize it out. 1498 */ 1499 if (!IS_ENABLED(CONFIG_NUMA) || !vma) { 1500 /* 1501 * If a specific node is requested, make sure to 1502 * get memory from there, but only when a node 1503 * is explicitly specified. 1504 */ 1505 if (nid != NUMA_NO_NODE) 1506 gfp |= __GFP_THISNODE; 1507 /* 1508 * Make sure to call something that can handle 1509 * nid=NUMA_NO_NODE 1510 */ 1511 return alloc_pages_node(nid, gfp, order); 1512 } 1513 1514 /* 1515 * OK, so we have a VMA. Fetch the mempolicy and try to 1516 * allocate a huge page with it. We will only reach this 1517 * when CONFIG_NUMA=y. 1518 */ 1519 do { 1520 struct page *page; 1521 struct mempolicy *mpol; 1522 struct zonelist *zl; 1523 nodemask_t *nodemask; 1524 1525 cpuset_mems_cookie = read_mems_allowed_begin(); 1526 zl = huge_zonelist(vma, addr, gfp, &mpol, &nodemask); 1527 mpol_cond_put(mpol); 1528 page = __alloc_pages_nodemask(gfp, order, zl, nodemask); 1529 if (page) 1530 return page; 1531 } while (read_mems_allowed_retry(cpuset_mems_cookie)); 1532 1533 return NULL; 1534 } 1535 1536 /* 1537 * There are two ways to allocate a huge page: 1538 * 1. When you have a VMA and an address (like a fault) 1539 * 2. When you have no VMA (like when setting /proc/.../nr_hugepages) 1540 * 1541 * 'vma' and 'addr' are only for (1). 'nid' is always NUMA_NO_NODE in 1542 * this case which signifies that the allocation should be done with 1543 * respect for the VMA's memory policy. 1544 * 1545 * For (2), we ignore 'vma' and 'addr' and use 'nid' exclusively. This 1546 * implies that memory policies will not be taken in to account. 1547 */ 1548 static struct page *__alloc_buddy_huge_page(struct hstate *h, 1549 struct vm_area_struct *vma, unsigned long addr, int nid) 1550 { 1551 struct page *page; 1552 unsigned int r_nid; 1553 1554 if (hstate_is_gigantic(h)) 1555 return NULL; 1556 1557 /* 1558 * Make sure that anyone specifying 'nid' is not also specifying a VMA. 1559 * This makes sure the caller is picking _one_ of the modes with which 1560 * we can call this function, not both. 1561 */ 1562 if (vma || (addr != -1)) { 1563 VM_WARN_ON_ONCE(addr == -1); 1564 VM_WARN_ON_ONCE(nid != NUMA_NO_NODE); 1565 } 1566 /* 1567 * Assume we will successfully allocate the surplus page to 1568 * prevent racing processes from causing the surplus to exceed 1569 * overcommit 1570 * 1571 * This however introduces a different race, where a process B 1572 * tries to grow the static hugepage pool while alloc_pages() is 1573 * called by process A. B will only examine the per-node 1574 * counters in determining if surplus huge pages can be 1575 * converted to normal huge pages in adjust_pool_surplus(). A 1576 * won't be able to increment the per-node counter, until the 1577 * lock is dropped by B, but B doesn't drop hugetlb_lock until 1578 * no more huge pages can be converted from surplus to normal 1579 * state (and doesn't try to convert again). Thus, we have a 1580 * case where a surplus huge page exists, the pool is grown, and 1581 * the surplus huge page still exists after, even though it 1582 * should just have been converted to a normal huge page. This 1583 * does not leak memory, though, as the hugepage will be freed 1584 * once it is out of use. It also does not allow the counters to 1585 * go out of whack in adjust_pool_surplus() as we don't modify 1586 * the node values until we've gotten the hugepage and only the 1587 * per-node value is checked there. 1588 */ 1589 spin_lock(&hugetlb_lock); 1590 if (h->surplus_huge_pages >= h->nr_overcommit_huge_pages) { 1591 spin_unlock(&hugetlb_lock); 1592 return NULL; 1593 } else { 1594 h->nr_huge_pages++; 1595 h->surplus_huge_pages++; 1596 } 1597 spin_unlock(&hugetlb_lock); 1598 1599 page = __hugetlb_alloc_buddy_huge_page(h, vma, addr, nid); 1600 1601 spin_lock(&hugetlb_lock); 1602 if (page) { 1603 INIT_LIST_HEAD(&page->lru); 1604 r_nid = page_to_nid(page); 1605 set_compound_page_dtor(page, HUGETLB_PAGE_DTOR); 1606 set_hugetlb_cgroup(page, NULL); 1607 /* 1608 * We incremented the global counters already 1609 */ 1610 h->nr_huge_pages_node[r_nid]++; 1611 h->surplus_huge_pages_node[r_nid]++; 1612 __count_vm_event(HTLB_BUDDY_PGALLOC); 1613 } else { 1614 h->nr_huge_pages--; 1615 h->surplus_huge_pages--; 1616 __count_vm_event(HTLB_BUDDY_PGALLOC_FAIL); 1617 } 1618 spin_unlock(&hugetlb_lock); 1619 1620 return page; 1621 } 1622 1623 /* 1624 * Allocate a huge page from 'nid'. Note, 'nid' may be 1625 * NUMA_NO_NODE, which means that it may be allocated 1626 * anywhere. 1627 */ 1628 static 1629 struct page *__alloc_buddy_huge_page_no_mpol(struct hstate *h, int nid) 1630 { 1631 unsigned long addr = -1; 1632 1633 return __alloc_buddy_huge_page(h, NULL, addr, nid); 1634 } 1635 1636 /* 1637 * Use the VMA's mpolicy to allocate a huge page from the buddy. 1638 */ 1639 static 1640 struct page *__alloc_buddy_huge_page_with_mpol(struct hstate *h, 1641 struct vm_area_struct *vma, unsigned long addr) 1642 { 1643 return __alloc_buddy_huge_page(h, vma, addr, NUMA_NO_NODE); 1644 } 1645 1646 /* 1647 * This allocation function is useful in the context where vma is irrelevant. 1648 * E.g. soft-offlining uses this function because it only cares physical 1649 * address of error page. 1650 */ 1651 struct page *alloc_huge_page_node(struct hstate *h, int nid) 1652 { 1653 struct page *page = NULL; 1654 1655 spin_lock(&hugetlb_lock); 1656 if (h->free_huge_pages - h->resv_huge_pages > 0) 1657 page = dequeue_huge_page_node(h, nid); 1658 spin_unlock(&hugetlb_lock); 1659 1660 if (!page) 1661 page = __alloc_buddy_huge_page_no_mpol(h, nid); 1662 1663 return page; 1664 } 1665 1666 /* 1667 * Increase the hugetlb pool such that it can accommodate a reservation 1668 * of size 'delta'. 1669 */ 1670 static int gather_surplus_pages(struct hstate *h, int delta) 1671 { 1672 struct list_head surplus_list; 1673 struct page *page, *tmp; 1674 int ret, i; 1675 int needed, allocated; 1676 bool alloc_ok = true; 1677 1678 needed = (h->resv_huge_pages + delta) - h->free_huge_pages; 1679 if (needed <= 0) { 1680 h->resv_huge_pages += delta; 1681 return 0; 1682 } 1683 1684 allocated = 0; 1685 INIT_LIST_HEAD(&surplus_list); 1686 1687 ret = -ENOMEM; 1688 retry: 1689 spin_unlock(&hugetlb_lock); 1690 for (i = 0; i < needed; i++) { 1691 page = __alloc_buddy_huge_page_no_mpol(h, NUMA_NO_NODE); 1692 if (!page) { 1693 alloc_ok = false; 1694 break; 1695 } 1696 list_add(&page->lru, &surplus_list); 1697 } 1698 allocated += i; 1699 1700 /* 1701 * After retaking hugetlb_lock, we need to recalculate 'needed' 1702 * because either resv_huge_pages or free_huge_pages may have changed. 1703 */ 1704 spin_lock(&hugetlb_lock); 1705 needed = (h->resv_huge_pages + delta) - 1706 (h->free_huge_pages + allocated); 1707 if (needed > 0) { 1708 if (alloc_ok) 1709 goto retry; 1710 /* 1711 * We were not able to allocate enough pages to 1712 * satisfy the entire reservation so we free what 1713 * we've allocated so far. 1714 */ 1715 goto free; 1716 } 1717 /* 1718 * The surplus_list now contains _at_least_ the number of extra pages 1719 * needed to accommodate the reservation. Add the appropriate number 1720 * of pages to the hugetlb pool and free the extras back to the buddy 1721 * allocator. Commit the entire reservation here to prevent another 1722 * process from stealing the pages as they are added to the pool but 1723 * before they are reserved. 1724 */ 1725 needed += allocated; 1726 h->resv_huge_pages += delta; 1727 ret = 0; 1728 1729 /* Free the needed pages to the hugetlb pool */ 1730 list_for_each_entry_safe(page, tmp, &surplus_list, lru) { 1731 if ((--needed) < 0) 1732 break; 1733 /* 1734 * This page is now managed by the hugetlb allocator and has 1735 * no users -- drop the buddy allocator's reference. 1736 */ 1737 put_page_testzero(page); 1738 VM_BUG_ON_PAGE(page_count(page), page); 1739 enqueue_huge_page(h, page); 1740 } 1741 free: 1742 spin_unlock(&hugetlb_lock); 1743 1744 /* Free unnecessary surplus pages to the buddy allocator */ 1745 list_for_each_entry_safe(page, tmp, &surplus_list, lru) 1746 put_page(page); 1747 spin_lock(&hugetlb_lock); 1748 1749 return ret; 1750 } 1751 1752 /* 1753 * When releasing a hugetlb pool reservation, any surplus pages that were 1754 * allocated to satisfy the reservation must be explicitly freed if they were 1755 * never used. 1756 * Called with hugetlb_lock held. 1757 */ 1758 static void return_unused_surplus_pages(struct hstate *h, 1759 unsigned long unused_resv_pages) 1760 { 1761 unsigned long nr_pages; 1762 1763 /* Uncommit the reservation */ 1764 h->resv_huge_pages -= unused_resv_pages; 1765 1766 /* Cannot return gigantic pages currently */ 1767 if (hstate_is_gigantic(h)) 1768 return; 1769 1770 nr_pages = min(unused_resv_pages, h->surplus_huge_pages); 1771 1772 /* 1773 * We want to release as many surplus pages as possible, spread 1774 * evenly across all nodes with memory. Iterate across these nodes 1775 * until we can no longer free unreserved surplus pages. This occurs 1776 * when the nodes with surplus pages have no free pages. 1777 * free_pool_huge_page() will balance the the freed pages across the 1778 * on-line nodes with memory and will handle the hstate accounting. 1779 */ 1780 while (nr_pages--) { 1781 if (!free_pool_huge_page(h, &node_states[N_MEMORY], 1)) 1782 break; 1783 cond_resched_lock(&hugetlb_lock); 1784 } 1785 } 1786 1787 1788 /* 1789 * vma_needs_reservation, vma_commit_reservation and vma_end_reservation 1790 * are used by the huge page allocation routines to manage reservations. 1791 * 1792 * vma_needs_reservation is called to determine if the huge page at addr 1793 * within the vma has an associated reservation. If a reservation is 1794 * needed, the value 1 is returned. The caller is then responsible for 1795 * managing the global reservation and subpool usage counts. After 1796 * the huge page has been allocated, vma_commit_reservation is called 1797 * to add the page to the reservation map. If the page allocation fails, 1798 * the reservation must be ended instead of committed. vma_end_reservation 1799 * is called in such cases. 1800 * 1801 * In the normal case, vma_commit_reservation returns the same value 1802 * as the preceding vma_needs_reservation call. The only time this 1803 * is not the case is if a reserve map was changed between calls. It 1804 * is the responsibility of the caller to notice the difference and 1805 * take appropriate action. 1806 */ 1807 enum vma_resv_mode { 1808 VMA_NEEDS_RESV, 1809 VMA_COMMIT_RESV, 1810 VMA_END_RESV, 1811 }; 1812 static long __vma_reservation_common(struct hstate *h, 1813 struct vm_area_struct *vma, unsigned long addr, 1814 enum vma_resv_mode mode) 1815 { 1816 struct resv_map *resv; 1817 pgoff_t idx; 1818 long ret; 1819 1820 resv = vma_resv_map(vma); 1821 if (!resv) 1822 return 1; 1823 1824 idx = vma_hugecache_offset(h, vma, addr); 1825 switch (mode) { 1826 case VMA_NEEDS_RESV: 1827 ret = region_chg(resv, idx, idx + 1); 1828 break; 1829 case VMA_COMMIT_RESV: 1830 ret = region_add(resv, idx, idx + 1); 1831 break; 1832 case VMA_END_RESV: 1833 region_abort(resv, idx, idx + 1); 1834 ret = 0; 1835 break; 1836 default: 1837 BUG(); 1838 } 1839 1840 if (vma->vm_flags & VM_MAYSHARE) 1841 return ret; 1842 else if (is_vma_resv_set(vma, HPAGE_RESV_OWNER) && ret >= 0) { 1843 /* 1844 * In most cases, reserves always exist for private mappings. 1845 * However, a file associated with mapping could have been 1846 * hole punched or truncated after reserves were consumed. 1847 * As subsequent fault on such a range will not use reserves. 1848 * Subtle - The reserve map for private mappings has the 1849 * opposite meaning than that of shared mappings. If NO 1850 * entry is in the reserve map, it means a reservation exists. 1851 * If an entry exists in the reserve map, it means the 1852 * reservation has already been consumed. As a result, the 1853 * return value of this routine is the opposite of the 1854 * value returned from reserve map manipulation routines above. 1855 */ 1856 if (ret) 1857 return 0; 1858 else 1859 return 1; 1860 } 1861 else 1862 return ret < 0 ? ret : 0; 1863 } 1864 1865 static long vma_needs_reservation(struct hstate *h, 1866 struct vm_area_struct *vma, unsigned long addr) 1867 { 1868 return __vma_reservation_common(h, vma, addr, VMA_NEEDS_RESV); 1869 } 1870 1871 static long vma_commit_reservation(struct hstate *h, 1872 struct vm_area_struct *vma, unsigned long addr) 1873 { 1874 return __vma_reservation_common(h, vma, addr, VMA_COMMIT_RESV); 1875 } 1876 1877 static void vma_end_reservation(struct hstate *h, 1878 struct vm_area_struct *vma, unsigned long addr) 1879 { 1880 (void)__vma_reservation_common(h, vma, addr, VMA_END_RESV); 1881 } 1882 1883 struct page *alloc_huge_page(struct vm_area_struct *vma, 1884 unsigned long addr, int avoid_reserve) 1885 { 1886 struct hugepage_subpool *spool = subpool_vma(vma); 1887 struct hstate *h = hstate_vma(vma); 1888 struct page *page; 1889 long map_chg, map_commit; 1890 long gbl_chg; 1891 int ret, idx; 1892 struct hugetlb_cgroup *h_cg; 1893 1894 idx = hstate_index(h); 1895 /* 1896 * Examine the region/reserve map to determine if the process 1897 * has a reservation for the page to be allocated. A return 1898 * code of zero indicates a reservation exists (no change). 1899 */ 1900 map_chg = gbl_chg = vma_needs_reservation(h, vma, addr); 1901 if (map_chg < 0) 1902 return ERR_PTR(-ENOMEM); 1903 1904 /* 1905 * Processes that did not create the mapping will have no 1906 * reserves as indicated by the region/reserve map. Check 1907 * that the allocation will not exceed the subpool limit. 1908 * Allocations for MAP_NORESERVE mappings also need to be 1909 * checked against any subpool limit. 1910 */ 1911 if (map_chg || avoid_reserve) { 1912 gbl_chg = hugepage_subpool_get_pages(spool, 1); 1913 if (gbl_chg < 0) { 1914 vma_end_reservation(h, vma, addr); 1915 return ERR_PTR(-ENOSPC); 1916 } 1917 1918 /* 1919 * Even though there was no reservation in the region/reserve 1920 * map, there could be reservations associated with the 1921 * subpool that can be used. This would be indicated if the 1922 * return value of hugepage_subpool_get_pages() is zero. 1923 * However, if avoid_reserve is specified we still avoid even 1924 * the subpool reservations. 1925 */ 1926 if (avoid_reserve) 1927 gbl_chg = 1; 1928 } 1929 1930 ret = hugetlb_cgroup_charge_cgroup(idx, pages_per_huge_page(h), &h_cg); 1931 if (ret) 1932 goto out_subpool_put; 1933 1934 spin_lock(&hugetlb_lock); 1935 /* 1936 * glb_chg is passed to indicate whether or not a page must be taken 1937 * from the global free pool (global change). gbl_chg == 0 indicates 1938 * a reservation exists for the allocation. 1939 */ 1940 page = dequeue_huge_page_vma(h, vma, addr, avoid_reserve, gbl_chg); 1941 if (!page) { 1942 spin_unlock(&hugetlb_lock); 1943 page = __alloc_buddy_huge_page_with_mpol(h, vma, addr); 1944 if (!page) 1945 goto out_uncharge_cgroup; 1946 if (!avoid_reserve && vma_has_reserves(vma, gbl_chg)) { 1947 SetPagePrivate(page); 1948 h->resv_huge_pages--; 1949 } 1950 spin_lock(&hugetlb_lock); 1951 list_move(&page->lru, &h->hugepage_activelist); 1952 /* Fall through */ 1953 } 1954 hugetlb_cgroup_commit_charge(idx, pages_per_huge_page(h), h_cg, page); 1955 spin_unlock(&hugetlb_lock); 1956 1957 set_page_private(page, (unsigned long)spool); 1958 1959 map_commit = vma_commit_reservation(h, vma, addr); 1960 if (unlikely(map_chg > map_commit)) { 1961 /* 1962 * The page was added to the reservation map between 1963 * vma_needs_reservation and vma_commit_reservation. 1964 * This indicates a race with hugetlb_reserve_pages. 1965 * Adjust for the subpool count incremented above AND 1966 * in hugetlb_reserve_pages for the same page. Also, 1967 * the reservation count added in hugetlb_reserve_pages 1968 * no longer applies. 1969 */ 1970 long rsv_adjust; 1971 1972 rsv_adjust = hugepage_subpool_put_pages(spool, 1); 1973 hugetlb_acct_memory(h, -rsv_adjust); 1974 } 1975 return page; 1976 1977 out_uncharge_cgroup: 1978 hugetlb_cgroup_uncharge_cgroup(idx, pages_per_huge_page(h), h_cg); 1979 out_subpool_put: 1980 if (map_chg || avoid_reserve) 1981 hugepage_subpool_put_pages(spool, 1); 1982 vma_end_reservation(h, vma, addr); 1983 return ERR_PTR(-ENOSPC); 1984 } 1985 1986 /* 1987 * alloc_huge_page()'s wrapper which simply returns the page if allocation 1988 * succeeds, otherwise NULL. This function is called from new_vma_page(), 1989 * where no ERR_VALUE is expected to be returned. 1990 */ 1991 struct page *alloc_huge_page_noerr(struct vm_area_struct *vma, 1992 unsigned long addr, int avoid_reserve) 1993 { 1994 struct page *page = alloc_huge_page(vma, addr, avoid_reserve); 1995 if (IS_ERR(page)) 1996 page = NULL; 1997 return page; 1998 } 1999 2000 int __weak alloc_bootmem_huge_page(struct hstate *h) 2001 { 2002 struct huge_bootmem_page *m; 2003 int nr_nodes, node; 2004 2005 for_each_node_mask_to_alloc(h, nr_nodes, node, &node_states[N_MEMORY]) { 2006 void *addr; 2007 2008 addr = memblock_virt_alloc_try_nid_nopanic( 2009 huge_page_size(h), huge_page_size(h), 2010 0, BOOTMEM_ALLOC_ACCESSIBLE, node); 2011 if (addr) { 2012 /* 2013 * Use the beginning of the huge page to store the 2014 * huge_bootmem_page struct (until gather_bootmem 2015 * puts them into the mem_map). 2016 */ 2017 m = addr; 2018 goto found; 2019 } 2020 } 2021 return 0; 2022 2023 found: 2024 BUG_ON(!IS_ALIGNED(virt_to_phys(m), huge_page_size(h))); 2025 /* Put them into a private list first because mem_map is not up yet */ 2026 list_add(&m->list, &huge_boot_pages); 2027 m->hstate = h; 2028 return 1; 2029 } 2030 2031 static void __init prep_compound_huge_page(struct page *page, 2032 unsigned int order) 2033 { 2034 if (unlikely(order > (MAX_ORDER - 1))) 2035 prep_compound_gigantic_page(page, order); 2036 else 2037 prep_compound_page(page, order); 2038 } 2039 2040 /* Put bootmem huge pages into the standard lists after mem_map is up */ 2041 static void __init gather_bootmem_prealloc(void) 2042 { 2043 struct huge_bootmem_page *m; 2044 2045 list_for_each_entry(m, &huge_boot_pages, list) { 2046 struct hstate *h = m->hstate; 2047 struct page *page; 2048 2049 #ifdef CONFIG_HIGHMEM 2050 page = pfn_to_page(m->phys >> PAGE_SHIFT); 2051 memblock_free_late(__pa(m), 2052 sizeof(struct huge_bootmem_page)); 2053 #else 2054 page = virt_to_page(m); 2055 #endif 2056 WARN_ON(page_count(page) != 1); 2057 prep_compound_huge_page(page, h->order); 2058 WARN_ON(PageReserved(page)); 2059 prep_new_huge_page(h, page, page_to_nid(page)); 2060 /* 2061 * If we had gigantic hugepages allocated at boot time, we need 2062 * to restore the 'stolen' pages to totalram_pages in order to 2063 * fix confusing memory reports from free(1) and another 2064 * side-effects, like CommitLimit going negative. 2065 */ 2066 if (hstate_is_gigantic(h)) 2067 adjust_managed_page_count(page, 1 << h->order); 2068 } 2069 } 2070 2071 static void __init hugetlb_hstate_alloc_pages(struct hstate *h) 2072 { 2073 unsigned long i; 2074 2075 for (i = 0; i < h->max_huge_pages; ++i) { 2076 if (hstate_is_gigantic(h)) { 2077 if (!alloc_bootmem_huge_page(h)) 2078 break; 2079 } else if (!alloc_fresh_huge_page(h, 2080 &node_states[N_MEMORY])) 2081 break; 2082 } 2083 h->max_huge_pages = i; 2084 } 2085 2086 static void __init hugetlb_init_hstates(void) 2087 { 2088 struct hstate *h; 2089 2090 for_each_hstate(h) { 2091 if (minimum_order > huge_page_order(h)) 2092 minimum_order = huge_page_order(h); 2093 2094 /* oversize hugepages were init'ed in early boot */ 2095 if (!hstate_is_gigantic(h)) 2096 hugetlb_hstate_alloc_pages(h); 2097 } 2098 VM_BUG_ON(minimum_order == UINT_MAX); 2099 } 2100 2101 static char * __init memfmt(char *buf, unsigned long n) 2102 { 2103 if (n >= (1UL << 30)) 2104 sprintf(buf, "%lu GB", n >> 30); 2105 else if (n >= (1UL << 20)) 2106 sprintf(buf, "%lu MB", n >> 20); 2107 else 2108 sprintf(buf, "%lu KB", n >> 10); 2109 return buf; 2110 } 2111 2112 static void __init report_hugepages(void) 2113 { 2114 struct hstate *h; 2115 2116 for_each_hstate(h) { 2117 char buf[32]; 2118 pr_info("HugeTLB registered %s page size, pre-allocated %ld pages\n", 2119 memfmt(buf, huge_page_size(h)), 2120 h->free_huge_pages); 2121 } 2122 } 2123 2124 #ifdef CONFIG_HIGHMEM 2125 static void try_to_free_low(struct hstate *h, unsigned long count, 2126 nodemask_t *nodes_allowed) 2127 { 2128 int i; 2129 2130 if (hstate_is_gigantic(h)) 2131 return; 2132 2133 for_each_node_mask(i, *nodes_allowed) { 2134 struct page *page, *next; 2135 struct list_head *freel = &h->hugepage_freelists[i]; 2136 list_for_each_entry_safe(page, next, freel, lru) { 2137 if (count >= h->nr_huge_pages) 2138 return; 2139 if (PageHighMem(page)) 2140 continue; 2141 list_del(&page->lru); 2142 update_and_free_page(h, page); 2143 h->free_huge_pages--; 2144 h->free_huge_pages_node[page_to_nid(page)]--; 2145 } 2146 } 2147 } 2148 #else 2149 static inline void try_to_free_low(struct hstate *h, unsigned long count, 2150 nodemask_t *nodes_allowed) 2151 { 2152 } 2153 #endif 2154 2155 /* 2156 * Increment or decrement surplus_huge_pages. Keep node-specific counters 2157 * balanced by operating on them in a round-robin fashion. 2158 * Returns 1 if an adjustment was made. 2159 */ 2160 static int adjust_pool_surplus(struct hstate *h, nodemask_t *nodes_allowed, 2161 int delta) 2162 { 2163 int nr_nodes, node; 2164 2165 VM_BUG_ON(delta != -1 && delta != 1); 2166 2167 if (delta < 0) { 2168 for_each_node_mask_to_alloc(h, nr_nodes, node, nodes_allowed) { 2169 if (h->surplus_huge_pages_node[node]) 2170 goto found; 2171 } 2172 } else { 2173 for_each_node_mask_to_free(h, nr_nodes, node, nodes_allowed) { 2174 if (h->surplus_huge_pages_node[node] < 2175 h->nr_huge_pages_node[node]) 2176 goto found; 2177 } 2178 } 2179 return 0; 2180 2181 found: 2182 h->surplus_huge_pages += delta; 2183 h->surplus_huge_pages_node[node] += delta; 2184 return 1; 2185 } 2186 2187 #define persistent_huge_pages(h) (h->nr_huge_pages - h->surplus_huge_pages) 2188 static unsigned long set_max_huge_pages(struct hstate *h, unsigned long count, 2189 nodemask_t *nodes_allowed) 2190 { 2191 unsigned long min_count, ret; 2192 2193 if (hstate_is_gigantic(h) && !gigantic_page_supported()) 2194 return h->max_huge_pages; 2195 2196 /* 2197 * Increase the pool size 2198 * First take pages out of surplus state. Then make up the 2199 * remaining difference by allocating fresh huge pages. 2200 * 2201 * We might race with __alloc_buddy_huge_page() here and be unable 2202 * to convert a surplus huge page to a normal huge page. That is 2203 * not critical, though, it just means the overall size of the 2204 * pool might be one hugepage larger than it needs to be, but 2205 * within all the constraints specified by the sysctls. 2206 */ 2207 spin_lock(&hugetlb_lock); 2208 while (h->surplus_huge_pages && count > persistent_huge_pages(h)) { 2209 if (!adjust_pool_surplus(h, nodes_allowed, -1)) 2210 break; 2211 } 2212 2213 while (count > persistent_huge_pages(h)) { 2214 /* 2215 * If this allocation races such that we no longer need the 2216 * page, free_huge_page will handle it by freeing the page 2217 * and reducing the surplus. 2218 */ 2219 spin_unlock(&hugetlb_lock); 2220 2221 /* yield cpu to avoid soft lockup */ 2222 cond_resched(); 2223 2224 if (hstate_is_gigantic(h)) 2225 ret = alloc_fresh_gigantic_page(h, nodes_allowed); 2226 else 2227 ret = alloc_fresh_huge_page(h, nodes_allowed); 2228 spin_lock(&hugetlb_lock); 2229 if (!ret) 2230 goto out; 2231 2232 /* Bail for signals. Probably ctrl-c from user */ 2233 if (signal_pending(current)) 2234 goto out; 2235 } 2236 2237 /* 2238 * Decrease the pool size 2239 * First return free pages to the buddy allocator (being careful 2240 * to keep enough around to satisfy reservations). Then place 2241 * pages into surplus state as needed so the pool will shrink 2242 * to the desired size as pages become free. 2243 * 2244 * By placing pages into the surplus state independent of the 2245 * overcommit value, we are allowing the surplus pool size to 2246 * exceed overcommit. There are few sane options here. Since 2247 * __alloc_buddy_huge_page() is checking the global counter, 2248 * though, we'll note that we're not allowed to exceed surplus 2249 * and won't grow the pool anywhere else. Not until one of the 2250 * sysctls are changed, or the surplus pages go out of use. 2251 */ 2252 min_count = h->resv_huge_pages + h->nr_huge_pages - h->free_huge_pages; 2253 min_count = max(count, min_count); 2254 try_to_free_low(h, min_count, nodes_allowed); 2255 while (min_count < persistent_huge_pages(h)) { 2256 if (!free_pool_huge_page(h, nodes_allowed, 0)) 2257 break; 2258 cond_resched_lock(&hugetlb_lock); 2259 } 2260 while (count < persistent_huge_pages(h)) { 2261 if (!adjust_pool_surplus(h, nodes_allowed, 1)) 2262 break; 2263 } 2264 out: 2265 ret = persistent_huge_pages(h); 2266 spin_unlock(&hugetlb_lock); 2267 return ret; 2268 } 2269 2270 #define HSTATE_ATTR_RO(_name) \ 2271 static struct kobj_attribute _name##_attr = __ATTR_RO(_name) 2272 2273 #define HSTATE_ATTR(_name) \ 2274 static struct kobj_attribute _name##_attr = \ 2275 __ATTR(_name, 0644, _name##_show, _name##_store) 2276 2277 static struct kobject *hugepages_kobj; 2278 static struct kobject *hstate_kobjs[HUGE_MAX_HSTATE]; 2279 2280 static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp); 2281 2282 static struct hstate *kobj_to_hstate(struct kobject *kobj, int *nidp) 2283 { 2284 int i; 2285 2286 for (i = 0; i < HUGE_MAX_HSTATE; i++) 2287 if (hstate_kobjs[i] == kobj) { 2288 if (nidp) 2289 *nidp = NUMA_NO_NODE; 2290 return &hstates[i]; 2291 } 2292 2293 return kobj_to_node_hstate(kobj, nidp); 2294 } 2295 2296 static ssize_t nr_hugepages_show_common(struct kobject *kobj, 2297 struct kobj_attribute *attr, char *buf) 2298 { 2299 struct hstate *h; 2300 unsigned long nr_huge_pages; 2301 int nid; 2302 2303 h = kobj_to_hstate(kobj, &nid); 2304 if (nid == NUMA_NO_NODE) 2305 nr_huge_pages = h->nr_huge_pages; 2306 else 2307 nr_huge_pages = h->nr_huge_pages_node[nid]; 2308 2309 return sprintf(buf, "%lu\n", nr_huge_pages); 2310 } 2311 2312 static ssize_t __nr_hugepages_store_common(bool obey_mempolicy, 2313 struct hstate *h, int nid, 2314 unsigned long count, size_t len) 2315 { 2316 int err; 2317 NODEMASK_ALLOC(nodemask_t, nodes_allowed, GFP_KERNEL | __GFP_NORETRY); 2318 2319 if (hstate_is_gigantic(h) && !gigantic_page_supported()) { 2320 err = -EINVAL; 2321 goto out; 2322 } 2323 2324 if (nid == NUMA_NO_NODE) { 2325 /* 2326 * global hstate attribute 2327 */ 2328 if (!(obey_mempolicy && 2329 init_nodemask_of_mempolicy(nodes_allowed))) { 2330 NODEMASK_FREE(nodes_allowed); 2331 nodes_allowed = &node_states[N_MEMORY]; 2332 } 2333 } else if (nodes_allowed) { 2334 /* 2335 * per node hstate attribute: adjust count to global, 2336 * but restrict alloc/free to the specified node. 2337 */ 2338 count += h->nr_huge_pages - h->nr_huge_pages_node[nid]; 2339 init_nodemask_of_node(nodes_allowed, nid); 2340 } else 2341 nodes_allowed = &node_states[N_MEMORY]; 2342 2343 h->max_huge_pages = set_max_huge_pages(h, count, nodes_allowed); 2344 2345 if (nodes_allowed != &node_states[N_MEMORY]) 2346 NODEMASK_FREE(nodes_allowed); 2347 2348 return len; 2349 out: 2350 NODEMASK_FREE(nodes_allowed); 2351 return err; 2352 } 2353 2354 static ssize_t nr_hugepages_store_common(bool obey_mempolicy, 2355 struct kobject *kobj, const char *buf, 2356 size_t len) 2357 { 2358 struct hstate *h; 2359 unsigned long count; 2360 int nid; 2361 int err; 2362 2363 err = kstrtoul(buf, 10, &count); 2364 if (err) 2365 return err; 2366 2367 h = kobj_to_hstate(kobj, &nid); 2368 return __nr_hugepages_store_common(obey_mempolicy, h, nid, count, len); 2369 } 2370 2371 static ssize_t nr_hugepages_show(struct kobject *kobj, 2372 struct kobj_attribute *attr, char *buf) 2373 { 2374 return nr_hugepages_show_common(kobj, attr, buf); 2375 } 2376 2377 static ssize_t nr_hugepages_store(struct kobject *kobj, 2378 struct kobj_attribute *attr, const char *buf, size_t len) 2379 { 2380 return nr_hugepages_store_common(false, kobj, buf, len); 2381 } 2382 HSTATE_ATTR(nr_hugepages); 2383 2384 #ifdef CONFIG_NUMA 2385 2386 /* 2387 * hstate attribute for optionally mempolicy-based constraint on persistent 2388 * huge page alloc/free. 2389 */ 2390 static ssize_t nr_hugepages_mempolicy_show(struct kobject *kobj, 2391 struct kobj_attribute *attr, char *buf) 2392 { 2393 return nr_hugepages_show_common(kobj, attr, buf); 2394 } 2395 2396 static ssize_t nr_hugepages_mempolicy_store(struct kobject *kobj, 2397 struct kobj_attribute *attr, const char *buf, size_t len) 2398 { 2399 return nr_hugepages_store_common(true, kobj, buf, len); 2400 } 2401 HSTATE_ATTR(nr_hugepages_mempolicy); 2402 #endif 2403 2404 2405 static ssize_t nr_overcommit_hugepages_show(struct kobject *kobj, 2406 struct kobj_attribute *attr, char *buf) 2407 { 2408 struct hstate *h = kobj_to_hstate(kobj, NULL); 2409 return sprintf(buf, "%lu\n", h->nr_overcommit_huge_pages); 2410 } 2411 2412 static ssize_t nr_overcommit_hugepages_store(struct kobject *kobj, 2413 struct kobj_attribute *attr, const char *buf, size_t count) 2414 { 2415 int err; 2416 unsigned long input; 2417 struct hstate *h = kobj_to_hstate(kobj, NULL); 2418 2419 if (hstate_is_gigantic(h)) 2420 return -EINVAL; 2421 2422 err = kstrtoul(buf, 10, &input); 2423 if (err) 2424 return err; 2425 2426 spin_lock(&hugetlb_lock); 2427 h->nr_overcommit_huge_pages = input; 2428 spin_unlock(&hugetlb_lock); 2429 2430 return count; 2431 } 2432 HSTATE_ATTR(nr_overcommit_hugepages); 2433 2434 static ssize_t free_hugepages_show(struct kobject *kobj, 2435 struct kobj_attribute *attr, char *buf) 2436 { 2437 struct hstate *h; 2438 unsigned long free_huge_pages; 2439 int nid; 2440 2441 h = kobj_to_hstate(kobj, &nid); 2442 if (nid == NUMA_NO_NODE) 2443 free_huge_pages = h->free_huge_pages; 2444 else 2445 free_huge_pages = h->free_huge_pages_node[nid]; 2446 2447 return sprintf(buf, "%lu\n", free_huge_pages); 2448 } 2449 HSTATE_ATTR_RO(free_hugepages); 2450 2451 static ssize_t resv_hugepages_show(struct kobject *kobj, 2452 struct kobj_attribute *attr, char *buf) 2453 { 2454 struct hstate *h = kobj_to_hstate(kobj, NULL); 2455 return sprintf(buf, "%lu\n", h->resv_huge_pages); 2456 } 2457 HSTATE_ATTR_RO(resv_hugepages); 2458 2459 static ssize_t surplus_hugepages_show(struct kobject *kobj, 2460 struct kobj_attribute *attr, char *buf) 2461 { 2462 struct hstate *h; 2463 unsigned long surplus_huge_pages; 2464 int nid; 2465 2466 h = kobj_to_hstate(kobj, &nid); 2467 if (nid == NUMA_NO_NODE) 2468 surplus_huge_pages = h->surplus_huge_pages; 2469 else 2470 surplus_huge_pages = h->surplus_huge_pages_node[nid]; 2471 2472 return sprintf(buf, "%lu\n", surplus_huge_pages); 2473 } 2474 HSTATE_ATTR_RO(surplus_hugepages); 2475 2476 static struct attribute *hstate_attrs[] = { 2477 &nr_hugepages_attr.attr, 2478 &nr_overcommit_hugepages_attr.attr, 2479 &free_hugepages_attr.attr, 2480 &resv_hugepages_attr.attr, 2481 &surplus_hugepages_attr.attr, 2482 #ifdef CONFIG_NUMA 2483 &nr_hugepages_mempolicy_attr.attr, 2484 #endif 2485 NULL, 2486 }; 2487 2488 static struct attribute_group hstate_attr_group = { 2489 .attrs = hstate_attrs, 2490 }; 2491 2492 static int hugetlb_sysfs_add_hstate(struct hstate *h, struct kobject *parent, 2493 struct kobject **hstate_kobjs, 2494 struct attribute_group *hstate_attr_group) 2495 { 2496 int retval; 2497 int hi = hstate_index(h); 2498 2499 hstate_kobjs[hi] = kobject_create_and_add(h->name, parent); 2500 if (!hstate_kobjs[hi]) 2501 return -ENOMEM; 2502 2503 retval = sysfs_create_group(hstate_kobjs[hi], hstate_attr_group); 2504 if (retval) 2505 kobject_put(hstate_kobjs[hi]); 2506 2507 return retval; 2508 } 2509 2510 static void __init hugetlb_sysfs_init(void) 2511 { 2512 struct hstate *h; 2513 int err; 2514 2515 hugepages_kobj = kobject_create_and_add("hugepages", mm_kobj); 2516 if (!hugepages_kobj) 2517 return; 2518 2519 for_each_hstate(h) { 2520 err = hugetlb_sysfs_add_hstate(h, hugepages_kobj, 2521 hstate_kobjs, &hstate_attr_group); 2522 if (err) 2523 pr_err("Hugetlb: Unable to add hstate %s", h->name); 2524 } 2525 } 2526 2527 #ifdef CONFIG_NUMA 2528 2529 /* 2530 * node_hstate/s - associate per node hstate attributes, via their kobjects, 2531 * with node devices in node_devices[] using a parallel array. The array 2532 * index of a node device or _hstate == node id. 2533 * This is here to avoid any static dependency of the node device driver, in 2534 * the base kernel, on the hugetlb module. 2535 */ 2536 struct node_hstate { 2537 struct kobject *hugepages_kobj; 2538 struct kobject *hstate_kobjs[HUGE_MAX_HSTATE]; 2539 }; 2540 static struct node_hstate node_hstates[MAX_NUMNODES]; 2541 2542 /* 2543 * A subset of global hstate attributes for node devices 2544 */ 2545 static struct attribute *per_node_hstate_attrs[] = { 2546 &nr_hugepages_attr.attr, 2547 &free_hugepages_attr.attr, 2548 &surplus_hugepages_attr.attr, 2549 NULL, 2550 }; 2551 2552 static struct attribute_group per_node_hstate_attr_group = { 2553 .attrs = per_node_hstate_attrs, 2554 }; 2555 2556 /* 2557 * kobj_to_node_hstate - lookup global hstate for node device hstate attr kobj. 2558 * Returns node id via non-NULL nidp. 2559 */ 2560 static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp) 2561 { 2562 int nid; 2563 2564 for (nid = 0; nid < nr_node_ids; nid++) { 2565 struct node_hstate *nhs = &node_hstates[nid]; 2566 int i; 2567 for (i = 0; i < HUGE_MAX_HSTATE; i++) 2568 if (nhs->hstate_kobjs[i] == kobj) { 2569 if (nidp) 2570 *nidp = nid; 2571 return &hstates[i]; 2572 } 2573 } 2574 2575 BUG(); 2576 return NULL; 2577 } 2578 2579 /* 2580 * Unregister hstate attributes from a single node device. 2581 * No-op if no hstate attributes attached. 2582 */ 2583 static void hugetlb_unregister_node(struct node *node) 2584 { 2585 struct hstate *h; 2586 struct node_hstate *nhs = &node_hstates[node->dev.id]; 2587 2588 if (!nhs->hugepages_kobj) 2589 return; /* no hstate attributes */ 2590 2591 for_each_hstate(h) { 2592 int idx = hstate_index(h); 2593 if (nhs->hstate_kobjs[idx]) { 2594 kobject_put(nhs->hstate_kobjs[idx]); 2595 nhs->hstate_kobjs[idx] = NULL; 2596 } 2597 } 2598 2599 kobject_put(nhs->hugepages_kobj); 2600 nhs->hugepages_kobj = NULL; 2601 } 2602 2603 2604 /* 2605 * Register hstate attributes for a single node device. 2606 * No-op if attributes already registered. 2607 */ 2608 static void hugetlb_register_node(struct node *node) 2609 { 2610 struct hstate *h; 2611 struct node_hstate *nhs = &node_hstates[node->dev.id]; 2612 int err; 2613 2614 if (nhs->hugepages_kobj) 2615 return; /* already allocated */ 2616 2617 nhs->hugepages_kobj = kobject_create_and_add("hugepages", 2618 &node->dev.kobj); 2619 if (!nhs->hugepages_kobj) 2620 return; 2621 2622 for_each_hstate(h) { 2623 err = hugetlb_sysfs_add_hstate(h, nhs->hugepages_kobj, 2624 nhs->hstate_kobjs, 2625 &per_node_hstate_attr_group); 2626 if (err) { 2627 pr_err("Hugetlb: Unable to add hstate %s for node %d\n", 2628 h->name, node->dev.id); 2629 hugetlb_unregister_node(node); 2630 break; 2631 } 2632 } 2633 } 2634 2635 /* 2636 * hugetlb init time: register hstate attributes for all registered node 2637 * devices of nodes that have memory. All on-line nodes should have 2638 * registered their associated device by this time. 2639 */ 2640 static void __init hugetlb_register_all_nodes(void) 2641 { 2642 int nid; 2643 2644 for_each_node_state(nid, N_MEMORY) { 2645 struct node *node = node_devices[nid]; 2646 if (node->dev.id == nid) 2647 hugetlb_register_node(node); 2648 } 2649 2650 /* 2651 * Let the node device driver know we're here so it can 2652 * [un]register hstate attributes on node hotplug. 2653 */ 2654 register_hugetlbfs_with_node(hugetlb_register_node, 2655 hugetlb_unregister_node); 2656 } 2657 #else /* !CONFIG_NUMA */ 2658 2659 static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp) 2660 { 2661 BUG(); 2662 if (nidp) 2663 *nidp = -1; 2664 return NULL; 2665 } 2666 2667 static void hugetlb_register_all_nodes(void) { } 2668 2669 #endif 2670 2671 static int __init hugetlb_init(void) 2672 { 2673 int i; 2674 2675 if (!hugepages_supported()) 2676 return 0; 2677 2678 if (!size_to_hstate(default_hstate_size)) { 2679 default_hstate_size = HPAGE_SIZE; 2680 if (!size_to_hstate(default_hstate_size)) 2681 hugetlb_add_hstate(HUGETLB_PAGE_ORDER); 2682 } 2683 default_hstate_idx = hstate_index(size_to_hstate(default_hstate_size)); 2684 if (default_hstate_max_huge_pages) { 2685 if (!default_hstate.max_huge_pages) 2686 default_hstate.max_huge_pages = default_hstate_max_huge_pages; 2687 } 2688 2689 hugetlb_init_hstates(); 2690 gather_bootmem_prealloc(); 2691 report_hugepages(); 2692 2693 hugetlb_sysfs_init(); 2694 hugetlb_register_all_nodes(); 2695 hugetlb_cgroup_file_init(); 2696 2697 #ifdef CONFIG_SMP 2698 num_fault_mutexes = roundup_pow_of_two(8 * num_possible_cpus()); 2699 #else 2700 num_fault_mutexes = 1; 2701 #endif 2702 hugetlb_fault_mutex_table = 2703 kmalloc(sizeof(struct mutex) * num_fault_mutexes, GFP_KERNEL); 2704 BUG_ON(!hugetlb_fault_mutex_table); 2705 2706 for (i = 0; i < num_fault_mutexes; i++) 2707 mutex_init(&hugetlb_fault_mutex_table[i]); 2708 return 0; 2709 } 2710 subsys_initcall(hugetlb_init); 2711 2712 /* Should be called on processing a hugepagesz=... option */ 2713 void __init hugetlb_bad_size(void) 2714 { 2715 parsed_valid_hugepagesz = false; 2716 } 2717 2718 void __init hugetlb_add_hstate(unsigned int order) 2719 { 2720 struct hstate *h; 2721 unsigned long i; 2722 2723 if (size_to_hstate(PAGE_SIZE << order)) { 2724 pr_warn("hugepagesz= specified twice, ignoring\n"); 2725 return; 2726 } 2727 BUG_ON(hugetlb_max_hstate >= HUGE_MAX_HSTATE); 2728 BUG_ON(order == 0); 2729 h = &hstates[hugetlb_max_hstate++]; 2730 h->order = order; 2731 h->mask = ~((1ULL << (order + PAGE_SHIFT)) - 1); 2732 h->nr_huge_pages = 0; 2733 h->free_huge_pages = 0; 2734 for (i = 0; i < MAX_NUMNODES; ++i) 2735 INIT_LIST_HEAD(&h->hugepage_freelists[i]); 2736 INIT_LIST_HEAD(&h->hugepage_activelist); 2737 h->next_nid_to_alloc = first_memory_node; 2738 h->next_nid_to_free = first_memory_node; 2739 snprintf(h->name, HSTATE_NAME_LEN, "hugepages-%lukB", 2740 huge_page_size(h)/1024); 2741 2742 parsed_hstate = h; 2743 } 2744 2745 static int __init hugetlb_nrpages_setup(char *s) 2746 { 2747 unsigned long *mhp; 2748 static unsigned long *last_mhp; 2749 2750 if (!parsed_valid_hugepagesz) { 2751 pr_warn("hugepages = %s preceded by " 2752 "an unsupported hugepagesz, ignoring\n", s); 2753 parsed_valid_hugepagesz = true; 2754 return 1; 2755 } 2756 /* 2757 * !hugetlb_max_hstate means we haven't parsed a hugepagesz= parameter yet, 2758 * so this hugepages= parameter goes to the "default hstate". 2759 */ 2760 else if (!hugetlb_max_hstate) 2761 mhp = &default_hstate_max_huge_pages; 2762 else 2763 mhp = &parsed_hstate->max_huge_pages; 2764 2765 if (mhp == last_mhp) { 2766 pr_warn("hugepages= specified twice without interleaving hugepagesz=, ignoring\n"); 2767 return 1; 2768 } 2769 2770 if (sscanf(s, "%lu", mhp) <= 0) 2771 *mhp = 0; 2772 2773 /* 2774 * Global state is always initialized later in hugetlb_init. 2775 * But we need to allocate >= MAX_ORDER hstates here early to still 2776 * use the bootmem allocator. 2777 */ 2778 if (hugetlb_max_hstate && parsed_hstate->order >= MAX_ORDER) 2779 hugetlb_hstate_alloc_pages(parsed_hstate); 2780 2781 last_mhp = mhp; 2782 2783 return 1; 2784 } 2785 __setup("hugepages=", hugetlb_nrpages_setup); 2786 2787 static int __init hugetlb_default_setup(char *s) 2788 { 2789 default_hstate_size = memparse(s, &s); 2790 return 1; 2791 } 2792 __setup("default_hugepagesz=", hugetlb_default_setup); 2793 2794 static unsigned int cpuset_mems_nr(unsigned int *array) 2795 { 2796 int node; 2797 unsigned int nr = 0; 2798 2799 for_each_node_mask(node, cpuset_current_mems_allowed) 2800 nr += array[node]; 2801 2802 return nr; 2803 } 2804 2805 #ifdef CONFIG_SYSCTL 2806 static int hugetlb_sysctl_handler_common(bool obey_mempolicy, 2807 struct ctl_table *table, int write, 2808 void __user *buffer, size_t *length, loff_t *ppos) 2809 { 2810 struct hstate *h = &default_hstate; 2811 unsigned long tmp = h->max_huge_pages; 2812 int ret; 2813 2814 if (!hugepages_supported()) 2815 return -EOPNOTSUPP; 2816 2817 table->data = &tmp; 2818 table->maxlen = sizeof(unsigned long); 2819 ret = proc_doulongvec_minmax(table, write, buffer, length, ppos); 2820 if (ret) 2821 goto out; 2822 2823 if (write) 2824 ret = __nr_hugepages_store_common(obey_mempolicy, h, 2825 NUMA_NO_NODE, tmp, *length); 2826 out: 2827 return ret; 2828 } 2829 2830 int hugetlb_sysctl_handler(struct ctl_table *table, int write, 2831 void __user *buffer, size_t *length, loff_t *ppos) 2832 { 2833 2834 return hugetlb_sysctl_handler_common(false, table, write, 2835 buffer, length, ppos); 2836 } 2837 2838 #ifdef CONFIG_NUMA 2839 int hugetlb_mempolicy_sysctl_handler(struct ctl_table *table, int write, 2840 void __user *buffer, size_t *length, loff_t *ppos) 2841 { 2842 return hugetlb_sysctl_handler_common(true, table, write, 2843 buffer, length, ppos); 2844 } 2845 #endif /* CONFIG_NUMA */ 2846 2847 int hugetlb_overcommit_handler(struct ctl_table *table, int write, 2848 void __user *buffer, 2849 size_t *length, loff_t *ppos) 2850 { 2851 struct hstate *h = &default_hstate; 2852 unsigned long tmp; 2853 int ret; 2854 2855 if (!hugepages_supported()) 2856 return -EOPNOTSUPP; 2857 2858 tmp = h->nr_overcommit_huge_pages; 2859 2860 if (write && hstate_is_gigantic(h)) 2861 return -EINVAL; 2862 2863 table->data = &tmp; 2864 table->maxlen = sizeof(unsigned long); 2865 ret = proc_doulongvec_minmax(table, write, buffer, length, ppos); 2866 if (ret) 2867 goto out; 2868 2869 if (write) { 2870 spin_lock(&hugetlb_lock); 2871 h->nr_overcommit_huge_pages = tmp; 2872 spin_unlock(&hugetlb_lock); 2873 } 2874 out: 2875 return ret; 2876 } 2877 2878 #endif /* CONFIG_SYSCTL */ 2879 2880 void hugetlb_report_meminfo(struct seq_file *m) 2881 { 2882 struct hstate *h = &default_hstate; 2883 if (!hugepages_supported()) 2884 return; 2885 seq_printf(m, 2886 "HugePages_Total: %5lu\n" 2887 "HugePages_Free: %5lu\n" 2888 "HugePages_Rsvd: %5lu\n" 2889 "HugePages_Surp: %5lu\n" 2890 "Hugepagesize: %8lu kB\n", 2891 h->nr_huge_pages, 2892 h->free_huge_pages, 2893 h->resv_huge_pages, 2894 h->surplus_huge_pages, 2895 1UL << (huge_page_order(h) + PAGE_SHIFT - 10)); 2896 } 2897 2898 int hugetlb_report_node_meminfo(int nid, char *buf) 2899 { 2900 struct hstate *h = &default_hstate; 2901 if (!hugepages_supported()) 2902 return 0; 2903 return sprintf(buf, 2904 "Node %d HugePages_Total: %5u\n" 2905 "Node %d HugePages_Free: %5u\n" 2906 "Node %d HugePages_Surp: %5u\n", 2907 nid, h->nr_huge_pages_node[nid], 2908 nid, h->free_huge_pages_node[nid], 2909 nid, h->surplus_huge_pages_node[nid]); 2910 } 2911 2912 void hugetlb_show_meminfo(void) 2913 { 2914 struct hstate *h; 2915 int nid; 2916 2917 if (!hugepages_supported()) 2918 return; 2919 2920 for_each_node_state(nid, N_MEMORY) 2921 for_each_hstate(h) 2922 pr_info("Node %d hugepages_total=%u hugepages_free=%u hugepages_surp=%u hugepages_size=%lukB\n", 2923 nid, 2924 h->nr_huge_pages_node[nid], 2925 h->free_huge_pages_node[nid], 2926 h->surplus_huge_pages_node[nid], 2927 1UL << (huge_page_order(h) + PAGE_SHIFT - 10)); 2928 } 2929 2930 void hugetlb_report_usage(struct seq_file *m, struct mm_struct *mm) 2931 { 2932 seq_printf(m, "HugetlbPages:\t%8lu kB\n", 2933 atomic_long_read(&mm->hugetlb_usage) << (PAGE_SHIFT - 10)); 2934 } 2935 2936 /* Return the number pages of memory we physically have, in PAGE_SIZE units. */ 2937 unsigned long hugetlb_total_pages(void) 2938 { 2939 struct hstate *h; 2940 unsigned long nr_total_pages = 0; 2941 2942 for_each_hstate(h) 2943 nr_total_pages += h->nr_huge_pages * pages_per_huge_page(h); 2944 return nr_total_pages; 2945 } 2946 2947 static int hugetlb_acct_memory(struct hstate *h, long delta) 2948 { 2949 int ret = -ENOMEM; 2950 2951 spin_lock(&hugetlb_lock); 2952 /* 2953 * When cpuset is configured, it breaks the strict hugetlb page 2954 * reservation as the accounting is done on a global variable. Such 2955 * reservation is completely rubbish in the presence of cpuset because 2956 * the reservation is not checked against page availability for the 2957 * current cpuset. Application can still potentially OOM'ed by kernel 2958 * with lack of free htlb page in cpuset that the task is in. 2959 * Attempt to enforce strict accounting with cpuset is almost 2960 * impossible (or too ugly) because cpuset is too fluid that 2961 * task or memory node can be dynamically moved between cpusets. 2962 * 2963 * The change of semantics for shared hugetlb mapping with cpuset is 2964 * undesirable. However, in order to preserve some of the semantics, 2965 * we fall back to check against current free page availability as 2966 * a best attempt and hopefully to minimize the impact of changing 2967 * semantics that cpuset has. 2968 */ 2969 if (delta > 0) { 2970 if (gather_surplus_pages(h, delta) < 0) 2971 goto out; 2972 2973 if (delta > cpuset_mems_nr(h->free_huge_pages_node)) { 2974 return_unused_surplus_pages(h, delta); 2975 goto out; 2976 } 2977 } 2978 2979 ret = 0; 2980 if (delta < 0) 2981 return_unused_surplus_pages(h, (unsigned long) -delta); 2982 2983 out: 2984 spin_unlock(&hugetlb_lock); 2985 return ret; 2986 } 2987 2988 static void hugetlb_vm_op_open(struct vm_area_struct *vma) 2989 { 2990 struct resv_map *resv = vma_resv_map(vma); 2991 2992 /* 2993 * This new VMA should share its siblings reservation map if present. 2994 * The VMA will only ever have a valid reservation map pointer where 2995 * it is being copied for another still existing VMA. As that VMA 2996 * has a reference to the reservation map it cannot disappear until 2997 * after this open call completes. It is therefore safe to take a 2998 * new reference here without additional locking. 2999 */ 3000 if (resv && is_vma_resv_set(vma, HPAGE_RESV_OWNER)) 3001 kref_get(&resv->refs); 3002 } 3003 3004 static void hugetlb_vm_op_close(struct vm_area_struct *vma) 3005 { 3006 struct hstate *h = hstate_vma(vma); 3007 struct resv_map *resv = vma_resv_map(vma); 3008 struct hugepage_subpool *spool = subpool_vma(vma); 3009 unsigned long reserve, start, end; 3010 long gbl_reserve; 3011 3012 if (!resv || !is_vma_resv_set(vma, HPAGE_RESV_OWNER)) 3013 return; 3014 3015 start = vma_hugecache_offset(h, vma, vma->vm_start); 3016 end = vma_hugecache_offset(h, vma, vma->vm_end); 3017 3018 reserve = (end - start) - region_count(resv, start, end); 3019 3020 kref_put(&resv->refs, resv_map_release); 3021 3022 if (reserve) { 3023 /* 3024 * Decrement reserve counts. The global reserve count may be 3025 * adjusted if the subpool has a minimum size. 3026 */ 3027 gbl_reserve = hugepage_subpool_put_pages(spool, reserve); 3028 hugetlb_acct_memory(h, -gbl_reserve); 3029 } 3030 } 3031 3032 /* 3033 * We cannot handle pagefaults against hugetlb pages at all. They cause 3034 * handle_mm_fault() to try to instantiate regular-sized pages in the 3035 * hugegpage VMA. do_page_fault() is supposed to trap this, so BUG is we get 3036 * this far. 3037 */ 3038 static int hugetlb_vm_op_fault(struct vm_area_struct *vma, struct vm_fault *vmf) 3039 { 3040 BUG(); 3041 return 0; 3042 } 3043 3044 const struct vm_operations_struct hugetlb_vm_ops = { 3045 .fault = hugetlb_vm_op_fault, 3046 .open = hugetlb_vm_op_open, 3047 .close = hugetlb_vm_op_close, 3048 }; 3049 3050 static pte_t make_huge_pte(struct vm_area_struct *vma, struct page *page, 3051 int writable) 3052 { 3053 pte_t entry; 3054 3055 if (writable) { 3056 entry = huge_pte_mkwrite(huge_pte_mkdirty(mk_huge_pte(page, 3057 vma->vm_page_prot))); 3058 } else { 3059 entry = huge_pte_wrprotect(mk_huge_pte(page, 3060 vma->vm_page_prot)); 3061 } 3062 entry = pte_mkyoung(entry); 3063 entry = pte_mkhuge(entry); 3064 entry = arch_make_huge_pte(entry, vma, page, writable); 3065 3066 return entry; 3067 } 3068 3069 static void set_huge_ptep_writable(struct vm_area_struct *vma, 3070 unsigned long address, pte_t *ptep) 3071 { 3072 pte_t entry; 3073 3074 entry = huge_pte_mkwrite(huge_pte_mkdirty(huge_ptep_get(ptep))); 3075 if (huge_ptep_set_access_flags(vma, address, ptep, entry, 1)) 3076 update_mmu_cache(vma, address, ptep); 3077 } 3078 3079 static int is_hugetlb_entry_migration(pte_t pte) 3080 { 3081 swp_entry_t swp; 3082 3083 if (huge_pte_none(pte) || pte_present(pte)) 3084 return 0; 3085 swp = pte_to_swp_entry(pte); 3086 if (non_swap_entry(swp) && is_migration_entry(swp)) 3087 return 1; 3088 else 3089 return 0; 3090 } 3091 3092 static int is_hugetlb_entry_hwpoisoned(pte_t pte) 3093 { 3094 swp_entry_t swp; 3095 3096 if (huge_pte_none(pte) || pte_present(pte)) 3097 return 0; 3098 swp = pte_to_swp_entry(pte); 3099 if (non_swap_entry(swp) && is_hwpoison_entry(swp)) 3100 return 1; 3101 else 3102 return 0; 3103 } 3104 3105 int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src, 3106 struct vm_area_struct *vma) 3107 { 3108 pte_t *src_pte, *dst_pte, entry; 3109 struct page *ptepage; 3110 unsigned long addr; 3111 int cow; 3112 struct hstate *h = hstate_vma(vma); 3113 unsigned long sz = huge_page_size(h); 3114 unsigned long mmun_start; /* For mmu_notifiers */ 3115 unsigned long mmun_end; /* For mmu_notifiers */ 3116 int ret = 0; 3117 3118 cow = (vma->vm_flags & (VM_SHARED | VM_MAYWRITE)) == VM_MAYWRITE; 3119 3120 mmun_start = vma->vm_start; 3121 mmun_end = vma->vm_end; 3122 if (cow) 3123 mmu_notifier_invalidate_range_start(src, mmun_start, mmun_end); 3124 3125 for (addr = vma->vm_start; addr < vma->vm_end; addr += sz) { 3126 spinlock_t *src_ptl, *dst_ptl; 3127 src_pte = huge_pte_offset(src, addr); 3128 if (!src_pte) 3129 continue; 3130 dst_pte = huge_pte_alloc(dst, addr, sz); 3131 if (!dst_pte) { 3132 ret = -ENOMEM; 3133 break; 3134 } 3135 3136 /* If the pagetables are shared don't copy or take references */ 3137 if (dst_pte == src_pte) 3138 continue; 3139 3140 dst_ptl = huge_pte_lock(h, dst, dst_pte); 3141 src_ptl = huge_pte_lockptr(h, src, src_pte); 3142 spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING); 3143 entry = huge_ptep_get(src_pte); 3144 if (huge_pte_none(entry)) { /* skip none entry */ 3145 ; 3146 } else if (unlikely(is_hugetlb_entry_migration(entry) || 3147 is_hugetlb_entry_hwpoisoned(entry))) { 3148 swp_entry_t swp_entry = pte_to_swp_entry(entry); 3149 3150 if (is_write_migration_entry(swp_entry) && cow) { 3151 /* 3152 * COW mappings require pages in both 3153 * parent and child to be set to read. 3154 */ 3155 make_migration_entry_read(&swp_entry); 3156 entry = swp_entry_to_pte(swp_entry); 3157 set_huge_pte_at(src, addr, src_pte, entry); 3158 } 3159 set_huge_pte_at(dst, addr, dst_pte, entry); 3160 } else { 3161 if (cow) { 3162 huge_ptep_set_wrprotect(src, addr, src_pte); 3163 mmu_notifier_invalidate_range(src, mmun_start, 3164 mmun_end); 3165 } 3166 entry = huge_ptep_get(src_pte); 3167 ptepage = pte_page(entry); 3168 get_page(ptepage); 3169 page_dup_rmap(ptepage, true); 3170 set_huge_pte_at(dst, addr, dst_pte, entry); 3171 hugetlb_count_add(pages_per_huge_page(h), dst); 3172 } 3173 spin_unlock(src_ptl); 3174 spin_unlock(dst_ptl); 3175 } 3176 3177 if (cow) 3178 mmu_notifier_invalidate_range_end(src, mmun_start, mmun_end); 3179 3180 return ret; 3181 } 3182 3183 void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct *vma, 3184 unsigned long start, unsigned long end, 3185 struct page *ref_page) 3186 { 3187 struct mm_struct *mm = vma->vm_mm; 3188 unsigned long address; 3189 pte_t *ptep; 3190 pte_t pte; 3191 spinlock_t *ptl; 3192 struct page *page; 3193 struct hstate *h = hstate_vma(vma); 3194 unsigned long sz = huge_page_size(h); 3195 const unsigned long mmun_start = start; /* For mmu_notifiers */ 3196 const unsigned long mmun_end = end; /* For mmu_notifiers */ 3197 3198 WARN_ON(!is_vm_hugetlb_page(vma)); 3199 BUG_ON(start & ~huge_page_mask(h)); 3200 BUG_ON(end & ~huge_page_mask(h)); 3201 3202 tlb_start_vma(tlb, vma); 3203 mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end); 3204 address = start; 3205 for (; address < end; address += sz) { 3206 ptep = huge_pte_offset(mm, address); 3207 if (!ptep) 3208 continue; 3209 3210 ptl = huge_pte_lock(h, mm, ptep); 3211 if (huge_pmd_unshare(mm, &address, ptep)) { 3212 spin_unlock(ptl); 3213 continue; 3214 } 3215 3216 pte = huge_ptep_get(ptep); 3217 if (huge_pte_none(pte)) { 3218 spin_unlock(ptl); 3219 continue; 3220 } 3221 3222 /* 3223 * Migrating hugepage or HWPoisoned hugepage is already 3224 * unmapped and its refcount is dropped, so just clear pte here. 3225 */ 3226 if (unlikely(!pte_present(pte))) { 3227 huge_pte_clear(mm, address, ptep); 3228 spin_unlock(ptl); 3229 continue; 3230 } 3231 3232 page = pte_page(pte); 3233 /* 3234 * If a reference page is supplied, it is because a specific 3235 * page is being unmapped, not a range. Ensure the page we 3236 * are about to unmap is the actual page of interest. 3237 */ 3238 if (ref_page) { 3239 if (page != ref_page) { 3240 spin_unlock(ptl); 3241 continue; 3242 } 3243 /* 3244 * Mark the VMA as having unmapped its page so that 3245 * future faults in this VMA will fail rather than 3246 * looking like data was lost 3247 */ 3248 set_vma_resv_flags(vma, HPAGE_RESV_UNMAPPED); 3249 } 3250 3251 pte = huge_ptep_get_and_clear(mm, address, ptep); 3252 tlb_remove_tlb_entry(tlb, ptep, address); 3253 if (huge_pte_dirty(pte)) 3254 set_page_dirty(page); 3255 3256 hugetlb_count_sub(pages_per_huge_page(h), mm); 3257 page_remove_rmap(page, true); 3258 3259 spin_unlock(ptl); 3260 tlb_remove_page_size(tlb, page, huge_page_size(h)); 3261 /* 3262 * Bail out after unmapping reference page if supplied 3263 */ 3264 if (ref_page) 3265 break; 3266 } 3267 mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end); 3268 tlb_end_vma(tlb, vma); 3269 } 3270 3271 void __unmap_hugepage_range_final(struct mmu_gather *tlb, 3272 struct vm_area_struct *vma, unsigned long start, 3273 unsigned long end, struct page *ref_page) 3274 { 3275 __unmap_hugepage_range(tlb, vma, start, end, ref_page); 3276 3277 /* 3278 * Clear this flag so that x86's huge_pmd_share page_table_shareable 3279 * test will fail on a vma being torn down, and not grab a page table 3280 * on its way out. We're lucky that the flag has such an appropriate 3281 * name, and can in fact be safely cleared here. We could clear it 3282 * before the __unmap_hugepage_range above, but all that's necessary 3283 * is to clear it before releasing the i_mmap_rwsem. This works 3284 * because in the context this is called, the VMA is about to be 3285 * destroyed and the i_mmap_rwsem is held. 3286 */ 3287 vma->vm_flags &= ~VM_MAYSHARE; 3288 } 3289 3290 void unmap_hugepage_range(struct vm_area_struct *vma, unsigned long start, 3291 unsigned long end, struct page *ref_page) 3292 { 3293 struct mm_struct *mm; 3294 struct mmu_gather tlb; 3295 3296 mm = vma->vm_mm; 3297 3298 tlb_gather_mmu(&tlb, mm, start, end); 3299 __unmap_hugepage_range(&tlb, vma, start, end, ref_page); 3300 tlb_finish_mmu(&tlb, start, end); 3301 } 3302 3303 /* 3304 * This is called when the original mapper is failing to COW a MAP_PRIVATE 3305 * mappping it owns the reserve page for. The intention is to unmap the page 3306 * from other VMAs and let the children be SIGKILLed if they are faulting the 3307 * same region. 3308 */ 3309 static void unmap_ref_private(struct mm_struct *mm, struct vm_area_struct *vma, 3310 struct page *page, unsigned long address) 3311 { 3312 struct hstate *h = hstate_vma(vma); 3313 struct vm_area_struct *iter_vma; 3314 struct address_space *mapping; 3315 pgoff_t pgoff; 3316 3317 /* 3318 * vm_pgoff is in PAGE_SIZE units, hence the different calculation 3319 * from page cache lookup which is in HPAGE_SIZE units. 3320 */ 3321 address = address & huge_page_mask(h); 3322 pgoff = ((address - vma->vm_start) >> PAGE_SHIFT) + 3323 vma->vm_pgoff; 3324 mapping = vma->vm_file->f_mapping; 3325 3326 /* 3327 * Take the mapping lock for the duration of the table walk. As 3328 * this mapping should be shared between all the VMAs, 3329 * __unmap_hugepage_range() is called as the lock is already held 3330 */ 3331 i_mmap_lock_write(mapping); 3332 vma_interval_tree_foreach(iter_vma, &mapping->i_mmap, pgoff, pgoff) { 3333 /* Do not unmap the current VMA */ 3334 if (iter_vma == vma) 3335 continue; 3336 3337 /* 3338 * Shared VMAs have their own reserves and do not affect 3339 * MAP_PRIVATE accounting but it is possible that a shared 3340 * VMA is using the same page so check and skip such VMAs. 3341 */ 3342 if (iter_vma->vm_flags & VM_MAYSHARE) 3343 continue; 3344 3345 /* 3346 * Unmap the page from other VMAs without their own reserves. 3347 * They get marked to be SIGKILLed if they fault in these 3348 * areas. This is because a future no-page fault on this VMA 3349 * could insert a zeroed page instead of the data existing 3350 * from the time of fork. This would look like data corruption 3351 */ 3352 if (!is_vma_resv_set(iter_vma, HPAGE_RESV_OWNER)) 3353 unmap_hugepage_range(iter_vma, address, 3354 address + huge_page_size(h), page); 3355 } 3356 i_mmap_unlock_write(mapping); 3357 } 3358 3359 /* 3360 * Hugetlb_cow() should be called with page lock of the original hugepage held. 3361 * Called with hugetlb_instantiation_mutex held and pte_page locked so we 3362 * cannot race with other handlers or page migration. 3363 * Keep the pte_same checks anyway to make transition from the mutex easier. 3364 */ 3365 static int hugetlb_cow(struct mm_struct *mm, struct vm_area_struct *vma, 3366 unsigned long address, pte_t *ptep, pte_t pte, 3367 struct page *pagecache_page, spinlock_t *ptl) 3368 { 3369 struct hstate *h = hstate_vma(vma); 3370 struct page *old_page, *new_page; 3371 int ret = 0, outside_reserve = 0; 3372 unsigned long mmun_start; /* For mmu_notifiers */ 3373 unsigned long mmun_end; /* For mmu_notifiers */ 3374 3375 old_page = pte_page(pte); 3376 3377 retry_avoidcopy: 3378 /* If no-one else is actually using this page, avoid the copy 3379 * and just make the page writable */ 3380 if (page_mapcount(old_page) == 1 && PageAnon(old_page)) { 3381 page_move_anon_rmap(old_page, vma); 3382 set_huge_ptep_writable(vma, address, ptep); 3383 return 0; 3384 } 3385 3386 /* 3387 * If the process that created a MAP_PRIVATE mapping is about to 3388 * perform a COW due to a shared page count, attempt to satisfy 3389 * the allocation without using the existing reserves. The pagecache 3390 * page is used to determine if the reserve at this address was 3391 * consumed or not. If reserves were used, a partial faulted mapping 3392 * at the time of fork() could consume its reserves on COW instead 3393 * of the full address range. 3394 */ 3395 if (is_vma_resv_set(vma, HPAGE_RESV_OWNER) && 3396 old_page != pagecache_page) 3397 outside_reserve = 1; 3398 3399 get_page(old_page); 3400 3401 /* 3402 * Drop page table lock as buddy allocator may be called. It will 3403 * be acquired again before returning to the caller, as expected. 3404 */ 3405 spin_unlock(ptl); 3406 new_page = alloc_huge_page(vma, address, outside_reserve); 3407 3408 if (IS_ERR(new_page)) { 3409 /* 3410 * If a process owning a MAP_PRIVATE mapping fails to COW, 3411 * it is due to references held by a child and an insufficient 3412 * huge page pool. To guarantee the original mappers 3413 * reliability, unmap the page from child processes. The child 3414 * may get SIGKILLed if it later faults. 3415 */ 3416 if (outside_reserve) { 3417 put_page(old_page); 3418 BUG_ON(huge_pte_none(pte)); 3419 unmap_ref_private(mm, vma, old_page, address); 3420 BUG_ON(huge_pte_none(pte)); 3421 spin_lock(ptl); 3422 ptep = huge_pte_offset(mm, address & huge_page_mask(h)); 3423 if (likely(ptep && 3424 pte_same(huge_ptep_get(ptep), pte))) 3425 goto retry_avoidcopy; 3426 /* 3427 * race occurs while re-acquiring page table 3428 * lock, and our job is done. 3429 */ 3430 return 0; 3431 } 3432 3433 ret = (PTR_ERR(new_page) == -ENOMEM) ? 3434 VM_FAULT_OOM : VM_FAULT_SIGBUS; 3435 goto out_release_old; 3436 } 3437 3438 /* 3439 * When the original hugepage is shared one, it does not have 3440 * anon_vma prepared. 3441 */ 3442 if (unlikely(anon_vma_prepare(vma))) { 3443 ret = VM_FAULT_OOM; 3444 goto out_release_all; 3445 } 3446 3447 copy_user_huge_page(new_page, old_page, address, vma, 3448 pages_per_huge_page(h)); 3449 __SetPageUptodate(new_page); 3450 set_page_huge_active(new_page); 3451 3452 mmun_start = address & huge_page_mask(h); 3453 mmun_end = mmun_start + huge_page_size(h); 3454 mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end); 3455 3456 /* 3457 * Retake the page table lock to check for racing updates 3458 * before the page tables are altered 3459 */ 3460 spin_lock(ptl); 3461 ptep = huge_pte_offset(mm, address & huge_page_mask(h)); 3462 if (likely(ptep && pte_same(huge_ptep_get(ptep), pte))) { 3463 ClearPagePrivate(new_page); 3464 3465 /* Break COW */ 3466 huge_ptep_clear_flush(vma, address, ptep); 3467 mmu_notifier_invalidate_range(mm, mmun_start, mmun_end); 3468 set_huge_pte_at(mm, address, ptep, 3469 make_huge_pte(vma, new_page, 1)); 3470 page_remove_rmap(old_page, true); 3471 hugepage_add_new_anon_rmap(new_page, vma, address); 3472 /* Make the old page be freed below */ 3473 new_page = old_page; 3474 } 3475 spin_unlock(ptl); 3476 mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end); 3477 out_release_all: 3478 put_page(new_page); 3479 out_release_old: 3480 put_page(old_page); 3481 3482 spin_lock(ptl); /* Caller expects lock to be held */ 3483 return ret; 3484 } 3485 3486 /* Return the pagecache page at a given address within a VMA */ 3487 static struct page *hugetlbfs_pagecache_page(struct hstate *h, 3488 struct vm_area_struct *vma, unsigned long address) 3489 { 3490 struct address_space *mapping; 3491 pgoff_t idx; 3492 3493 mapping = vma->vm_file->f_mapping; 3494 idx = vma_hugecache_offset(h, vma, address); 3495 3496 return find_lock_page(mapping, idx); 3497 } 3498 3499 /* 3500 * Return whether there is a pagecache page to back given address within VMA. 3501 * Caller follow_hugetlb_page() holds page_table_lock so we cannot lock_page. 3502 */ 3503 static bool hugetlbfs_pagecache_present(struct hstate *h, 3504 struct vm_area_struct *vma, unsigned long address) 3505 { 3506 struct address_space *mapping; 3507 pgoff_t idx; 3508 struct page *page; 3509 3510 mapping = vma->vm_file->f_mapping; 3511 idx = vma_hugecache_offset(h, vma, address); 3512 3513 page = find_get_page(mapping, idx); 3514 if (page) 3515 put_page(page); 3516 return page != NULL; 3517 } 3518 3519 int huge_add_to_page_cache(struct page *page, struct address_space *mapping, 3520 pgoff_t idx) 3521 { 3522 struct inode *inode = mapping->host; 3523 struct hstate *h = hstate_inode(inode); 3524 int err = add_to_page_cache(page, mapping, idx, GFP_KERNEL); 3525 3526 if (err) 3527 return err; 3528 ClearPagePrivate(page); 3529 3530 spin_lock(&inode->i_lock); 3531 inode->i_blocks += blocks_per_huge_page(h); 3532 spin_unlock(&inode->i_lock); 3533 return 0; 3534 } 3535 3536 static int hugetlb_no_page(struct mm_struct *mm, struct vm_area_struct *vma, 3537 struct address_space *mapping, pgoff_t idx, 3538 unsigned long address, pte_t *ptep, unsigned int flags) 3539 { 3540 struct hstate *h = hstate_vma(vma); 3541 int ret = VM_FAULT_SIGBUS; 3542 int anon_rmap = 0; 3543 unsigned long size; 3544 struct page *page; 3545 pte_t new_pte; 3546 spinlock_t *ptl; 3547 3548 /* 3549 * Currently, we are forced to kill the process in the event the 3550 * original mapper has unmapped pages from the child due to a failed 3551 * COW. Warn that such a situation has occurred as it may not be obvious 3552 */ 3553 if (is_vma_resv_set(vma, HPAGE_RESV_UNMAPPED)) { 3554 pr_warn_ratelimited("PID %d killed due to inadequate hugepage pool\n", 3555 current->pid); 3556 return ret; 3557 } 3558 3559 /* 3560 * Use page lock to guard against racing truncation 3561 * before we get page_table_lock. 3562 */ 3563 retry: 3564 page = find_lock_page(mapping, idx); 3565 if (!page) { 3566 size = i_size_read(mapping->host) >> huge_page_shift(h); 3567 if (idx >= size) 3568 goto out; 3569 page = alloc_huge_page(vma, address, 0); 3570 if (IS_ERR(page)) { 3571 ret = PTR_ERR(page); 3572 if (ret == -ENOMEM) 3573 ret = VM_FAULT_OOM; 3574 else 3575 ret = VM_FAULT_SIGBUS; 3576 goto out; 3577 } 3578 clear_huge_page(page, address, pages_per_huge_page(h)); 3579 __SetPageUptodate(page); 3580 set_page_huge_active(page); 3581 3582 if (vma->vm_flags & VM_MAYSHARE) { 3583 int err = huge_add_to_page_cache(page, mapping, idx); 3584 if (err) { 3585 put_page(page); 3586 if (err == -EEXIST) 3587 goto retry; 3588 goto out; 3589 } 3590 } else { 3591 lock_page(page); 3592 if (unlikely(anon_vma_prepare(vma))) { 3593 ret = VM_FAULT_OOM; 3594 goto backout_unlocked; 3595 } 3596 anon_rmap = 1; 3597 } 3598 } else { 3599 /* 3600 * If memory error occurs between mmap() and fault, some process 3601 * don't have hwpoisoned swap entry for errored virtual address. 3602 * So we need to block hugepage fault by PG_hwpoison bit check. 3603 */ 3604 if (unlikely(PageHWPoison(page))) { 3605 ret = VM_FAULT_HWPOISON | 3606 VM_FAULT_SET_HINDEX(hstate_index(h)); 3607 goto backout_unlocked; 3608 } 3609 } 3610 3611 /* 3612 * If we are going to COW a private mapping later, we examine the 3613 * pending reservations for this page now. This will ensure that 3614 * any allocations necessary to record that reservation occur outside 3615 * the spinlock. 3616 */ 3617 if ((flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED)) { 3618 if (vma_needs_reservation(h, vma, address) < 0) { 3619 ret = VM_FAULT_OOM; 3620 goto backout_unlocked; 3621 } 3622 /* Just decrements count, does not deallocate */ 3623 vma_end_reservation(h, vma, address); 3624 } 3625 3626 ptl = huge_pte_lockptr(h, mm, ptep); 3627 spin_lock(ptl); 3628 size = i_size_read(mapping->host) >> huge_page_shift(h); 3629 if (idx >= size) 3630 goto backout; 3631 3632 ret = 0; 3633 if (!huge_pte_none(huge_ptep_get(ptep))) 3634 goto backout; 3635 3636 if (anon_rmap) { 3637 ClearPagePrivate(page); 3638 hugepage_add_new_anon_rmap(page, vma, address); 3639 } else 3640 page_dup_rmap(page, true); 3641 new_pte = make_huge_pte(vma, page, ((vma->vm_flags & VM_WRITE) 3642 && (vma->vm_flags & VM_SHARED))); 3643 set_huge_pte_at(mm, address, ptep, new_pte); 3644 3645 hugetlb_count_add(pages_per_huge_page(h), mm); 3646 if ((flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED)) { 3647 /* Optimization, do the COW without a second fault */ 3648 ret = hugetlb_cow(mm, vma, address, ptep, new_pte, page, ptl); 3649 } 3650 3651 spin_unlock(ptl); 3652 unlock_page(page); 3653 out: 3654 return ret; 3655 3656 backout: 3657 spin_unlock(ptl); 3658 backout_unlocked: 3659 unlock_page(page); 3660 put_page(page); 3661 goto out; 3662 } 3663 3664 #ifdef CONFIG_SMP 3665 u32 hugetlb_fault_mutex_hash(struct hstate *h, struct mm_struct *mm, 3666 struct vm_area_struct *vma, 3667 struct address_space *mapping, 3668 pgoff_t idx, unsigned long address) 3669 { 3670 unsigned long key[2]; 3671 u32 hash; 3672 3673 if (vma->vm_flags & VM_SHARED) { 3674 key[0] = (unsigned long) mapping; 3675 key[1] = idx; 3676 } else { 3677 key[0] = (unsigned long) mm; 3678 key[1] = address >> huge_page_shift(h); 3679 } 3680 3681 hash = jhash2((u32 *)&key, sizeof(key)/sizeof(u32), 0); 3682 3683 return hash & (num_fault_mutexes - 1); 3684 } 3685 #else 3686 /* 3687 * For uniprocesor systems we always use a single mutex, so just 3688 * return 0 and avoid the hashing overhead. 3689 */ 3690 u32 hugetlb_fault_mutex_hash(struct hstate *h, struct mm_struct *mm, 3691 struct vm_area_struct *vma, 3692 struct address_space *mapping, 3693 pgoff_t idx, unsigned long address) 3694 { 3695 return 0; 3696 } 3697 #endif 3698 3699 int hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma, 3700 unsigned long address, unsigned int flags) 3701 { 3702 pte_t *ptep, entry; 3703 spinlock_t *ptl; 3704 int ret; 3705 u32 hash; 3706 pgoff_t idx; 3707 struct page *page = NULL; 3708 struct page *pagecache_page = NULL; 3709 struct hstate *h = hstate_vma(vma); 3710 struct address_space *mapping; 3711 int need_wait_lock = 0; 3712 3713 address &= huge_page_mask(h); 3714 3715 ptep = huge_pte_offset(mm, address); 3716 if (ptep) { 3717 entry = huge_ptep_get(ptep); 3718 if (unlikely(is_hugetlb_entry_migration(entry))) { 3719 migration_entry_wait_huge(vma, mm, ptep); 3720 return 0; 3721 } else if (unlikely(is_hugetlb_entry_hwpoisoned(entry))) 3722 return VM_FAULT_HWPOISON_LARGE | 3723 VM_FAULT_SET_HINDEX(hstate_index(h)); 3724 } else { 3725 ptep = huge_pte_alloc(mm, address, huge_page_size(h)); 3726 if (!ptep) 3727 return VM_FAULT_OOM; 3728 } 3729 3730 mapping = vma->vm_file->f_mapping; 3731 idx = vma_hugecache_offset(h, vma, address); 3732 3733 /* 3734 * Serialize hugepage allocation and instantiation, so that we don't 3735 * get spurious allocation failures if two CPUs race to instantiate 3736 * the same page in the page cache. 3737 */ 3738 hash = hugetlb_fault_mutex_hash(h, mm, vma, mapping, idx, address); 3739 mutex_lock(&hugetlb_fault_mutex_table[hash]); 3740 3741 entry = huge_ptep_get(ptep); 3742 if (huge_pte_none(entry)) { 3743 ret = hugetlb_no_page(mm, vma, mapping, idx, address, ptep, flags); 3744 goto out_mutex; 3745 } 3746 3747 ret = 0; 3748 3749 /* 3750 * entry could be a migration/hwpoison entry at this point, so this 3751 * check prevents the kernel from going below assuming that we have 3752 * a active hugepage in pagecache. This goto expects the 2nd page fault, 3753 * and is_hugetlb_entry_(migration|hwpoisoned) check will properly 3754 * handle it. 3755 */ 3756 if (!pte_present(entry)) 3757 goto out_mutex; 3758 3759 /* 3760 * If we are going to COW the mapping later, we examine the pending 3761 * reservations for this page now. This will ensure that any 3762 * allocations necessary to record that reservation occur outside the 3763 * spinlock. For private mappings, we also lookup the pagecache 3764 * page now as it is used to determine if a reservation has been 3765 * consumed. 3766 */ 3767 if ((flags & FAULT_FLAG_WRITE) && !huge_pte_write(entry)) { 3768 if (vma_needs_reservation(h, vma, address) < 0) { 3769 ret = VM_FAULT_OOM; 3770 goto out_mutex; 3771 } 3772 /* Just decrements count, does not deallocate */ 3773 vma_end_reservation(h, vma, address); 3774 3775 if (!(vma->vm_flags & VM_MAYSHARE)) 3776 pagecache_page = hugetlbfs_pagecache_page(h, 3777 vma, address); 3778 } 3779 3780 ptl = huge_pte_lock(h, mm, ptep); 3781 3782 /* Check for a racing update before calling hugetlb_cow */ 3783 if (unlikely(!pte_same(entry, huge_ptep_get(ptep)))) 3784 goto out_ptl; 3785 3786 /* 3787 * hugetlb_cow() requires page locks of pte_page(entry) and 3788 * pagecache_page, so here we need take the former one 3789 * when page != pagecache_page or !pagecache_page. 3790 */ 3791 page = pte_page(entry); 3792 if (page != pagecache_page) 3793 if (!trylock_page(page)) { 3794 need_wait_lock = 1; 3795 goto out_ptl; 3796 } 3797 3798 get_page(page); 3799 3800 if (flags & FAULT_FLAG_WRITE) { 3801 if (!huge_pte_write(entry)) { 3802 ret = hugetlb_cow(mm, vma, address, ptep, entry, 3803 pagecache_page, ptl); 3804 goto out_put_page; 3805 } 3806 entry = huge_pte_mkdirty(entry); 3807 } 3808 entry = pte_mkyoung(entry); 3809 if (huge_ptep_set_access_flags(vma, address, ptep, entry, 3810 flags & FAULT_FLAG_WRITE)) 3811 update_mmu_cache(vma, address, ptep); 3812 out_put_page: 3813 if (page != pagecache_page) 3814 unlock_page(page); 3815 put_page(page); 3816 out_ptl: 3817 spin_unlock(ptl); 3818 3819 if (pagecache_page) { 3820 unlock_page(pagecache_page); 3821 put_page(pagecache_page); 3822 } 3823 out_mutex: 3824 mutex_unlock(&hugetlb_fault_mutex_table[hash]); 3825 /* 3826 * Generally it's safe to hold refcount during waiting page lock. But 3827 * here we just wait to defer the next page fault to avoid busy loop and 3828 * the page is not used after unlocked before returning from the current 3829 * page fault. So we are safe from accessing freed page, even if we wait 3830 * here without taking refcount. 3831 */ 3832 if (need_wait_lock) 3833 wait_on_page_locked(page); 3834 return ret; 3835 } 3836 3837 long follow_hugetlb_page(struct mm_struct *mm, struct vm_area_struct *vma, 3838 struct page **pages, struct vm_area_struct **vmas, 3839 unsigned long *position, unsigned long *nr_pages, 3840 long i, unsigned int flags) 3841 { 3842 unsigned long pfn_offset; 3843 unsigned long vaddr = *position; 3844 unsigned long remainder = *nr_pages; 3845 struct hstate *h = hstate_vma(vma); 3846 3847 while (vaddr < vma->vm_end && remainder) { 3848 pte_t *pte; 3849 spinlock_t *ptl = NULL; 3850 int absent; 3851 struct page *page; 3852 3853 /* 3854 * If we have a pending SIGKILL, don't keep faulting pages and 3855 * potentially allocating memory. 3856 */ 3857 if (unlikely(fatal_signal_pending(current))) { 3858 remainder = 0; 3859 break; 3860 } 3861 3862 /* 3863 * Some archs (sparc64, sh*) have multiple pte_ts to 3864 * each hugepage. We have to make sure we get the 3865 * first, for the page indexing below to work. 3866 * 3867 * Note that page table lock is not held when pte is null. 3868 */ 3869 pte = huge_pte_offset(mm, vaddr & huge_page_mask(h)); 3870 if (pte) 3871 ptl = huge_pte_lock(h, mm, pte); 3872 absent = !pte || huge_pte_none(huge_ptep_get(pte)); 3873 3874 /* 3875 * When coredumping, it suits get_dump_page if we just return 3876 * an error where there's an empty slot with no huge pagecache 3877 * to back it. This way, we avoid allocating a hugepage, and 3878 * the sparse dumpfile avoids allocating disk blocks, but its 3879 * huge holes still show up with zeroes where they need to be. 3880 */ 3881 if (absent && (flags & FOLL_DUMP) && 3882 !hugetlbfs_pagecache_present(h, vma, vaddr)) { 3883 if (pte) 3884 spin_unlock(ptl); 3885 remainder = 0; 3886 break; 3887 } 3888 3889 /* 3890 * We need call hugetlb_fault for both hugepages under migration 3891 * (in which case hugetlb_fault waits for the migration,) and 3892 * hwpoisoned hugepages (in which case we need to prevent the 3893 * caller from accessing to them.) In order to do this, we use 3894 * here is_swap_pte instead of is_hugetlb_entry_migration and 3895 * is_hugetlb_entry_hwpoisoned. This is because it simply covers 3896 * both cases, and because we can't follow correct pages 3897 * directly from any kind of swap entries. 3898 */ 3899 if (absent || is_swap_pte(huge_ptep_get(pte)) || 3900 ((flags & FOLL_WRITE) && 3901 !huge_pte_write(huge_ptep_get(pte)))) { 3902 int ret; 3903 3904 if (pte) 3905 spin_unlock(ptl); 3906 ret = hugetlb_fault(mm, vma, vaddr, 3907 (flags & FOLL_WRITE) ? FAULT_FLAG_WRITE : 0); 3908 if (!(ret & VM_FAULT_ERROR)) 3909 continue; 3910 3911 remainder = 0; 3912 break; 3913 } 3914 3915 pfn_offset = (vaddr & ~huge_page_mask(h)) >> PAGE_SHIFT; 3916 page = pte_page(huge_ptep_get(pte)); 3917 same_page: 3918 if (pages) { 3919 pages[i] = mem_map_offset(page, pfn_offset); 3920 get_page(pages[i]); 3921 } 3922 3923 if (vmas) 3924 vmas[i] = vma; 3925 3926 vaddr += PAGE_SIZE; 3927 ++pfn_offset; 3928 --remainder; 3929 ++i; 3930 if (vaddr < vma->vm_end && remainder && 3931 pfn_offset < pages_per_huge_page(h)) { 3932 /* 3933 * We use pfn_offset to avoid touching the pageframes 3934 * of this compound page. 3935 */ 3936 goto same_page; 3937 } 3938 spin_unlock(ptl); 3939 } 3940 *nr_pages = remainder; 3941 *position = vaddr; 3942 3943 return i ? i : -EFAULT; 3944 } 3945 3946 #ifndef __HAVE_ARCH_FLUSH_HUGETLB_TLB_RANGE 3947 /* 3948 * ARCHes with special requirements for evicting HUGETLB backing TLB entries can 3949 * implement this. 3950 */ 3951 #define flush_hugetlb_tlb_range(vma, addr, end) flush_tlb_range(vma, addr, end) 3952 #endif 3953 3954 unsigned long hugetlb_change_protection(struct vm_area_struct *vma, 3955 unsigned long address, unsigned long end, pgprot_t newprot) 3956 { 3957 struct mm_struct *mm = vma->vm_mm; 3958 unsigned long start = address; 3959 pte_t *ptep; 3960 pte_t pte; 3961 struct hstate *h = hstate_vma(vma); 3962 unsigned long pages = 0; 3963 3964 BUG_ON(address >= end); 3965 flush_cache_range(vma, address, end); 3966 3967 mmu_notifier_invalidate_range_start(mm, start, end); 3968 i_mmap_lock_write(vma->vm_file->f_mapping); 3969 for (; address < end; address += huge_page_size(h)) { 3970 spinlock_t *ptl; 3971 ptep = huge_pte_offset(mm, address); 3972 if (!ptep) 3973 continue; 3974 ptl = huge_pte_lock(h, mm, ptep); 3975 if (huge_pmd_unshare(mm, &address, ptep)) { 3976 pages++; 3977 spin_unlock(ptl); 3978 continue; 3979 } 3980 pte = huge_ptep_get(ptep); 3981 if (unlikely(is_hugetlb_entry_hwpoisoned(pte))) { 3982 spin_unlock(ptl); 3983 continue; 3984 } 3985 if (unlikely(is_hugetlb_entry_migration(pte))) { 3986 swp_entry_t entry = pte_to_swp_entry(pte); 3987 3988 if (is_write_migration_entry(entry)) { 3989 pte_t newpte; 3990 3991 make_migration_entry_read(&entry); 3992 newpte = swp_entry_to_pte(entry); 3993 set_huge_pte_at(mm, address, ptep, newpte); 3994 pages++; 3995 } 3996 spin_unlock(ptl); 3997 continue; 3998 } 3999 if (!huge_pte_none(pte)) { 4000 pte = huge_ptep_get_and_clear(mm, address, ptep); 4001 pte = pte_mkhuge(huge_pte_modify(pte, newprot)); 4002 pte = arch_make_huge_pte(pte, vma, NULL, 0); 4003 set_huge_pte_at(mm, address, ptep, pte); 4004 pages++; 4005 } 4006 spin_unlock(ptl); 4007 } 4008 /* 4009 * Must flush TLB before releasing i_mmap_rwsem: x86's huge_pmd_unshare 4010 * may have cleared our pud entry and done put_page on the page table: 4011 * once we release i_mmap_rwsem, another task can do the final put_page 4012 * and that page table be reused and filled with junk. 4013 */ 4014 flush_hugetlb_tlb_range(vma, start, end); 4015 mmu_notifier_invalidate_range(mm, start, end); 4016 i_mmap_unlock_write(vma->vm_file->f_mapping); 4017 mmu_notifier_invalidate_range_end(mm, start, end); 4018 4019 return pages << h->order; 4020 } 4021 4022 int hugetlb_reserve_pages(struct inode *inode, 4023 long from, long to, 4024 struct vm_area_struct *vma, 4025 vm_flags_t vm_flags) 4026 { 4027 long ret, chg; 4028 struct hstate *h = hstate_inode(inode); 4029 struct hugepage_subpool *spool = subpool_inode(inode); 4030 struct resv_map *resv_map; 4031 long gbl_reserve; 4032 4033 /* 4034 * Only apply hugepage reservation if asked. At fault time, an 4035 * attempt will be made for VM_NORESERVE to allocate a page 4036 * without using reserves 4037 */ 4038 if (vm_flags & VM_NORESERVE) 4039 return 0; 4040 4041 /* 4042 * Shared mappings base their reservation on the number of pages that 4043 * are already allocated on behalf of the file. Private mappings need 4044 * to reserve the full area even if read-only as mprotect() may be 4045 * called to make the mapping read-write. Assume !vma is a shm mapping 4046 */ 4047 if (!vma || vma->vm_flags & VM_MAYSHARE) { 4048 resv_map = inode_resv_map(inode); 4049 4050 chg = region_chg(resv_map, from, to); 4051 4052 } else { 4053 resv_map = resv_map_alloc(); 4054 if (!resv_map) 4055 return -ENOMEM; 4056 4057 chg = to - from; 4058 4059 set_vma_resv_map(vma, resv_map); 4060 set_vma_resv_flags(vma, HPAGE_RESV_OWNER); 4061 } 4062 4063 if (chg < 0) { 4064 ret = chg; 4065 goto out_err; 4066 } 4067 4068 /* 4069 * There must be enough pages in the subpool for the mapping. If 4070 * the subpool has a minimum size, there may be some global 4071 * reservations already in place (gbl_reserve). 4072 */ 4073 gbl_reserve = hugepage_subpool_get_pages(spool, chg); 4074 if (gbl_reserve < 0) { 4075 ret = -ENOSPC; 4076 goto out_err; 4077 } 4078 4079 /* 4080 * Check enough hugepages are available for the reservation. 4081 * Hand the pages back to the subpool if there are not 4082 */ 4083 ret = hugetlb_acct_memory(h, gbl_reserve); 4084 if (ret < 0) { 4085 /* put back original number of pages, chg */ 4086 (void)hugepage_subpool_put_pages(spool, chg); 4087 goto out_err; 4088 } 4089 4090 /* 4091 * Account for the reservations made. Shared mappings record regions 4092 * that have reservations as they are shared by multiple VMAs. 4093 * When the last VMA disappears, the region map says how much 4094 * the reservation was and the page cache tells how much of 4095 * the reservation was consumed. Private mappings are per-VMA and 4096 * only the consumed reservations are tracked. When the VMA 4097 * disappears, the original reservation is the VMA size and the 4098 * consumed reservations are stored in the map. Hence, nothing 4099 * else has to be done for private mappings here 4100 */ 4101 if (!vma || vma->vm_flags & VM_MAYSHARE) { 4102 long add = region_add(resv_map, from, to); 4103 4104 if (unlikely(chg > add)) { 4105 /* 4106 * pages in this range were added to the reserve 4107 * map between region_chg and region_add. This 4108 * indicates a race with alloc_huge_page. Adjust 4109 * the subpool and reserve counts modified above 4110 * based on the difference. 4111 */ 4112 long rsv_adjust; 4113 4114 rsv_adjust = hugepage_subpool_put_pages(spool, 4115 chg - add); 4116 hugetlb_acct_memory(h, -rsv_adjust); 4117 } 4118 } 4119 return 0; 4120 out_err: 4121 if (!vma || vma->vm_flags & VM_MAYSHARE) 4122 region_abort(resv_map, from, to); 4123 if (vma && is_vma_resv_set(vma, HPAGE_RESV_OWNER)) 4124 kref_put(&resv_map->refs, resv_map_release); 4125 return ret; 4126 } 4127 4128 long hugetlb_unreserve_pages(struct inode *inode, long start, long end, 4129 long freed) 4130 { 4131 struct hstate *h = hstate_inode(inode); 4132 struct resv_map *resv_map = inode_resv_map(inode); 4133 long chg = 0; 4134 struct hugepage_subpool *spool = subpool_inode(inode); 4135 long gbl_reserve; 4136 4137 if (resv_map) { 4138 chg = region_del(resv_map, start, end); 4139 /* 4140 * region_del() can fail in the rare case where a region 4141 * must be split and another region descriptor can not be 4142 * allocated. If end == LONG_MAX, it will not fail. 4143 */ 4144 if (chg < 0) 4145 return chg; 4146 } 4147 4148 spin_lock(&inode->i_lock); 4149 inode->i_blocks -= (blocks_per_huge_page(h) * freed); 4150 spin_unlock(&inode->i_lock); 4151 4152 /* 4153 * If the subpool has a minimum size, the number of global 4154 * reservations to be released may be adjusted. 4155 */ 4156 gbl_reserve = hugepage_subpool_put_pages(spool, (chg - freed)); 4157 hugetlb_acct_memory(h, -gbl_reserve); 4158 4159 return 0; 4160 } 4161 4162 #ifdef CONFIG_ARCH_WANT_HUGE_PMD_SHARE 4163 static unsigned long page_table_shareable(struct vm_area_struct *svma, 4164 struct vm_area_struct *vma, 4165 unsigned long addr, pgoff_t idx) 4166 { 4167 unsigned long saddr = ((idx - svma->vm_pgoff) << PAGE_SHIFT) + 4168 svma->vm_start; 4169 unsigned long sbase = saddr & PUD_MASK; 4170 unsigned long s_end = sbase + PUD_SIZE; 4171 4172 /* Allow segments to share if only one is marked locked */ 4173 unsigned long vm_flags = vma->vm_flags & VM_LOCKED_CLEAR_MASK; 4174 unsigned long svm_flags = svma->vm_flags & VM_LOCKED_CLEAR_MASK; 4175 4176 /* 4177 * match the virtual addresses, permission and the alignment of the 4178 * page table page. 4179 */ 4180 if (pmd_index(addr) != pmd_index(saddr) || 4181 vm_flags != svm_flags || 4182 sbase < svma->vm_start || svma->vm_end < s_end) 4183 return 0; 4184 4185 return saddr; 4186 } 4187 4188 static bool vma_shareable(struct vm_area_struct *vma, unsigned long addr) 4189 { 4190 unsigned long base = addr & PUD_MASK; 4191 unsigned long end = base + PUD_SIZE; 4192 4193 /* 4194 * check on proper vm_flags and page table alignment 4195 */ 4196 if (vma->vm_flags & VM_MAYSHARE && 4197 vma->vm_start <= base && end <= vma->vm_end) 4198 return true; 4199 return false; 4200 } 4201 4202 /* 4203 * Search for a shareable pmd page for hugetlb. In any case calls pmd_alloc() 4204 * and returns the corresponding pte. While this is not necessary for the 4205 * !shared pmd case because we can allocate the pmd later as well, it makes the 4206 * code much cleaner. pmd allocation is essential for the shared case because 4207 * pud has to be populated inside the same i_mmap_rwsem section - otherwise 4208 * racing tasks could either miss the sharing (see huge_pte_offset) or select a 4209 * bad pmd for sharing. 4210 */ 4211 pte_t *huge_pmd_share(struct mm_struct *mm, unsigned long addr, pud_t *pud) 4212 { 4213 struct vm_area_struct *vma = find_vma(mm, addr); 4214 struct address_space *mapping = vma->vm_file->f_mapping; 4215 pgoff_t idx = ((addr - vma->vm_start) >> PAGE_SHIFT) + 4216 vma->vm_pgoff; 4217 struct vm_area_struct *svma; 4218 unsigned long saddr; 4219 pte_t *spte = NULL; 4220 pte_t *pte; 4221 spinlock_t *ptl; 4222 4223 if (!vma_shareable(vma, addr)) 4224 return (pte_t *)pmd_alloc(mm, pud, addr); 4225 4226 i_mmap_lock_write(mapping); 4227 vma_interval_tree_foreach(svma, &mapping->i_mmap, idx, idx) { 4228 if (svma == vma) 4229 continue; 4230 4231 saddr = page_table_shareable(svma, vma, addr, idx); 4232 if (saddr) { 4233 spte = huge_pte_offset(svma->vm_mm, saddr); 4234 if (spte) { 4235 get_page(virt_to_page(spte)); 4236 break; 4237 } 4238 } 4239 } 4240 4241 if (!spte) 4242 goto out; 4243 4244 ptl = huge_pte_lockptr(hstate_vma(vma), mm, spte); 4245 spin_lock(ptl); 4246 if (pud_none(*pud)) { 4247 pud_populate(mm, pud, 4248 (pmd_t *)((unsigned long)spte & PAGE_MASK)); 4249 mm_inc_nr_pmds(mm); 4250 } else { 4251 put_page(virt_to_page(spte)); 4252 } 4253 spin_unlock(ptl); 4254 out: 4255 pte = (pte_t *)pmd_alloc(mm, pud, addr); 4256 i_mmap_unlock_write(mapping); 4257 return pte; 4258 } 4259 4260 /* 4261 * unmap huge page backed by shared pte. 4262 * 4263 * Hugetlb pte page is ref counted at the time of mapping. If pte is shared 4264 * indicated by page_count > 1, unmap is achieved by clearing pud and 4265 * decrementing the ref count. If count == 1, the pte page is not shared. 4266 * 4267 * called with page table lock held. 4268 * 4269 * returns: 1 successfully unmapped a shared pte page 4270 * 0 the underlying pte page is not shared, or it is the last user 4271 */ 4272 int huge_pmd_unshare(struct mm_struct *mm, unsigned long *addr, pte_t *ptep) 4273 { 4274 pgd_t *pgd = pgd_offset(mm, *addr); 4275 pud_t *pud = pud_offset(pgd, *addr); 4276 4277 BUG_ON(page_count(virt_to_page(ptep)) == 0); 4278 if (page_count(virt_to_page(ptep)) == 1) 4279 return 0; 4280 4281 pud_clear(pud); 4282 put_page(virt_to_page(ptep)); 4283 mm_dec_nr_pmds(mm); 4284 *addr = ALIGN(*addr, HPAGE_SIZE * PTRS_PER_PTE) - HPAGE_SIZE; 4285 return 1; 4286 } 4287 #define want_pmd_share() (1) 4288 #else /* !CONFIG_ARCH_WANT_HUGE_PMD_SHARE */ 4289 pte_t *huge_pmd_share(struct mm_struct *mm, unsigned long addr, pud_t *pud) 4290 { 4291 return NULL; 4292 } 4293 4294 int huge_pmd_unshare(struct mm_struct *mm, unsigned long *addr, pte_t *ptep) 4295 { 4296 return 0; 4297 } 4298 #define want_pmd_share() (0) 4299 #endif /* CONFIG_ARCH_WANT_HUGE_PMD_SHARE */ 4300 4301 #ifdef CONFIG_ARCH_WANT_GENERAL_HUGETLB 4302 pte_t *huge_pte_alloc(struct mm_struct *mm, 4303 unsigned long addr, unsigned long sz) 4304 { 4305 pgd_t *pgd; 4306 pud_t *pud; 4307 pte_t *pte = NULL; 4308 4309 pgd = pgd_offset(mm, addr); 4310 pud = pud_alloc(mm, pgd, addr); 4311 if (pud) { 4312 if (sz == PUD_SIZE) { 4313 pte = (pte_t *)pud; 4314 } else { 4315 BUG_ON(sz != PMD_SIZE); 4316 if (want_pmd_share() && pud_none(*pud)) 4317 pte = huge_pmd_share(mm, addr, pud); 4318 else 4319 pte = (pte_t *)pmd_alloc(mm, pud, addr); 4320 } 4321 } 4322 BUG_ON(pte && pte_present(*pte) && !pte_huge(*pte)); 4323 4324 return pte; 4325 } 4326 4327 pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr) 4328 { 4329 pgd_t *pgd; 4330 pud_t *pud; 4331 pmd_t *pmd = NULL; 4332 4333 pgd = pgd_offset(mm, addr); 4334 if (pgd_present(*pgd)) { 4335 pud = pud_offset(pgd, addr); 4336 if (pud_present(*pud)) { 4337 if (pud_huge(*pud)) 4338 return (pte_t *)pud; 4339 pmd = pmd_offset(pud, addr); 4340 } 4341 } 4342 return (pte_t *) pmd; 4343 } 4344 4345 #endif /* CONFIG_ARCH_WANT_GENERAL_HUGETLB */ 4346 4347 /* 4348 * These functions are overwritable if your architecture needs its own 4349 * behavior. 4350 */ 4351 struct page * __weak 4352 follow_huge_addr(struct mm_struct *mm, unsigned long address, 4353 int write) 4354 { 4355 return ERR_PTR(-EINVAL); 4356 } 4357 4358 struct page * __weak 4359 follow_huge_pmd(struct mm_struct *mm, unsigned long address, 4360 pmd_t *pmd, int flags) 4361 { 4362 struct page *page = NULL; 4363 spinlock_t *ptl; 4364 retry: 4365 ptl = pmd_lockptr(mm, pmd); 4366 spin_lock(ptl); 4367 /* 4368 * make sure that the address range covered by this pmd is not 4369 * unmapped from other threads. 4370 */ 4371 if (!pmd_huge(*pmd)) 4372 goto out; 4373 if (pmd_present(*pmd)) { 4374 page = pmd_page(*pmd) + ((address & ~PMD_MASK) >> PAGE_SHIFT); 4375 if (flags & FOLL_GET) 4376 get_page(page); 4377 } else { 4378 if (is_hugetlb_entry_migration(huge_ptep_get((pte_t *)pmd))) { 4379 spin_unlock(ptl); 4380 __migration_entry_wait(mm, (pte_t *)pmd, ptl); 4381 goto retry; 4382 } 4383 /* 4384 * hwpoisoned entry is treated as no_page_table in 4385 * follow_page_mask(). 4386 */ 4387 } 4388 out: 4389 spin_unlock(ptl); 4390 return page; 4391 } 4392 4393 struct page * __weak 4394 follow_huge_pud(struct mm_struct *mm, unsigned long address, 4395 pud_t *pud, int flags) 4396 { 4397 if (flags & FOLL_GET) 4398 return NULL; 4399 4400 return pte_page(*(pte_t *)pud) + ((address & ~PUD_MASK) >> PAGE_SHIFT); 4401 } 4402 4403 #ifdef CONFIG_MEMORY_FAILURE 4404 4405 /* 4406 * This function is called from memory failure code. 4407 */ 4408 int dequeue_hwpoisoned_huge_page(struct page *hpage) 4409 { 4410 struct hstate *h = page_hstate(hpage); 4411 int nid = page_to_nid(hpage); 4412 int ret = -EBUSY; 4413 4414 spin_lock(&hugetlb_lock); 4415 /* 4416 * Just checking !page_huge_active is not enough, because that could be 4417 * an isolated/hwpoisoned hugepage (which have >0 refcount). 4418 */ 4419 if (!page_huge_active(hpage) && !page_count(hpage)) { 4420 /* 4421 * Hwpoisoned hugepage isn't linked to activelist or freelist, 4422 * but dangling hpage->lru can trigger list-debug warnings 4423 * (this happens when we call unpoison_memory() on it), 4424 * so let it point to itself with list_del_init(). 4425 */ 4426 list_del_init(&hpage->lru); 4427 set_page_refcounted(hpage); 4428 h->free_huge_pages--; 4429 h->free_huge_pages_node[nid]--; 4430 ret = 0; 4431 } 4432 spin_unlock(&hugetlb_lock); 4433 return ret; 4434 } 4435 #endif 4436 4437 bool isolate_huge_page(struct page *page, struct list_head *list) 4438 { 4439 bool ret = true; 4440 4441 VM_BUG_ON_PAGE(!PageHead(page), page); 4442 spin_lock(&hugetlb_lock); 4443 if (!page_huge_active(page) || !get_page_unless_zero(page)) { 4444 ret = false; 4445 goto unlock; 4446 } 4447 clear_page_huge_active(page); 4448 list_move_tail(&page->lru, list); 4449 unlock: 4450 spin_unlock(&hugetlb_lock); 4451 return ret; 4452 } 4453 4454 void putback_active_hugepage(struct page *page) 4455 { 4456 VM_BUG_ON_PAGE(!PageHead(page), page); 4457 spin_lock(&hugetlb_lock); 4458 set_page_huge_active(page); 4459 list_move_tail(&page->lru, &(page_hstate(page))->hugepage_activelist); 4460 spin_unlock(&hugetlb_lock); 4461 put_page(page); 4462 } 4463