1 /* 2 * Generic hugetlb support. 3 * (C) Nadia Yvette Chambers, April 2004 4 */ 5 #include <linux/list.h> 6 #include <linux/init.h> 7 #include <linux/mm.h> 8 #include <linux/seq_file.h> 9 #include <linux/sysctl.h> 10 #include <linux/highmem.h> 11 #include <linux/mmu_notifier.h> 12 #include <linux/nodemask.h> 13 #include <linux/pagemap.h> 14 #include <linux/mempolicy.h> 15 #include <linux/compiler.h> 16 #include <linux/cpuset.h> 17 #include <linux/mutex.h> 18 #include <linux/bootmem.h> 19 #include <linux/sysfs.h> 20 #include <linux/slab.h> 21 #include <linux/rmap.h> 22 #include <linux/swap.h> 23 #include <linux/swapops.h> 24 #include <linux/page-isolation.h> 25 #include <linux/jhash.h> 26 27 #include <asm/page.h> 28 #include <asm/pgtable.h> 29 #include <asm/tlb.h> 30 31 #include <linux/io.h> 32 #include <linux/hugetlb.h> 33 #include <linux/hugetlb_cgroup.h> 34 #include <linux/node.h> 35 #include "internal.h" 36 37 int hugepages_treat_as_movable; 38 39 int hugetlb_max_hstate __read_mostly; 40 unsigned int default_hstate_idx; 41 struct hstate hstates[HUGE_MAX_HSTATE]; 42 /* 43 * Minimum page order among possible hugepage sizes, set to a proper value 44 * at boot time. 45 */ 46 static unsigned int minimum_order __read_mostly = UINT_MAX; 47 48 __initdata LIST_HEAD(huge_boot_pages); 49 50 /* for command line parsing */ 51 static struct hstate * __initdata parsed_hstate; 52 static unsigned long __initdata default_hstate_max_huge_pages; 53 static unsigned long __initdata default_hstate_size; 54 static bool __initdata parsed_valid_hugepagesz = true; 55 56 /* 57 * Protects updates to hugepage_freelists, hugepage_activelist, nr_huge_pages, 58 * free_huge_pages, and surplus_huge_pages. 59 */ 60 DEFINE_SPINLOCK(hugetlb_lock); 61 62 /* 63 * Serializes faults on the same logical page. This is used to 64 * prevent spurious OOMs when the hugepage pool is fully utilized. 65 */ 66 static int num_fault_mutexes; 67 struct mutex *hugetlb_fault_mutex_table ____cacheline_aligned_in_smp; 68 69 /* Forward declaration */ 70 static int hugetlb_acct_memory(struct hstate *h, long delta); 71 72 static inline void unlock_or_release_subpool(struct hugepage_subpool *spool) 73 { 74 bool free = (spool->count == 0) && (spool->used_hpages == 0); 75 76 spin_unlock(&spool->lock); 77 78 /* If no pages are used, and no other handles to the subpool 79 * remain, give up any reservations mased on minimum size and 80 * free the subpool */ 81 if (free) { 82 if (spool->min_hpages != -1) 83 hugetlb_acct_memory(spool->hstate, 84 -spool->min_hpages); 85 kfree(spool); 86 } 87 } 88 89 struct hugepage_subpool *hugepage_new_subpool(struct hstate *h, long max_hpages, 90 long min_hpages) 91 { 92 struct hugepage_subpool *spool; 93 94 spool = kzalloc(sizeof(*spool), GFP_KERNEL); 95 if (!spool) 96 return NULL; 97 98 spin_lock_init(&spool->lock); 99 spool->count = 1; 100 spool->max_hpages = max_hpages; 101 spool->hstate = h; 102 spool->min_hpages = min_hpages; 103 104 if (min_hpages != -1 && hugetlb_acct_memory(h, min_hpages)) { 105 kfree(spool); 106 return NULL; 107 } 108 spool->rsv_hpages = min_hpages; 109 110 return spool; 111 } 112 113 void hugepage_put_subpool(struct hugepage_subpool *spool) 114 { 115 spin_lock(&spool->lock); 116 BUG_ON(!spool->count); 117 spool->count--; 118 unlock_or_release_subpool(spool); 119 } 120 121 /* 122 * Subpool accounting for allocating and reserving pages. 123 * Return -ENOMEM if there are not enough resources to satisfy the 124 * the request. Otherwise, return the number of pages by which the 125 * global pools must be adjusted (upward). The returned value may 126 * only be different than the passed value (delta) in the case where 127 * a subpool minimum size must be manitained. 128 */ 129 static long hugepage_subpool_get_pages(struct hugepage_subpool *spool, 130 long delta) 131 { 132 long ret = delta; 133 134 if (!spool) 135 return ret; 136 137 spin_lock(&spool->lock); 138 139 if (spool->max_hpages != -1) { /* maximum size accounting */ 140 if ((spool->used_hpages + delta) <= spool->max_hpages) 141 spool->used_hpages += delta; 142 else { 143 ret = -ENOMEM; 144 goto unlock_ret; 145 } 146 } 147 148 /* minimum size accounting */ 149 if (spool->min_hpages != -1 && spool->rsv_hpages) { 150 if (delta > spool->rsv_hpages) { 151 /* 152 * Asking for more reserves than those already taken on 153 * behalf of subpool. Return difference. 154 */ 155 ret = delta - spool->rsv_hpages; 156 spool->rsv_hpages = 0; 157 } else { 158 ret = 0; /* reserves already accounted for */ 159 spool->rsv_hpages -= delta; 160 } 161 } 162 163 unlock_ret: 164 spin_unlock(&spool->lock); 165 return ret; 166 } 167 168 /* 169 * Subpool accounting for freeing and unreserving pages. 170 * Return the number of global page reservations that must be dropped. 171 * The return value may only be different than the passed value (delta) 172 * in the case where a subpool minimum size must be maintained. 173 */ 174 static long hugepage_subpool_put_pages(struct hugepage_subpool *spool, 175 long delta) 176 { 177 long ret = delta; 178 179 if (!spool) 180 return delta; 181 182 spin_lock(&spool->lock); 183 184 if (spool->max_hpages != -1) /* maximum size accounting */ 185 spool->used_hpages -= delta; 186 187 /* minimum size accounting */ 188 if (spool->min_hpages != -1 && spool->used_hpages < spool->min_hpages) { 189 if (spool->rsv_hpages + delta <= spool->min_hpages) 190 ret = 0; 191 else 192 ret = spool->rsv_hpages + delta - spool->min_hpages; 193 194 spool->rsv_hpages += delta; 195 if (spool->rsv_hpages > spool->min_hpages) 196 spool->rsv_hpages = spool->min_hpages; 197 } 198 199 /* 200 * If hugetlbfs_put_super couldn't free spool due to an outstanding 201 * quota reference, free it now. 202 */ 203 unlock_or_release_subpool(spool); 204 205 return ret; 206 } 207 208 static inline struct hugepage_subpool *subpool_inode(struct inode *inode) 209 { 210 return HUGETLBFS_SB(inode->i_sb)->spool; 211 } 212 213 static inline struct hugepage_subpool *subpool_vma(struct vm_area_struct *vma) 214 { 215 return subpool_inode(file_inode(vma->vm_file)); 216 } 217 218 /* 219 * Region tracking -- allows tracking of reservations and instantiated pages 220 * across the pages in a mapping. 221 * 222 * The region data structures are embedded into a resv_map and protected 223 * by a resv_map's lock. The set of regions within the resv_map represent 224 * reservations for huge pages, or huge pages that have already been 225 * instantiated within the map. The from and to elements are huge page 226 * indicies into the associated mapping. from indicates the starting index 227 * of the region. to represents the first index past the end of the region. 228 * 229 * For example, a file region structure with from == 0 and to == 4 represents 230 * four huge pages in a mapping. It is important to note that the to element 231 * represents the first element past the end of the region. This is used in 232 * arithmetic as 4(to) - 0(from) = 4 huge pages in the region. 233 * 234 * Interval notation of the form [from, to) will be used to indicate that 235 * the endpoint from is inclusive and to is exclusive. 236 */ 237 struct file_region { 238 struct list_head link; 239 long from; 240 long to; 241 }; 242 243 /* 244 * Add the huge page range represented by [f, t) to the reserve 245 * map. In the normal case, existing regions will be expanded 246 * to accommodate the specified range. Sufficient regions should 247 * exist for expansion due to the previous call to region_chg 248 * with the same range. However, it is possible that region_del 249 * could have been called after region_chg and modifed the map 250 * in such a way that no region exists to be expanded. In this 251 * case, pull a region descriptor from the cache associated with 252 * the map and use that for the new range. 253 * 254 * Return the number of new huge pages added to the map. This 255 * number is greater than or equal to zero. 256 */ 257 static long region_add(struct resv_map *resv, long f, long t) 258 { 259 struct list_head *head = &resv->regions; 260 struct file_region *rg, *nrg, *trg; 261 long add = 0; 262 263 spin_lock(&resv->lock); 264 /* Locate the region we are either in or before. */ 265 list_for_each_entry(rg, head, link) 266 if (f <= rg->to) 267 break; 268 269 /* 270 * If no region exists which can be expanded to include the 271 * specified range, the list must have been modified by an 272 * interleving call to region_del(). Pull a region descriptor 273 * from the cache and use it for this range. 274 */ 275 if (&rg->link == head || t < rg->from) { 276 VM_BUG_ON(resv->region_cache_count <= 0); 277 278 resv->region_cache_count--; 279 nrg = list_first_entry(&resv->region_cache, struct file_region, 280 link); 281 list_del(&nrg->link); 282 283 nrg->from = f; 284 nrg->to = t; 285 list_add(&nrg->link, rg->link.prev); 286 287 add += t - f; 288 goto out_locked; 289 } 290 291 /* Round our left edge to the current segment if it encloses us. */ 292 if (f > rg->from) 293 f = rg->from; 294 295 /* Check for and consume any regions we now overlap with. */ 296 nrg = rg; 297 list_for_each_entry_safe(rg, trg, rg->link.prev, link) { 298 if (&rg->link == head) 299 break; 300 if (rg->from > t) 301 break; 302 303 /* If this area reaches higher then extend our area to 304 * include it completely. If this is not the first area 305 * which we intend to reuse, free it. */ 306 if (rg->to > t) 307 t = rg->to; 308 if (rg != nrg) { 309 /* Decrement return value by the deleted range. 310 * Another range will span this area so that by 311 * end of routine add will be >= zero 312 */ 313 add -= (rg->to - rg->from); 314 list_del(&rg->link); 315 kfree(rg); 316 } 317 } 318 319 add += (nrg->from - f); /* Added to beginning of region */ 320 nrg->from = f; 321 add += t - nrg->to; /* Added to end of region */ 322 nrg->to = t; 323 324 out_locked: 325 resv->adds_in_progress--; 326 spin_unlock(&resv->lock); 327 VM_BUG_ON(add < 0); 328 return add; 329 } 330 331 /* 332 * Examine the existing reserve map and determine how many 333 * huge pages in the specified range [f, t) are NOT currently 334 * represented. This routine is called before a subsequent 335 * call to region_add that will actually modify the reserve 336 * map to add the specified range [f, t). region_chg does 337 * not change the number of huge pages represented by the 338 * map. However, if the existing regions in the map can not 339 * be expanded to represent the new range, a new file_region 340 * structure is added to the map as a placeholder. This is 341 * so that the subsequent region_add call will have all the 342 * regions it needs and will not fail. 343 * 344 * Upon entry, region_chg will also examine the cache of region descriptors 345 * associated with the map. If there are not enough descriptors cached, one 346 * will be allocated for the in progress add operation. 347 * 348 * Returns the number of huge pages that need to be added to the existing 349 * reservation map for the range [f, t). This number is greater or equal to 350 * zero. -ENOMEM is returned if a new file_region structure or cache entry 351 * is needed and can not be allocated. 352 */ 353 static long region_chg(struct resv_map *resv, long f, long t) 354 { 355 struct list_head *head = &resv->regions; 356 struct file_region *rg, *nrg = NULL; 357 long chg = 0; 358 359 retry: 360 spin_lock(&resv->lock); 361 retry_locked: 362 resv->adds_in_progress++; 363 364 /* 365 * Check for sufficient descriptors in the cache to accommodate 366 * the number of in progress add operations. 367 */ 368 if (resv->adds_in_progress > resv->region_cache_count) { 369 struct file_region *trg; 370 371 VM_BUG_ON(resv->adds_in_progress - resv->region_cache_count > 1); 372 /* Must drop lock to allocate a new descriptor. */ 373 resv->adds_in_progress--; 374 spin_unlock(&resv->lock); 375 376 trg = kmalloc(sizeof(*trg), GFP_KERNEL); 377 if (!trg) { 378 kfree(nrg); 379 return -ENOMEM; 380 } 381 382 spin_lock(&resv->lock); 383 list_add(&trg->link, &resv->region_cache); 384 resv->region_cache_count++; 385 goto retry_locked; 386 } 387 388 /* Locate the region we are before or in. */ 389 list_for_each_entry(rg, head, link) 390 if (f <= rg->to) 391 break; 392 393 /* If we are below the current region then a new region is required. 394 * Subtle, allocate a new region at the position but make it zero 395 * size such that we can guarantee to record the reservation. */ 396 if (&rg->link == head || t < rg->from) { 397 if (!nrg) { 398 resv->adds_in_progress--; 399 spin_unlock(&resv->lock); 400 nrg = kmalloc(sizeof(*nrg), GFP_KERNEL); 401 if (!nrg) 402 return -ENOMEM; 403 404 nrg->from = f; 405 nrg->to = f; 406 INIT_LIST_HEAD(&nrg->link); 407 goto retry; 408 } 409 410 list_add(&nrg->link, rg->link.prev); 411 chg = t - f; 412 goto out_nrg; 413 } 414 415 /* Round our left edge to the current segment if it encloses us. */ 416 if (f > rg->from) 417 f = rg->from; 418 chg = t - f; 419 420 /* Check for and consume any regions we now overlap with. */ 421 list_for_each_entry(rg, rg->link.prev, link) { 422 if (&rg->link == head) 423 break; 424 if (rg->from > t) 425 goto out; 426 427 /* We overlap with this area, if it extends further than 428 * us then we must extend ourselves. Account for its 429 * existing reservation. */ 430 if (rg->to > t) { 431 chg += rg->to - t; 432 t = rg->to; 433 } 434 chg -= rg->to - rg->from; 435 } 436 437 out: 438 spin_unlock(&resv->lock); 439 /* We already know we raced and no longer need the new region */ 440 kfree(nrg); 441 return chg; 442 out_nrg: 443 spin_unlock(&resv->lock); 444 return chg; 445 } 446 447 /* 448 * Abort the in progress add operation. The adds_in_progress field 449 * of the resv_map keeps track of the operations in progress between 450 * calls to region_chg and region_add. Operations are sometimes 451 * aborted after the call to region_chg. In such cases, region_abort 452 * is called to decrement the adds_in_progress counter. 453 * 454 * NOTE: The range arguments [f, t) are not needed or used in this 455 * routine. They are kept to make reading the calling code easier as 456 * arguments will match the associated region_chg call. 457 */ 458 static void region_abort(struct resv_map *resv, long f, long t) 459 { 460 spin_lock(&resv->lock); 461 VM_BUG_ON(!resv->region_cache_count); 462 resv->adds_in_progress--; 463 spin_unlock(&resv->lock); 464 } 465 466 /* 467 * Delete the specified range [f, t) from the reserve map. If the 468 * t parameter is LONG_MAX, this indicates that ALL regions after f 469 * should be deleted. Locate the regions which intersect [f, t) 470 * and either trim, delete or split the existing regions. 471 * 472 * Returns the number of huge pages deleted from the reserve map. 473 * In the normal case, the return value is zero or more. In the 474 * case where a region must be split, a new region descriptor must 475 * be allocated. If the allocation fails, -ENOMEM will be returned. 476 * NOTE: If the parameter t == LONG_MAX, then we will never split 477 * a region and possibly return -ENOMEM. Callers specifying 478 * t == LONG_MAX do not need to check for -ENOMEM error. 479 */ 480 static long region_del(struct resv_map *resv, long f, long t) 481 { 482 struct list_head *head = &resv->regions; 483 struct file_region *rg, *trg; 484 struct file_region *nrg = NULL; 485 long del = 0; 486 487 retry: 488 spin_lock(&resv->lock); 489 list_for_each_entry_safe(rg, trg, head, link) { 490 /* 491 * Skip regions before the range to be deleted. file_region 492 * ranges are normally of the form [from, to). However, there 493 * may be a "placeholder" entry in the map which is of the form 494 * (from, to) with from == to. Check for placeholder entries 495 * at the beginning of the range to be deleted. 496 */ 497 if (rg->to <= f && (rg->to != rg->from || rg->to != f)) 498 continue; 499 500 if (rg->from >= t) 501 break; 502 503 if (f > rg->from && t < rg->to) { /* Must split region */ 504 /* 505 * Check for an entry in the cache before dropping 506 * lock and attempting allocation. 507 */ 508 if (!nrg && 509 resv->region_cache_count > resv->adds_in_progress) { 510 nrg = list_first_entry(&resv->region_cache, 511 struct file_region, 512 link); 513 list_del(&nrg->link); 514 resv->region_cache_count--; 515 } 516 517 if (!nrg) { 518 spin_unlock(&resv->lock); 519 nrg = kmalloc(sizeof(*nrg), GFP_KERNEL); 520 if (!nrg) 521 return -ENOMEM; 522 goto retry; 523 } 524 525 del += t - f; 526 527 /* New entry for end of split region */ 528 nrg->from = t; 529 nrg->to = rg->to; 530 INIT_LIST_HEAD(&nrg->link); 531 532 /* Original entry is trimmed */ 533 rg->to = f; 534 535 list_add(&nrg->link, &rg->link); 536 nrg = NULL; 537 break; 538 } 539 540 if (f <= rg->from && t >= rg->to) { /* Remove entire region */ 541 del += rg->to - rg->from; 542 list_del(&rg->link); 543 kfree(rg); 544 continue; 545 } 546 547 if (f <= rg->from) { /* Trim beginning of region */ 548 del += t - rg->from; 549 rg->from = t; 550 } else { /* Trim end of region */ 551 del += rg->to - f; 552 rg->to = f; 553 } 554 } 555 556 spin_unlock(&resv->lock); 557 kfree(nrg); 558 return del; 559 } 560 561 /* 562 * A rare out of memory error was encountered which prevented removal of 563 * the reserve map region for a page. The huge page itself was free'ed 564 * and removed from the page cache. This routine will adjust the subpool 565 * usage count, and the global reserve count if needed. By incrementing 566 * these counts, the reserve map entry which could not be deleted will 567 * appear as a "reserved" entry instead of simply dangling with incorrect 568 * counts. 569 */ 570 void hugetlb_fix_reserve_counts(struct inode *inode) 571 { 572 struct hugepage_subpool *spool = subpool_inode(inode); 573 long rsv_adjust; 574 575 rsv_adjust = hugepage_subpool_get_pages(spool, 1); 576 if (rsv_adjust) { 577 struct hstate *h = hstate_inode(inode); 578 579 hugetlb_acct_memory(h, 1); 580 } 581 } 582 583 /* 584 * Count and return the number of huge pages in the reserve map 585 * that intersect with the range [f, t). 586 */ 587 static long region_count(struct resv_map *resv, long f, long t) 588 { 589 struct list_head *head = &resv->regions; 590 struct file_region *rg; 591 long chg = 0; 592 593 spin_lock(&resv->lock); 594 /* Locate each segment we overlap with, and count that overlap. */ 595 list_for_each_entry(rg, head, link) { 596 long seg_from; 597 long seg_to; 598 599 if (rg->to <= f) 600 continue; 601 if (rg->from >= t) 602 break; 603 604 seg_from = max(rg->from, f); 605 seg_to = min(rg->to, t); 606 607 chg += seg_to - seg_from; 608 } 609 spin_unlock(&resv->lock); 610 611 return chg; 612 } 613 614 /* 615 * Convert the address within this vma to the page offset within 616 * the mapping, in pagecache page units; huge pages here. 617 */ 618 static pgoff_t vma_hugecache_offset(struct hstate *h, 619 struct vm_area_struct *vma, unsigned long address) 620 { 621 return ((address - vma->vm_start) >> huge_page_shift(h)) + 622 (vma->vm_pgoff >> huge_page_order(h)); 623 } 624 625 pgoff_t linear_hugepage_index(struct vm_area_struct *vma, 626 unsigned long address) 627 { 628 return vma_hugecache_offset(hstate_vma(vma), vma, address); 629 } 630 EXPORT_SYMBOL_GPL(linear_hugepage_index); 631 632 /* 633 * Return the size of the pages allocated when backing a VMA. In the majority 634 * cases this will be same size as used by the page table entries. 635 */ 636 unsigned long vma_kernel_pagesize(struct vm_area_struct *vma) 637 { 638 struct hstate *hstate; 639 640 if (!is_vm_hugetlb_page(vma)) 641 return PAGE_SIZE; 642 643 hstate = hstate_vma(vma); 644 645 return 1UL << huge_page_shift(hstate); 646 } 647 EXPORT_SYMBOL_GPL(vma_kernel_pagesize); 648 649 /* 650 * Return the page size being used by the MMU to back a VMA. In the majority 651 * of cases, the page size used by the kernel matches the MMU size. On 652 * architectures where it differs, an architecture-specific version of this 653 * function is required. 654 */ 655 #ifndef vma_mmu_pagesize 656 unsigned long vma_mmu_pagesize(struct vm_area_struct *vma) 657 { 658 return vma_kernel_pagesize(vma); 659 } 660 #endif 661 662 /* 663 * Flags for MAP_PRIVATE reservations. These are stored in the bottom 664 * bits of the reservation map pointer, which are always clear due to 665 * alignment. 666 */ 667 #define HPAGE_RESV_OWNER (1UL << 0) 668 #define HPAGE_RESV_UNMAPPED (1UL << 1) 669 #define HPAGE_RESV_MASK (HPAGE_RESV_OWNER | HPAGE_RESV_UNMAPPED) 670 671 /* 672 * These helpers are used to track how many pages are reserved for 673 * faults in a MAP_PRIVATE mapping. Only the process that called mmap() 674 * is guaranteed to have their future faults succeed. 675 * 676 * With the exception of reset_vma_resv_huge_pages() which is called at fork(), 677 * the reserve counters are updated with the hugetlb_lock held. It is safe 678 * to reset the VMA at fork() time as it is not in use yet and there is no 679 * chance of the global counters getting corrupted as a result of the values. 680 * 681 * The private mapping reservation is represented in a subtly different 682 * manner to a shared mapping. A shared mapping has a region map associated 683 * with the underlying file, this region map represents the backing file 684 * pages which have ever had a reservation assigned which this persists even 685 * after the page is instantiated. A private mapping has a region map 686 * associated with the original mmap which is attached to all VMAs which 687 * reference it, this region map represents those offsets which have consumed 688 * reservation ie. where pages have been instantiated. 689 */ 690 static unsigned long get_vma_private_data(struct vm_area_struct *vma) 691 { 692 return (unsigned long)vma->vm_private_data; 693 } 694 695 static void set_vma_private_data(struct vm_area_struct *vma, 696 unsigned long value) 697 { 698 vma->vm_private_data = (void *)value; 699 } 700 701 struct resv_map *resv_map_alloc(void) 702 { 703 struct resv_map *resv_map = kmalloc(sizeof(*resv_map), GFP_KERNEL); 704 struct file_region *rg = kmalloc(sizeof(*rg), GFP_KERNEL); 705 706 if (!resv_map || !rg) { 707 kfree(resv_map); 708 kfree(rg); 709 return NULL; 710 } 711 712 kref_init(&resv_map->refs); 713 spin_lock_init(&resv_map->lock); 714 INIT_LIST_HEAD(&resv_map->regions); 715 716 resv_map->adds_in_progress = 0; 717 718 INIT_LIST_HEAD(&resv_map->region_cache); 719 list_add(&rg->link, &resv_map->region_cache); 720 resv_map->region_cache_count = 1; 721 722 return resv_map; 723 } 724 725 void resv_map_release(struct kref *ref) 726 { 727 struct resv_map *resv_map = container_of(ref, struct resv_map, refs); 728 struct list_head *head = &resv_map->region_cache; 729 struct file_region *rg, *trg; 730 731 /* Clear out any active regions before we release the map. */ 732 region_del(resv_map, 0, LONG_MAX); 733 734 /* ... and any entries left in the cache */ 735 list_for_each_entry_safe(rg, trg, head, link) { 736 list_del(&rg->link); 737 kfree(rg); 738 } 739 740 VM_BUG_ON(resv_map->adds_in_progress); 741 742 kfree(resv_map); 743 } 744 745 static inline struct resv_map *inode_resv_map(struct inode *inode) 746 { 747 return inode->i_mapping->private_data; 748 } 749 750 static struct resv_map *vma_resv_map(struct vm_area_struct *vma) 751 { 752 VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma); 753 if (vma->vm_flags & VM_MAYSHARE) { 754 struct address_space *mapping = vma->vm_file->f_mapping; 755 struct inode *inode = mapping->host; 756 757 return inode_resv_map(inode); 758 759 } else { 760 return (struct resv_map *)(get_vma_private_data(vma) & 761 ~HPAGE_RESV_MASK); 762 } 763 } 764 765 static void set_vma_resv_map(struct vm_area_struct *vma, struct resv_map *map) 766 { 767 VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma); 768 VM_BUG_ON_VMA(vma->vm_flags & VM_MAYSHARE, vma); 769 770 set_vma_private_data(vma, (get_vma_private_data(vma) & 771 HPAGE_RESV_MASK) | (unsigned long)map); 772 } 773 774 static void set_vma_resv_flags(struct vm_area_struct *vma, unsigned long flags) 775 { 776 VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma); 777 VM_BUG_ON_VMA(vma->vm_flags & VM_MAYSHARE, vma); 778 779 set_vma_private_data(vma, get_vma_private_data(vma) | flags); 780 } 781 782 static int is_vma_resv_set(struct vm_area_struct *vma, unsigned long flag) 783 { 784 VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma); 785 786 return (get_vma_private_data(vma) & flag) != 0; 787 } 788 789 /* Reset counters to 0 and clear all HPAGE_RESV_* flags */ 790 void reset_vma_resv_huge_pages(struct vm_area_struct *vma) 791 { 792 VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma); 793 if (!(vma->vm_flags & VM_MAYSHARE)) 794 vma->vm_private_data = (void *)0; 795 } 796 797 /* Returns true if the VMA has associated reserve pages */ 798 static bool vma_has_reserves(struct vm_area_struct *vma, long chg) 799 { 800 if (vma->vm_flags & VM_NORESERVE) { 801 /* 802 * This address is already reserved by other process(chg == 0), 803 * so, we should decrement reserved count. Without decrementing, 804 * reserve count remains after releasing inode, because this 805 * allocated page will go into page cache and is regarded as 806 * coming from reserved pool in releasing step. Currently, we 807 * don't have any other solution to deal with this situation 808 * properly, so add work-around here. 809 */ 810 if (vma->vm_flags & VM_MAYSHARE && chg == 0) 811 return true; 812 else 813 return false; 814 } 815 816 /* Shared mappings always use reserves */ 817 if (vma->vm_flags & VM_MAYSHARE) { 818 /* 819 * We know VM_NORESERVE is not set. Therefore, there SHOULD 820 * be a region map for all pages. The only situation where 821 * there is no region map is if a hole was punched via 822 * fallocate. In this case, there really are no reverves to 823 * use. This situation is indicated if chg != 0. 824 */ 825 if (chg) 826 return false; 827 else 828 return true; 829 } 830 831 /* 832 * Only the process that called mmap() has reserves for 833 * private mappings. 834 */ 835 if (is_vma_resv_set(vma, HPAGE_RESV_OWNER)) { 836 /* 837 * Like the shared case above, a hole punch or truncate 838 * could have been performed on the private mapping. 839 * Examine the value of chg to determine if reserves 840 * actually exist or were previously consumed. 841 * Very Subtle - The value of chg comes from a previous 842 * call to vma_needs_reserves(). The reserve map for 843 * private mappings has different (opposite) semantics 844 * than that of shared mappings. vma_needs_reserves() 845 * has already taken this difference in semantics into 846 * account. Therefore, the meaning of chg is the same 847 * as in the shared case above. Code could easily be 848 * combined, but keeping it separate draws attention to 849 * subtle differences. 850 */ 851 if (chg) 852 return false; 853 else 854 return true; 855 } 856 857 return false; 858 } 859 860 static void enqueue_huge_page(struct hstate *h, struct page *page) 861 { 862 int nid = page_to_nid(page); 863 list_move(&page->lru, &h->hugepage_freelists[nid]); 864 h->free_huge_pages++; 865 h->free_huge_pages_node[nid]++; 866 } 867 868 static struct page *dequeue_huge_page_node(struct hstate *h, int nid) 869 { 870 struct page *page; 871 872 list_for_each_entry(page, &h->hugepage_freelists[nid], lru) 873 if (!is_migrate_isolate_page(page)) 874 break; 875 /* 876 * if 'non-isolated free hugepage' not found on the list, 877 * the allocation fails. 878 */ 879 if (&h->hugepage_freelists[nid] == &page->lru) 880 return NULL; 881 list_move(&page->lru, &h->hugepage_activelist); 882 set_page_refcounted(page); 883 h->free_huge_pages--; 884 h->free_huge_pages_node[nid]--; 885 return page; 886 } 887 888 /* Movability of hugepages depends on migration support. */ 889 static inline gfp_t htlb_alloc_mask(struct hstate *h) 890 { 891 if (hugepages_treat_as_movable || hugepage_migration_supported(h)) 892 return GFP_HIGHUSER_MOVABLE; 893 else 894 return GFP_HIGHUSER; 895 } 896 897 static struct page *dequeue_huge_page_vma(struct hstate *h, 898 struct vm_area_struct *vma, 899 unsigned long address, int avoid_reserve, 900 long chg) 901 { 902 struct page *page = NULL; 903 struct mempolicy *mpol; 904 nodemask_t *nodemask; 905 struct zonelist *zonelist; 906 struct zone *zone; 907 struct zoneref *z; 908 unsigned int cpuset_mems_cookie; 909 910 /* 911 * A child process with MAP_PRIVATE mappings created by their parent 912 * have no page reserves. This check ensures that reservations are 913 * not "stolen". The child may still get SIGKILLed 914 */ 915 if (!vma_has_reserves(vma, chg) && 916 h->free_huge_pages - h->resv_huge_pages == 0) 917 goto err; 918 919 /* If reserves cannot be used, ensure enough pages are in the pool */ 920 if (avoid_reserve && h->free_huge_pages - h->resv_huge_pages == 0) 921 goto err; 922 923 retry_cpuset: 924 cpuset_mems_cookie = read_mems_allowed_begin(); 925 zonelist = huge_zonelist(vma, address, 926 htlb_alloc_mask(h), &mpol, &nodemask); 927 928 for_each_zone_zonelist_nodemask(zone, z, zonelist, 929 MAX_NR_ZONES - 1, nodemask) { 930 if (cpuset_zone_allowed(zone, htlb_alloc_mask(h))) { 931 page = dequeue_huge_page_node(h, zone_to_nid(zone)); 932 if (page) { 933 if (avoid_reserve) 934 break; 935 if (!vma_has_reserves(vma, chg)) 936 break; 937 938 SetPagePrivate(page); 939 h->resv_huge_pages--; 940 break; 941 } 942 } 943 } 944 945 mpol_cond_put(mpol); 946 if (unlikely(!page && read_mems_allowed_retry(cpuset_mems_cookie))) 947 goto retry_cpuset; 948 return page; 949 950 err: 951 return NULL; 952 } 953 954 /* 955 * common helper functions for hstate_next_node_to_{alloc|free}. 956 * We may have allocated or freed a huge page based on a different 957 * nodes_allowed previously, so h->next_node_to_{alloc|free} might 958 * be outside of *nodes_allowed. Ensure that we use an allowed 959 * node for alloc or free. 960 */ 961 static int next_node_allowed(int nid, nodemask_t *nodes_allowed) 962 { 963 nid = next_node_in(nid, *nodes_allowed); 964 VM_BUG_ON(nid >= MAX_NUMNODES); 965 966 return nid; 967 } 968 969 static int get_valid_node_allowed(int nid, nodemask_t *nodes_allowed) 970 { 971 if (!node_isset(nid, *nodes_allowed)) 972 nid = next_node_allowed(nid, nodes_allowed); 973 return nid; 974 } 975 976 /* 977 * returns the previously saved node ["this node"] from which to 978 * allocate a persistent huge page for the pool and advance the 979 * next node from which to allocate, handling wrap at end of node 980 * mask. 981 */ 982 static int hstate_next_node_to_alloc(struct hstate *h, 983 nodemask_t *nodes_allowed) 984 { 985 int nid; 986 987 VM_BUG_ON(!nodes_allowed); 988 989 nid = get_valid_node_allowed(h->next_nid_to_alloc, nodes_allowed); 990 h->next_nid_to_alloc = next_node_allowed(nid, nodes_allowed); 991 992 return nid; 993 } 994 995 /* 996 * helper for free_pool_huge_page() - return the previously saved 997 * node ["this node"] from which to free a huge page. Advance the 998 * next node id whether or not we find a free huge page to free so 999 * that the next attempt to free addresses the next node. 1000 */ 1001 static int hstate_next_node_to_free(struct hstate *h, nodemask_t *nodes_allowed) 1002 { 1003 int nid; 1004 1005 VM_BUG_ON(!nodes_allowed); 1006 1007 nid = get_valid_node_allowed(h->next_nid_to_free, nodes_allowed); 1008 h->next_nid_to_free = next_node_allowed(nid, nodes_allowed); 1009 1010 return nid; 1011 } 1012 1013 #define for_each_node_mask_to_alloc(hs, nr_nodes, node, mask) \ 1014 for (nr_nodes = nodes_weight(*mask); \ 1015 nr_nodes > 0 && \ 1016 ((node = hstate_next_node_to_alloc(hs, mask)) || 1); \ 1017 nr_nodes--) 1018 1019 #define for_each_node_mask_to_free(hs, nr_nodes, node, mask) \ 1020 for (nr_nodes = nodes_weight(*mask); \ 1021 nr_nodes > 0 && \ 1022 ((node = hstate_next_node_to_free(hs, mask)) || 1); \ 1023 nr_nodes--) 1024 1025 #if defined(CONFIG_ARCH_HAS_GIGANTIC_PAGE) && \ 1026 ((defined(CONFIG_MEMORY_ISOLATION) && defined(CONFIG_COMPACTION)) || \ 1027 defined(CONFIG_CMA)) 1028 static void destroy_compound_gigantic_page(struct page *page, 1029 unsigned int order) 1030 { 1031 int i; 1032 int nr_pages = 1 << order; 1033 struct page *p = page + 1; 1034 1035 atomic_set(compound_mapcount_ptr(page), 0); 1036 for (i = 1; i < nr_pages; i++, p = mem_map_next(p, page, i)) { 1037 clear_compound_head(p); 1038 set_page_refcounted(p); 1039 } 1040 1041 set_compound_order(page, 0); 1042 __ClearPageHead(page); 1043 } 1044 1045 static void free_gigantic_page(struct page *page, unsigned int order) 1046 { 1047 free_contig_range(page_to_pfn(page), 1 << order); 1048 } 1049 1050 static int __alloc_gigantic_page(unsigned long start_pfn, 1051 unsigned long nr_pages) 1052 { 1053 unsigned long end_pfn = start_pfn + nr_pages; 1054 return alloc_contig_range(start_pfn, end_pfn, MIGRATE_MOVABLE); 1055 } 1056 1057 static bool pfn_range_valid_gigantic(struct zone *z, 1058 unsigned long start_pfn, unsigned long nr_pages) 1059 { 1060 unsigned long i, end_pfn = start_pfn + nr_pages; 1061 struct page *page; 1062 1063 for (i = start_pfn; i < end_pfn; i++) { 1064 if (!pfn_valid(i)) 1065 return false; 1066 1067 page = pfn_to_page(i); 1068 1069 if (page_zone(page) != z) 1070 return false; 1071 1072 if (PageReserved(page)) 1073 return false; 1074 1075 if (page_count(page) > 0) 1076 return false; 1077 1078 if (PageHuge(page)) 1079 return false; 1080 } 1081 1082 return true; 1083 } 1084 1085 static bool zone_spans_last_pfn(const struct zone *zone, 1086 unsigned long start_pfn, unsigned long nr_pages) 1087 { 1088 unsigned long last_pfn = start_pfn + nr_pages - 1; 1089 return zone_spans_pfn(zone, last_pfn); 1090 } 1091 1092 static struct page *alloc_gigantic_page(int nid, unsigned int order) 1093 { 1094 unsigned long nr_pages = 1 << order; 1095 unsigned long ret, pfn, flags; 1096 struct zone *z; 1097 1098 z = NODE_DATA(nid)->node_zones; 1099 for (; z - NODE_DATA(nid)->node_zones < MAX_NR_ZONES; z++) { 1100 spin_lock_irqsave(&z->lock, flags); 1101 1102 pfn = ALIGN(z->zone_start_pfn, nr_pages); 1103 while (zone_spans_last_pfn(z, pfn, nr_pages)) { 1104 if (pfn_range_valid_gigantic(z, pfn, nr_pages)) { 1105 /* 1106 * We release the zone lock here because 1107 * alloc_contig_range() will also lock the zone 1108 * at some point. If there's an allocation 1109 * spinning on this lock, it may win the race 1110 * and cause alloc_contig_range() to fail... 1111 */ 1112 spin_unlock_irqrestore(&z->lock, flags); 1113 ret = __alloc_gigantic_page(pfn, nr_pages); 1114 if (!ret) 1115 return pfn_to_page(pfn); 1116 spin_lock_irqsave(&z->lock, flags); 1117 } 1118 pfn += nr_pages; 1119 } 1120 1121 spin_unlock_irqrestore(&z->lock, flags); 1122 } 1123 1124 return NULL; 1125 } 1126 1127 static void prep_new_huge_page(struct hstate *h, struct page *page, int nid); 1128 static void prep_compound_gigantic_page(struct page *page, unsigned int order); 1129 1130 static struct page *alloc_fresh_gigantic_page_node(struct hstate *h, int nid) 1131 { 1132 struct page *page; 1133 1134 page = alloc_gigantic_page(nid, huge_page_order(h)); 1135 if (page) { 1136 prep_compound_gigantic_page(page, huge_page_order(h)); 1137 prep_new_huge_page(h, page, nid); 1138 } 1139 1140 return page; 1141 } 1142 1143 static int alloc_fresh_gigantic_page(struct hstate *h, 1144 nodemask_t *nodes_allowed) 1145 { 1146 struct page *page = NULL; 1147 int nr_nodes, node; 1148 1149 for_each_node_mask_to_alloc(h, nr_nodes, node, nodes_allowed) { 1150 page = alloc_fresh_gigantic_page_node(h, node); 1151 if (page) 1152 return 1; 1153 } 1154 1155 return 0; 1156 } 1157 1158 static inline bool gigantic_page_supported(void) { return true; } 1159 #else 1160 static inline bool gigantic_page_supported(void) { return false; } 1161 static inline void free_gigantic_page(struct page *page, unsigned int order) { } 1162 static inline void destroy_compound_gigantic_page(struct page *page, 1163 unsigned int order) { } 1164 static inline int alloc_fresh_gigantic_page(struct hstate *h, 1165 nodemask_t *nodes_allowed) { return 0; } 1166 #endif 1167 1168 static void update_and_free_page(struct hstate *h, struct page *page) 1169 { 1170 int i; 1171 1172 if (hstate_is_gigantic(h) && !gigantic_page_supported()) 1173 return; 1174 1175 h->nr_huge_pages--; 1176 h->nr_huge_pages_node[page_to_nid(page)]--; 1177 for (i = 0; i < pages_per_huge_page(h); i++) { 1178 page[i].flags &= ~(1 << PG_locked | 1 << PG_error | 1179 1 << PG_referenced | 1 << PG_dirty | 1180 1 << PG_active | 1 << PG_private | 1181 1 << PG_writeback); 1182 } 1183 VM_BUG_ON_PAGE(hugetlb_cgroup_from_page(page), page); 1184 set_compound_page_dtor(page, NULL_COMPOUND_DTOR); 1185 set_page_refcounted(page); 1186 if (hstate_is_gigantic(h)) { 1187 destroy_compound_gigantic_page(page, huge_page_order(h)); 1188 free_gigantic_page(page, huge_page_order(h)); 1189 } else { 1190 __free_pages(page, huge_page_order(h)); 1191 } 1192 } 1193 1194 struct hstate *size_to_hstate(unsigned long size) 1195 { 1196 struct hstate *h; 1197 1198 for_each_hstate(h) { 1199 if (huge_page_size(h) == size) 1200 return h; 1201 } 1202 return NULL; 1203 } 1204 1205 /* 1206 * Test to determine whether the hugepage is "active/in-use" (i.e. being linked 1207 * to hstate->hugepage_activelist.) 1208 * 1209 * This function can be called for tail pages, but never returns true for them. 1210 */ 1211 bool page_huge_active(struct page *page) 1212 { 1213 VM_BUG_ON_PAGE(!PageHuge(page), page); 1214 return PageHead(page) && PagePrivate(&page[1]); 1215 } 1216 1217 /* never called for tail page */ 1218 static void set_page_huge_active(struct page *page) 1219 { 1220 VM_BUG_ON_PAGE(!PageHeadHuge(page), page); 1221 SetPagePrivate(&page[1]); 1222 } 1223 1224 static void clear_page_huge_active(struct page *page) 1225 { 1226 VM_BUG_ON_PAGE(!PageHeadHuge(page), page); 1227 ClearPagePrivate(&page[1]); 1228 } 1229 1230 void free_huge_page(struct page *page) 1231 { 1232 /* 1233 * Can't pass hstate in here because it is called from the 1234 * compound page destructor. 1235 */ 1236 struct hstate *h = page_hstate(page); 1237 int nid = page_to_nid(page); 1238 struct hugepage_subpool *spool = 1239 (struct hugepage_subpool *)page_private(page); 1240 bool restore_reserve; 1241 1242 set_page_private(page, 0); 1243 page->mapping = NULL; 1244 VM_BUG_ON_PAGE(page_count(page), page); 1245 VM_BUG_ON_PAGE(page_mapcount(page), page); 1246 restore_reserve = PagePrivate(page); 1247 ClearPagePrivate(page); 1248 1249 /* 1250 * A return code of zero implies that the subpool will be under its 1251 * minimum size if the reservation is not restored after page is free. 1252 * Therefore, force restore_reserve operation. 1253 */ 1254 if (hugepage_subpool_put_pages(spool, 1) == 0) 1255 restore_reserve = true; 1256 1257 spin_lock(&hugetlb_lock); 1258 clear_page_huge_active(page); 1259 hugetlb_cgroup_uncharge_page(hstate_index(h), 1260 pages_per_huge_page(h), page); 1261 if (restore_reserve) 1262 h->resv_huge_pages++; 1263 1264 if (h->surplus_huge_pages_node[nid]) { 1265 /* remove the page from active list */ 1266 list_del(&page->lru); 1267 update_and_free_page(h, page); 1268 h->surplus_huge_pages--; 1269 h->surplus_huge_pages_node[nid]--; 1270 } else { 1271 arch_clear_hugepage_flags(page); 1272 enqueue_huge_page(h, page); 1273 } 1274 spin_unlock(&hugetlb_lock); 1275 } 1276 1277 static void prep_new_huge_page(struct hstate *h, struct page *page, int nid) 1278 { 1279 INIT_LIST_HEAD(&page->lru); 1280 set_compound_page_dtor(page, HUGETLB_PAGE_DTOR); 1281 spin_lock(&hugetlb_lock); 1282 set_hugetlb_cgroup(page, NULL); 1283 h->nr_huge_pages++; 1284 h->nr_huge_pages_node[nid]++; 1285 spin_unlock(&hugetlb_lock); 1286 put_page(page); /* free it into the hugepage allocator */ 1287 } 1288 1289 static void prep_compound_gigantic_page(struct page *page, unsigned int order) 1290 { 1291 int i; 1292 int nr_pages = 1 << order; 1293 struct page *p = page + 1; 1294 1295 /* we rely on prep_new_huge_page to set the destructor */ 1296 set_compound_order(page, order); 1297 __ClearPageReserved(page); 1298 __SetPageHead(page); 1299 for (i = 1; i < nr_pages; i++, p = mem_map_next(p, page, i)) { 1300 /* 1301 * For gigantic hugepages allocated through bootmem at 1302 * boot, it's safer to be consistent with the not-gigantic 1303 * hugepages and clear the PG_reserved bit from all tail pages 1304 * too. Otherwse drivers using get_user_pages() to access tail 1305 * pages may get the reference counting wrong if they see 1306 * PG_reserved set on a tail page (despite the head page not 1307 * having PG_reserved set). Enforcing this consistency between 1308 * head and tail pages allows drivers to optimize away a check 1309 * on the head page when they need know if put_page() is needed 1310 * after get_user_pages(). 1311 */ 1312 __ClearPageReserved(p); 1313 set_page_count(p, 0); 1314 set_compound_head(p, page); 1315 } 1316 atomic_set(compound_mapcount_ptr(page), -1); 1317 } 1318 1319 /* 1320 * PageHuge() only returns true for hugetlbfs pages, but not for normal or 1321 * transparent huge pages. See the PageTransHuge() documentation for more 1322 * details. 1323 */ 1324 int PageHuge(struct page *page) 1325 { 1326 if (!PageCompound(page)) 1327 return 0; 1328 1329 page = compound_head(page); 1330 return page[1].compound_dtor == HUGETLB_PAGE_DTOR; 1331 } 1332 EXPORT_SYMBOL_GPL(PageHuge); 1333 1334 /* 1335 * PageHeadHuge() only returns true for hugetlbfs head page, but not for 1336 * normal or transparent huge pages. 1337 */ 1338 int PageHeadHuge(struct page *page_head) 1339 { 1340 if (!PageHead(page_head)) 1341 return 0; 1342 1343 return get_compound_page_dtor(page_head) == free_huge_page; 1344 } 1345 1346 pgoff_t __basepage_index(struct page *page) 1347 { 1348 struct page *page_head = compound_head(page); 1349 pgoff_t index = page_index(page_head); 1350 unsigned long compound_idx; 1351 1352 if (!PageHuge(page_head)) 1353 return page_index(page); 1354 1355 if (compound_order(page_head) >= MAX_ORDER) 1356 compound_idx = page_to_pfn(page) - page_to_pfn(page_head); 1357 else 1358 compound_idx = page - page_head; 1359 1360 return (index << compound_order(page_head)) + compound_idx; 1361 } 1362 1363 static struct page *alloc_fresh_huge_page_node(struct hstate *h, int nid) 1364 { 1365 struct page *page; 1366 1367 page = __alloc_pages_node(nid, 1368 htlb_alloc_mask(h)|__GFP_COMP|__GFP_THISNODE| 1369 __GFP_REPEAT|__GFP_NOWARN, 1370 huge_page_order(h)); 1371 if (page) { 1372 prep_new_huge_page(h, page, nid); 1373 } 1374 1375 return page; 1376 } 1377 1378 static int alloc_fresh_huge_page(struct hstate *h, nodemask_t *nodes_allowed) 1379 { 1380 struct page *page; 1381 int nr_nodes, node; 1382 int ret = 0; 1383 1384 for_each_node_mask_to_alloc(h, nr_nodes, node, nodes_allowed) { 1385 page = alloc_fresh_huge_page_node(h, node); 1386 if (page) { 1387 ret = 1; 1388 break; 1389 } 1390 } 1391 1392 if (ret) 1393 count_vm_event(HTLB_BUDDY_PGALLOC); 1394 else 1395 count_vm_event(HTLB_BUDDY_PGALLOC_FAIL); 1396 1397 return ret; 1398 } 1399 1400 /* 1401 * Free huge page from pool from next node to free. 1402 * Attempt to keep persistent huge pages more or less 1403 * balanced over allowed nodes. 1404 * Called with hugetlb_lock locked. 1405 */ 1406 static int free_pool_huge_page(struct hstate *h, nodemask_t *nodes_allowed, 1407 bool acct_surplus) 1408 { 1409 int nr_nodes, node; 1410 int ret = 0; 1411 1412 for_each_node_mask_to_free(h, nr_nodes, node, nodes_allowed) { 1413 /* 1414 * If we're returning unused surplus pages, only examine 1415 * nodes with surplus pages. 1416 */ 1417 if ((!acct_surplus || h->surplus_huge_pages_node[node]) && 1418 !list_empty(&h->hugepage_freelists[node])) { 1419 struct page *page = 1420 list_entry(h->hugepage_freelists[node].next, 1421 struct page, lru); 1422 list_del(&page->lru); 1423 h->free_huge_pages--; 1424 h->free_huge_pages_node[node]--; 1425 if (acct_surplus) { 1426 h->surplus_huge_pages--; 1427 h->surplus_huge_pages_node[node]--; 1428 } 1429 update_and_free_page(h, page); 1430 ret = 1; 1431 break; 1432 } 1433 } 1434 1435 return ret; 1436 } 1437 1438 /* 1439 * Dissolve a given free hugepage into free buddy pages. This function does 1440 * nothing for in-use (including surplus) hugepages. Returns -EBUSY if the 1441 * number of free hugepages would be reduced below the number of reserved 1442 * hugepages. 1443 */ 1444 static int dissolve_free_huge_page(struct page *page) 1445 { 1446 int rc = 0; 1447 1448 spin_lock(&hugetlb_lock); 1449 if (PageHuge(page) && !page_count(page)) { 1450 struct page *head = compound_head(page); 1451 struct hstate *h = page_hstate(head); 1452 int nid = page_to_nid(head); 1453 if (h->free_huge_pages - h->resv_huge_pages == 0) { 1454 rc = -EBUSY; 1455 goto out; 1456 } 1457 list_del(&head->lru); 1458 h->free_huge_pages--; 1459 h->free_huge_pages_node[nid]--; 1460 h->max_huge_pages--; 1461 update_and_free_page(h, head); 1462 } 1463 out: 1464 spin_unlock(&hugetlb_lock); 1465 return rc; 1466 } 1467 1468 /* 1469 * Dissolve free hugepages in a given pfn range. Used by memory hotplug to 1470 * make specified memory blocks removable from the system. 1471 * Note that this will dissolve a free gigantic hugepage completely, if any 1472 * part of it lies within the given range. 1473 * Also note that if dissolve_free_huge_page() returns with an error, all 1474 * free hugepages that were dissolved before that error are lost. 1475 */ 1476 int dissolve_free_huge_pages(unsigned long start_pfn, unsigned long end_pfn) 1477 { 1478 unsigned long pfn; 1479 struct page *page; 1480 int rc = 0; 1481 1482 if (!hugepages_supported()) 1483 return rc; 1484 1485 for (pfn = start_pfn; pfn < end_pfn; pfn += 1 << minimum_order) { 1486 page = pfn_to_page(pfn); 1487 if (PageHuge(page) && !page_count(page)) { 1488 rc = dissolve_free_huge_page(page); 1489 if (rc) 1490 break; 1491 } 1492 } 1493 1494 return rc; 1495 } 1496 1497 /* 1498 * There are 3 ways this can get called: 1499 * 1. With vma+addr: we use the VMA's memory policy 1500 * 2. With !vma, but nid=NUMA_NO_NODE: We try to allocate a huge 1501 * page from any node, and let the buddy allocator itself figure 1502 * it out. 1503 * 3. With !vma, but nid!=NUMA_NO_NODE. We allocate a huge page 1504 * strictly from 'nid' 1505 */ 1506 static struct page *__hugetlb_alloc_buddy_huge_page(struct hstate *h, 1507 struct vm_area_struct *vma, unsigned long addr, int nid) 1508 { 1509 int order = huge_page_order(h); 1510 gfp_t gfp = htlb_alloc_mask(h)|__GFP_COMP|__GFP_REPEAT|__GFP_NOWARN; 1511 unsigned int cpuset_mems_cookie; 1512 1513 /* 1514 * We need a VMA to get a memory policy. If we do not 1515 * have one, we use the 'nid' argument. 1516 * 1517 * The mempolicy stuff below has some non-inlined bits 1518 * and calls ->vm_ops. That makes it hard to optimize at 1519 * compile-time, even when NUMA is off and it does 1520 * nothing. This helps the compiler optimize it out. 1521 */ 1522 if (!IS_ENABLED(CONFIG_NUMA) || !vma) { 1523 /* 1524 * If a specific node is requested, make sure to 1525 * get memory from there, but only when a node 1526 * is explicitly specified. 1527 */ 1528 if (nid != NUMA_NO_NODE) 1529 gfp |= __GFP_THISNODE; 1530 /* 1531 * Make sure to call something that can handle 1532 * nid=NUMA_NO_NODE 1533 */ 1534 return alloc_pages_node(nid, gfp, order); 1535 } 1536 1537 /* 1538 * OK, so we have a VMA. Fetch the mempolicy and try to 1539 * allocate a huge page with it. We will only reach this 1540 * when CONFIG_NUMA=y. 1541 */ 1542 do { 1543 struct page *page; 1544 struct mempolicy *mpol; 1545 struct zonelist *zl; 1546 nodemask_t *nodemask; 1547 1548 cpuset_mems_cookie = read_mems_allowed_begin(); 1549 zl = huge_zonelist(vma, addr, gfp, &mpol, &nodemask); 1550 mpol_cond_put(mpol); 1551 page = __alloc_pages_nodemask(gfp, order, zl, nodemask); 1552 if (page) 1553 return page; 1554 } while (read_mems_allowed_retry(cpuset_mems_cookie)); 1555 1556 return NULL; 1557 } 1558 1559 /* 1560 * There are two ways to allocate a huge page: 1561 * 1. When you have a VMA and an address (like a fault) 1562 * 2. When you have no VMA (like when setting /proc/.../nr_hugepages) 1563 * 1564 * 'vma' and 'addr' are only for (1). 'nid' is always NUMA_NO_NODE in 1565 * this case which signifies that the allocation should be done with 1566 * respect for the VMA's memory policy. 1567 * 1568 * For (2), we ignore 'vma' and 'addr' and use 'nid' exclusively. This 1569 * implies that memory policies will not be taken in to account. 1570 */ 1571 static struct page *__alloc_buddy_huge_page(struct hstate *h, 1572 struct vm_area_struct *vma, unsigned long addr, int nid) 1573 { 1574 struct page *page; 1575 unsigned int r_nid; 1576 1577 if (hstate_is_gigantic(h)) 1578 return NULL; 1579 1580 /* 1581 * Make sure that anyone specifying 'nid' is not also specifying a VMA. 1582 * This makes sure the caller is picking _one_ of the modes with which 1583 * we can call this function, not both. 1584 */ 1585 if (vma || (addr != -1)) { 1586 VM_WARN_ON_ONCE(addr == -1); 1587 VM_WARN_ON_ONCE(nid != NUMA_NO_NODE); 1588 } 1589 /* 1590 * Assume we will successfully allocate the surplus page to 1591 * prevent racing processes from causing the surplus to exceed 1592 * overcommit 1593 * 1594 * This however introduces a different race, where a process B 1595 * tries to grow the static hugepage pool while alloc_pages() is 1596 * called by process A. B will only examine the per-node 1597 * counters in determining if surplus huge pages can be 1598 * converted to normal huge pages in adjust_pool_surplus(). A 1599 * won't be able to increment the per-node counter, until the 1600 * lock is dropped by B, but B doesn't drop hugetlb_lock until 1601 * no more huge pages can be converted from surplus to normal 1602 * state (and doesn't try to convert again). Thus, we have a 1603 * case where a surplus huge page exists, the pool is grown, and 1604 * the surplus huge page still exists after, even though it 1605 * should just have been converted to a normal huge page. This 1606 * does not leak memory, though, as the hugepage will be freed 1607 * once it is out of use. It also does not allow the counters to 1608 * go out of whack in adjust_pool_surplus() as we don't modify 1609 * the node values until we've gotten the hugepage and only the 1610 * per-node value is checked there. 1611 */ 1612 spin_lock(&hugetlb_lock); 1613 if (h->surplus_huge_pages >= h->nr_overcommit_huge_pages) { 1614 spin_unlock(&hugetlb_lock); 1615 return NULL; 1616 } else { 1617 h->nr_huge_pages++; 1618 h->surplus_huge_pages++; 1619 } 1620 spin_unlock(&hugetlb_lock); 1621 1622 page = __hugetlb_alloc_buddy_huge_page(h, vma, addr, nid); 1623 1624 spin_lock(&hugetlb_lock); 1625 if (page) { 1626 INIT_LIST_HEAD(&page->lru); 1627 r_nid = page_to_nid(page); 1628 set_compound_page_dtor(page, HUGETLB_PAGE_DTOR); 1629 set_hugetlb_cgroup(page, NULL); 1630 /* 1631 * We incremented the global counters already 1632 */ 1633 h->nr_huge_pages_node[r_nid]++; 1634 h->surplus_huge_pages_node[r_nid]++; 1635 __count_vm_event(HTLB_BUDDY_PGALLOC); 1636 } else { 1637 h->nr_huge_pages--; 1638 h->surplus_huge_pages--; 1639 __count_vm_event(HTLB_BUDDY_PGALLOC_FAIL); 1640 } 1641 spin_unlock(&hugetlb_lock); 1642 1643 return page; 1644 } 1645 1646 /* 1647 * Allocate a huge page from 'nid'. Note, 'nid' may be 1648 * NUMA_NO_NODE, which means that it may be allocated 1649 * anywhere. 1650 */ 1651 static 1652 struct page *__alloc_buddy_huge_page_no_mpol(struct hstate *h, int nid) 1653 { 1654 unsigned long addr = -1; 1655 1656 return __alloc_buddy_huge_page(h, NULL, addr, nid); 1657 } 1658 1659 /* 1660 * Use the VMA's mpolicy to allocate a huge page from the buddy. 1661 */ 1662 static 1663 struct page *__alloc_buddy_huge_page_with_mpol(struct hstate *h, 1664 struct vm_area_struct *vma, unsigned long addr) 1665 { 1666 return __alloc_buddy_huge_page(h, vma, addr, NUMA_NO_NODE); 1667 } 1668 1669 /* 1670 * This allocation function is useful in the context where vma is irrelevant. 1671 * E.g. soft-offlining uses this function because it only cares physical 1672 * address of error page. 1673 */ 1674 struct page *alloc_huge_page_node(struct hstate *h, int nid) 1675 { 1676 struct page *page = NULL; 1677 1678 spin_lock(&hugetlb_lock); 1679 if (h->free_huge_pages - h->resv_huge_pages > 0) 1680 page = dequeue_huge_page_node(h, nid); 1681 spin_unlock(&hugetlb_lock); 1682 1683 if (!page) 1684 page = __alloc_buddy_huge_page_no_mpol(h, nid); 1685 1686 return page; 1687 } 1688 1689 /* 1690 * Increase the hugetlb pool such that it can accommodate a reservation 1691 * of size 'delta'. 1692 */ 1693 static int gather_surplus_pages(struct hstate *h, int delta) 1694 { 1695 struct list_head surplus_list; 1696 struct page *page, *tmp; 1697 int ret, i; 1698 int needed, allocated; 1699 bool alloc_ok = true; 1700 1701 needed = (h->resv_huge_pages + delta) - h->free_huge_pages; 1702 if (needed <= 0) { 1703 h->resv_huge_pages += delta; 1704 return 0; 1705 } 1706 1707 allocated = 0; 1708 INIT_LIST_HEAD(&surplus_list); 1709 1710 ret = -ENOMEM; 1711 retry: 1712 spin_unlock(&hugetlb_lock); 1713 for (i = 0; i < needed; i++) { 1714 page = __alloc_buddy_huge_page_no_mpol(h, NUMA_NO_NODE); 1715 if (!page) { 1716 alloc_ok = false; 1717 break; 1718 } 1719 list_add(&page->lru, &surplus_list); 1720 } 1721 allocated += i; 1722 1723 /* 1724 * After retaking hugetlb_lock, we need to recalculate 'needed' 1725 * because either resv_huge_pages or free_huge_pages may have changed. 1726 */ 1727 spin_lock(&hugetlb_lock); 1728 needed = (h->resv_huge_pages + delta) - 1729 (h->free_huge_pages + allocated); 1730 if (needed > 0) { 1731 if (alloc_ok) 1732 goto retry; 1733 /* 1734 * We were not able to allocate enough pages to 1735 * satisfy the entire reservation so we free what 1736 * we've allocated so far. 1737 */ 1738 goto free; 1739 } 1740 /* 1741 * The surplus_list now contains _at_least_ the number of extra pages 1742 * needed to accommodate the reservation. Add the appropriate number 1743 * of pages to the hugetlb pool and free the extras back to the buddy 1744 * allocator. Commit the entire reservation here to prevent another 1745 * process from stealing the pages as they are added to the pool but 1746 * before they are reserved. 1747 */ 1748 needed += allocated; 1749 h->resv_huge_pages += delta; 1750 ret = 0; 1751 1752 /* Free the needed pages to the hugetlb pool */ 1753 list_for_each_entry_safe(page, tmp, &surplus_list, lru) { 1754 if ((--needed) < 0) 1755 break; 1756 /* 1757 * This page is now managed by the hugetlb allocator and has 1758 * no users -- drop the buddy allocator's reference. 1759 */ 1760 put_page_testzero(page); 1761 VM_BUG_ON_PAGE(page_count(page), page); 1762 enqueue_huge_page(h, page); 1763 } 1764 free: 1765 spin_unlock(&hugetlb_lock); 1766 1767 /* Free unnecessary surplus pages to the buddy allocator */ 1768 list_for_each_entry_safe(page, tmp, &surplus_list, lru) 1769 put_page(page); 1770 spin_lock(&hugetlb_lock); 1771 1772 return ret; 1773 } 1774 1775 /* 1776 * When releasing a hugetlb pool reservation, any surplus pages that were 1777 * allocated to satisfy the reservation must be explicitly freed if they were 1778 * never used. 1779 * Called with hugetlb_lock held. 1780 */ 1781 static void return_unused_surplus_pages(struct hstate *h, 1782 unsigned long unused_resv_pages) 1783 { 1784 unsigned long nr_pages; 1785 1786 /* Uncommit the reservation */ 1787 h->resv_huge_pages -= unused_resv_pages; 1788 1789 /* Cannot return gigantic pages currently */ 1790 if (hstate_is_gigantic(h)) 1791 return; 1792 1793 nr_pages = min(unused_resv_pages, h->surplus_huge_pages); 1794 1795 /* 1796 * We want to release as many surplus pages as possible, spread 1797 * evenly across all nodes with memory. Iterate across these nodes 1798 * until we can no longer free unreserved surplus pages. This occurs 1799 * when the nodes with surplus pages have no free pages. 1800 * free_pool_huge_page() will balance the the freed pages across the 1801 * on-line nodes with memory and will handle the hstate accounting. 1802 */ 1803 while (nr_pages--) { 1804 if (!free_pool_huge_page(h, &node_states[N_MEMORY], 1)) 1805 break; 1806 cond_resched_lock(&hugetlb_lock); 1807 } 1808 } 1809 1810 1811 /* 1812 * vma_needs_reservation, vma_commit_reservation and vma_end_reservation 1813 * are used by the huge page allocation routines to manage reservations. 1814 * 1815 * vma_needs_reservation is called to determine if the huge page at addr 1816 * within the vma has an associated reservation. If a reservation is 1817 * needed, the value 1 is returned. The caller is then responsible for 1818 * managing the global reservation and subpool usage counts. After 1819 * the huge page has been allocated, vma_commit_reservation is called 1820 * to add the page to the reservation map. If the page allocation fails, 1821 * the reservation must be ended instead of committed. vma_end_reservation 1822 * is called in such cases. 1823 * 1824 * In the normal case, vma_commit_reservation returns the same value 1825 * as the preceding vma_needs_reservation call. The only time this 1826 * is not the case is if a reserve map was changed between calls. It 1827 * is the responsibility of the caller to notice the difference and 1828 * take appropriate action. 1829 */ 1830 enum vma_resv_mode { 1831 VMA_NEEDS_RESV, 1832 VMA_COMMIT_RESV, 1833 VMA_END_RESV, 1834 }; 1835 static long __vma_reservation_common(struct hstate *h, 1836 struct vm_area_struct *vma, unsigned long addr, 1837 enum vma_resv_mode mode) 1838 { 1839 struct resv_map *resv; 1840 pgoff_t idx; 1841 long ret; 1842 1843 resv = vma_resv_map(vma); 1844 if (!resv) 1845 return 1; 1846 1847 idx = vma_hugecache_offset(h, vma, addr); 1848 switch (mode) { 1849 case VMA_NEEDS_RESV: 1850 ret = region_chg(resv, idx, idx + 1); 1851 break; 1852 case VMA_COMMIT_RESV: 1853 ret = region_add(resv, idx, idx + 1); 1854 break; 1855 case VMA_END_RESV: 1856 region_abort(resv, idx, idx + 1); 1857 ret = 0; 1858 break; 1859 default: 1860 BUG(); 1861 } 1862 1863 if (vma->vm_flags & VM_MAYSHARE) 1864 return ret; 1865 else if (is_vma_resv_set(vma, HPAGE_RESV_OWNER) && ret >= 0) { 1866 /* 1867 * In most cases, reserves always exist for private mappings. 1868 * However, a file associated with mapping could have been 1869 * hole punched or truncated after reserves were consumed. 1870 * As subsequent fault on such a range will not use reserves. 1871 * Subtle - The reserve map for private mappings has the 1872 * opposite meaning than that of shared mappings. If NO 1873 * entry is in the reserve map, it means a reservation exists. 1874 * If an entry exists in the reserve map, it means the 1875 * reservation has already been consumed. As a result, the 1876 * return value of this routine is the opposite of the 1877 * value returned from reserve map manipulation routines above. 1878 */ 1879 if (ret) 1880 return 0; 1881 else 1882 return 1; 1883 } 1884 else 1885 return ret < 0 ? ret : 0; 1886 } 1887 1888 static long vma_needs_reservation(struct hstate *h, 1889 struct vm_area_struct *vma, unsigned long addr) 1890 { 1891 return __vma_reservation_common(h, vma, addr, VMA_NEEDS_RESV); 1892 } 1893 1894 static long vma_commit_reservation(struct hstate *h, 1895 struct vm_area_struct *vma, unsigned long addr) 1896 { 1897 return __vma_reservation_common(h, vma, addr, VMA_COMMIT_RESV); 1898 } 1899 1900 static void vma_end_reservation(struct hstate *h, 1901 struct vm_area_struct *vma, unsigned long addr) 1902 { 1903 (void)__vma_reservation_common(h, vma, addr, VMA_END_RESV); 1904 } 1905 1906 struct page *alloc_huge_page(struct vm_area_struct *vma, 1907 unsigned long addr, int avoid_reserve) 1908 { 1909 struct hugepage_subpool *spool = subpool_vma(vma); 1910 struct hstate *h = hstate_vma(vma); 1911 struct page *page; 1912 long map_chg, map_commit; 1913 long gbl_chg; 1914 int ret, idx; 1915 struct hugetlb_cgroup *h_cg; 1916 1917 idx = hstate_index(h); 1918 /* 1919 * Examine the region/reserve map to determine if the process 1920 * has a reservation for the page to be allocated. A return 1921 * code of zero indicates a reservation exists (no change). 1922 */ 1923 map_chg = gbl_chg = vma_needs_reservation(h, vma, addr); 1924 if (map_chg < 0) 1925 return ERR_PTR(-ENOMEM); 1926 1927 /* 1928 * Processes that did not create the mapping will have no 1929 * reserves as indicated by the region/reserve map. Check 1930 * that the allocation will not exceed the subpool limit. 1931 * Allocations for MAP_NORESERVE mappings also need to be 1932 * checked against any subpool limit. 1933 */ 1934 if (map_chg || avoid_reserve) { 1935 gbl_chg = hugepage_subpool_get_pages(spool, 1); 1936 if (gbl_chg < 0) { 1937 vma_end_reservation(h, vma, addr); 1938 return ERR_PTR(-ENOSPC); 1939 } 1940 1941 /* 1942 * Even though there was no reservation in the region/reserve 1943 * map, there could be reservations associated with the 1944 * subpool that can be used. This would be indicated if the 1945 * return value of hugepage_subpool_get_pages() is zero. 1946 * However, if avoid_reserve is specified we still avoid even 1947 * the subpool reservations. 1948 */ 1949 if (avoid_reserve) 1950 gbl_chg = 1; 1951 } 1952 1953 ret = hugetlb_cgroup_charge_cgroup(idx, pages_per_huge_page(h), &h_cg); 1954 if (ret) 1955 goto out_subpool_put; 1956 1957 spin_lock(&hugetlb_lock); 1958 /* 1959 * glb_chg is passed to indicate whether or not a page must be taken 1960 * from the global free pool (global change). gbl_chg == 0 indicates 1961 * a reservation exists for the allocation. 1962 */ 1963 page = dequeue_huge_page_vma(h, vma, addr, avoid_reserve, gbl_chg); 1964 if (!page) { 1965 spin_unlock(&hugetlb_lock); 1966 page = __alloc_buddy_huge_page_with_mpol(h, vma, addr); 1967 if (!page) 1968 goto out_uncharge_cgroup; 1969 if (!avoid_reserve && vma_has_reserves(vma, gbl_chg)) { 1970 SetPagePrivate(page); 1971 h->resv_huge_pages--; 1972 } 1973 spin_lock(&hugetlb_lock); 1974 list_move(&page->lru, &h->hugepage_activelist); 1975 /* Fall through */ 1976 } 1977 hugetlb_cgroup_commit_charge(idx, pages_per_huge_page(h), h_cg, page); 1978 spin_unlock(&hugetlb_lock); 1979 1980 set_page_private(page, (unsigned long)spool); 1981 1982 map_commit = vma_commit_reservation(h, vma, addr); 1983 if (unlikely(map_chg > map_commit)) { 1984 /* 1985 * The page was added to the reservation map between 1986 * vma_needs_reservation and vma_commit_reservation. 1987 * This indicates a race with hugetlb_reserve_pages. 1988 * Adjust for the subpool count incremented above AND 1989 * in hugetlb_reserve_pages for the same page. Also, 1990 * the reservation count added in hugetlb_reserve_pages 1991 * no longer applies. 1992 */ 1993 long rsv_adjust; 1994 1995 rsv_adjust = hugepage_subpool_put_pages(spool, 1); 1996 hugetlb_acct_memory(h, -rsv_adjust); 1997 } 1998 return page; 1999 2000 out_uncharge_cgroup: 2001 hugetlb_cgroup_uncharge_cgroup(idx, pages_per_huge_page(h), h_cg); 2002 out_subpool_put: 2003 if (map_chg || avoid_reserve) 2004 hugepage_subpool_put_pages(spool, 1); 2005 vma_end_reservation(h, vma, addr); 2006 return ERR_PTR(-ENOSPC); 2007 } 2008 2009 /* 2010 * alloc_huge_page()'s wrapper which simply returns the page if allocation 2011 * succeeds, otherwise NULL. This function is called from new_vma_page(), 2012 * where no ERR_VALUE is expected to be returned. 2013 */ 2014 struct page *alloc_huge_page_noerr(struct vm_area_struct *vma, 2015 unsigned long addr, int avoid_reserve) 2016 { 2017 struct page *page = alloc_huge_page(vma, addr, avoid_reserve); 2018 if (IS_ERR(page)) 2019 page = NULL; 2020 return page; 2021 } 2022 2023 int __weak alloc_bootmem_huge_page(struct hstate *h) 2024 { 2025 struct huge_bootmem_page *m; 2026 int nr_nodes, node; 2027 2028 for_each_node_mask_to_alloc(h, nr_nodes, node, &node_states[N_MEMORY]) { 2029 void *addr; 2030 2031 addr = memblock_virt_alloc_try_nid_nopanic( 2032 huge_page_size(h), huge_page_size(h), 2033 0, BOOTMEM_ALLOC_ACCESSIBLE, node); 2034 if (addr) { 2035 /* 2036 * Use the beginning of the huge page to store the 2037 * huge_bootmem_page struct (until gather_bootmem 2038 * puts them into the mem_map). 2039 */ 2040 m = addr; 2041 goto found; 2042 } 2043 } 2044 return 0; 2045 2046 found: 2047 BUG_ON(!IS_ALIGNED(virt_to_phys(m), huge_page_size(h))); 2048 /* Put them into a private list first because mem_map is not up yet */ 2049 list_add(&m->list, &huge_boot_pages); 2050 m->hstate = h; 2051 return 1; 2052 } 2053 2054 static void __init prep_compound_huge_page(struct page *page, 2055 unsigned int order) 2056 { 2057 if (unlikely(order > (MAX_ORDER - 1))) 2058 prep_compound_gigantic_page(page, order); 2059 else 2060 prep_compound_page(page, order); 2061 } 2062 2063 /* Put bootmem huge pages into the standard lists after mem_map is up */ 2064 static void __init gather_bootmem_prealloc(void) 2065 { 2066 struct huge_bootmem_page *m; 2067 2068 list_for_each_entry(m, &huge_boot_pages, list) { 2069 struct hstate *h = m->hstate; 2070 struct page *page; 2071 2072 #ifdef CONFIG_HIGHMEM 2073 page = pfn_to_page(m->phys >> PAGE_SHIFT); 2074 memblock_free_late(__pa(m), 2075 sizeof(struct huge_bootmem_page)); 2076 #else 2077 page = virt_to_page(m); 2078 #endif 2079 WARN_ON(page_count(page) != 1); 2080 prep_compound_huge_page(page, h->order); 2081 WARN_ON(PageReserved(page)); 2082 prep_new_huge_page(h, page, page_to_nid(page)); 2083 /* 2084 * If we had gigantic hugepages allocated at boot time, we need 2085 * to restore the 'stolen' pages to totalram_pages in order to 2086 * fix confusing memory reports from free(1) and another 2087 * side-effects, like CommitLimit going negative. 2088 */ 2089 if (hstate_is_gigantic(h)) 2090 adjust_managed_page_count(page, 1 << h->order); 2091 } 2092 } 2093 2094 static void __init hugetlb_hstate_alloc_pages(struct hstate *h) 2095 { 2096 unsigned long i; 2097 2098 for (i = 0; i < h->max_huge_pages; ++i) { 2099 if (hstate_is_gigantic(h)) { 2100 if (!alloc_bootmem_huge_page(h)) 2101 break; 2102 } else if (!alloc_fresh_huge_page(h, 2103 &node_states[N_MEMORY])) 2104 break; 2105 } 2106 h->max_huge_pages = i; 2107 } 2108 2109 static void __init hugetlb_init_hstates(void) 2110 { 2111 struct hstate *h; 2112 2113 for_each_hstate(h) { 2114 if (minimum_order > huge_page_order(h)) 2115 minimum_order = huge_page_order(h); 2116 2117 /* oversize hugepages were init'ed in early boot */ 2118 if (!hstate_is_gigantic(h)) 2119 hugetlb_hstate_alloc_pages(h); 2120 } 2121 VM_BUG_ON(minimum_order == UINT_MAX); 2122 } 2123 2124 static char * __init memfmt(char *buf, unsigned long n) 2125 { 2126 if (n >= (1UL << 30)) 2127 sprintf(buf, "%lu GB", n >> 30); 2128 else if (n >= (1UL << 20)) 2129 sprintf(buf, "%lu MB", n >> 20); 2130 else 2131 sprintf(buf, "%lu KB", n >> 10); 2132 return buf; 2133 } 2134 2135 static void __init report_hugepages(void) 2136 { 2137 struct hstate *h; 2138 2139 for_each_hstate(h) { 2140 char buf[32]; 2141 pr_info("HugeTLB registered %s page size, pre-allocated %ld pages\n", 2142 memfmt(buf, huge_page_size(h)), 2143 h->free_huge_pages); 2144 } 2145 } 2146 2147 #ifdef CONFIG_HIGHMEM 2148 static void try_to_free_low(struct hstate *h, unsigned long count, 2149 nodemask_t *nodes_allowed) 2150 { 2151 int i; 2152 2153 if (hstate_is_gigantic(h)) 2154 return; 2155 2156 for_each_node_mask(i, *nodes_allowed) { 2157 struct page *page, *next; 2158 struct list_head *freel = &h->hugepage_freelists[i]; 2159 list_for_each_entry_safe(page, next, freel, lru) { 2160 if (count >= h->nr_huge_pages) 2161 return; 2162 if (PageHighMem(page)) 2163 continue; 2164 list_del(&page->lru); 2165 update_and_free_page(h, page); 2166 h->free_huge_pages--; 2167 h->free_huge_pages_node[page_to_nid(page)]--; 2168 } 2169 } 2170 } 2171 #else 2172 static inline void try_to_free_low(struct hstate *h, unsigned long count, 2173 nodemask_t *nodes_allowed) 2174 { 2175 } 2176 #endif 2177 2178 /* 2179 * Increment or decrement surplus_huge_pages. Keep node-specific counters 2180 * balanced by operating on them in a round-robin fashion. 2181 * Returns 1 if an adjustment was made. 2182 */ 2183 static int adjust_pool_surplus(struct hstate *h, nodemask_t *nodes_allowed, 2184 int delta) 2185 { 2186 int nr_nodes, node; 2187 2188 VM_BUG_ON(delta != -1 && delta != 1); 2189 2190 if (delta < 0) { 2191 for_each_node_mask_to_alloc(h, nr_nodes, node, nodes_allowed) { 2192 if (h->surplus_huge_pages_node[node]) 2193 goto found; 2194 } 2195 } else { 2196 for_each_node_mask_to_free(h, nr_nodes, node, nodes_allowed) { 2197 if (h->surplus_huge_pages_node[node] < 2198 h->nr_huge_pages_node[node]) 2199 goto found; 2200 } 2201 } 2202 return 0; 2203 2204 found: 2205 h->surplus_huge_pages += delta; 2206 h->surplus_huge_pages_node[node] += delta; 2207 return 1; 2208 } 2209 2210 #define persistent_huge_pages(h) (h->nr_huge_pages - h->surplus_huge_pages) 2211 static unsigned long set_max_huge_pages(struct hstate *h, unsigned long count, 2212 nodemask_t *nodes_allowed) 2213 { 2214 unsigned long min_count, ret; 2215 2216 if (hstate_is_gigantic(h) && !gigantic_page_supported()) 2217 return h->max_huge_pages; 2218 2219 /* 2220 * Increase the pool size 2221 * First take pages out of surplus state. Then make up the 2222 * remaining difference by allocating fresh huge pages. 2223 * 2224 * We might race with __alloc_buddy_huge_page() here and be unable 2225 * to convert a surplus huge page to a normal huge page. That is 2226 * not critical, though, it just means the overall size of the 2227 * pool might be one hugepage larger than it needs to be, but 2228 * within all the constraints specified by the sysctls. 2229 */ 2230 spin_lock(&hugetlb_lock); 2231 while (h->surplus_huge_pages && count > persistent_huge_pages(h)) { 2232 if (!adjust_pool_surplus(h, nodes_allowed, -1)) 2233 break; 2234 } 2235 2236 while (count > persistent_huge_pages(h)) { 2237 /* 2238 * If this allocation races such that we no longer need the 2239 * page, free_huge_page will handle it by freeing the page 2240 * and reducing the surplus. 2241 */ 2242 spin_unlock(&hugetlb_lock); 2243 2244 /* yield cpu to avoid soft lockup */ 2245 cond_resched(); 2246 2247 if (hstate_is_gigantic(h)) 2248 ret = alloc_fresh_gigantic_page(h, nodes_allowed); 2249 else 2250 ret = alloc_fresh_huge_page(h, nodes_allowed); 2251 spin_lock(&hugetlb_lock); 2252 if (!ret) 2253 goto out; 2254 2255 /* Bail for signals. Probably ctrl-c from user */ 2256 if (signal_pending(current)) 2257 goto out; 2258 } 2259 2260 /* 2261 * Decrease the pool size 2262 * First return free pages to the buddy allocator (being careful 2263 * to keep enough around to satisfy reservations). Then place 2264 * pages into surplus state as needed so the pool will shrink 2265 * to the desired size as pages become free. 2266 * 2267 * By placing pages into the surplus state independent of the 2268 * overcommit value, we are allowing the surplus pool size to 2269 * exceed overcommit. There are few sane options here. Since 2270 * __alloc_buddy_huge_page() is checking the global counter, 2271 * though, we'll note that we're not allowed to exceed surplus 2272 * and won't grow the pool anywhere else. Not until one of the 2273 * sysctls are changed, or the surplus pages go out of use. 2274 */ 2275 min_count = h->resv_huge_pages + h->nr_huge_pages - h->free_huge_pages; 2276 min_count = max(count, min_count); 2277 try_to_free_low(h, min_count, nodes_allowed); 2278 while (min_count < persistent_huge_pages(h)) { 2279 if (!free_pool_huge_page(h, nodes_allowed, 0)) 2280 break; 2281 cond_resched_lock(&hugetlb_lock); 2282 } 2283 while (count < persistent_huge_pages(h)) { 2284 if (!adjust_pool_surplus(h, nodes_allowed, 1)) 2285 break; 2286 } 2287 out: 2288 ret = persistent_huge_pages(h); 2289 spin_unlock(&hugetlb_lock); 2290 return ret; 2291 } 2292 2293 #define HSTATE_ATTR_RO(_name) \ 2294 static struct kobj_attribute _name##_attr = __ATTR_RO(_name) 2295 2296 #define HSTATE_ATTR(_name) \ 2297 static struct kobj_attribute _name##_attr = \ 2298 __ATTR(_name, 0644, _name##_show, _name##_store) 2299 2300 static struct kobject *hugepages_kobj; 2301 static struct kobject *hstate_kobjs[HUGE_MAX_HSTATE]; 2302 2303 static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp); 2304 2305 static struct hstate *kobj_to_hstate(struct kobject *kobj, int *nidp) 2306 { 2307 int i; 2308 2309 for (i = 0; i < HUGE_MAX_HSTATE; i++) 2310 if (hstate_kobjs[i] == kobj) { 2311 if (nidp) 2312 *nidp = NUMA_NO_NODE; 2313 return &hstates[i]; 2314 } 2315 2316 return kobj_to_node_hstate(kobj, nidp); 2317 } 2318 2319 static ssize_t nr_hugepages_show_common(struct kobject *kobj, 2320 struct kobj_attribute *attr, char *buf) 2321 { 2322 struct hstate *h; 2323 unsigned long nr_huge_pages; 2324 int nid; 2325 2326 h = kobj_to_hstate(kobj, &nid); 2327 if (nid == NUMA_NO_NODE) 2328 nr_huge_pages = h->nr_huge_pages; 2329 else 2330 nr_huge_pages = h->nr_huge_pages_node[nid]; 2331 2332 return sprintf(buf, "%lu\n", nr_huge_pages); 2333 } 2334 2335 static ssize_t __nr_hugepages_store_common(bool obey_mempolicy, 2336 struct hstate *h, int nid, 2337 unsigned long count, size_t len) 2338 { 2339 int err; 2340 NODEMASK_ALLOC(nodemask_t, nodes_allowed, GFP_KERNEL | __GFP_NORETRY); 2341 2342 if (hstate_is_gigantic(h) && !gigantic_page_supported()) { 2343 err = -EINVAL; 2344 goto out; 2345 } 2346 2347 if (nid == NUMA_NO_NODE) { 2348 /* 2349 * global hstate attribute 2350 */ 2351 if (!(obey_mempolicy && 2352 init_nodemask_of_mempolicy(nodes_allowed))) { 2353 NODEMASK_FREE(nodes_allowed); 2354 nodes_allowed = &node_states[N_MEMORY]; 2355 } 2356 } else if (nodes_allowed) { 2357 /* 2358 * per node hstate attribute: adjust count to global, 2359 * but restrict alloc/free to the specified node. 2360 */ 2361 count += h->nr_huge_pages - h->nr_huge_pages_node[nid]; 2362 init_nodemask_of_node(nodes_allowed, nid); 2363 } else 2364 nodes_allowed = &node_states[N_MEMORY]; 2365 2366 h->max_huge_pages = set_max_huge_pages(h, count, nodes_allowed); 2367 2368 if (nodes_allowed != &node_states[N_MEMORY]) 2369 NODEMASK_FREE(nodes_allowed); 2370 2371 return len; 2372 out: 2373 NODEMASK_FREE(nodes_allowed); 2374 return err; 2375 } 2376 2377 static ssize_t nr_hugepages_store_common(bool obey_mempolicy, 2378 struct kobject *kobj, const char *buf, 2379 size_t len) 2380 { 2381 struct hstate *h; 2382 unsigned long count; 2383 int nid; 2384 int err; 2385 2386 err = kstrtoul(buf, 10, &count); 2387 if (err) 2388 return err; 2389 2390 h = kobj_to_hstate(kobj, &nid); 2391 return __nr_hugepages_store_common(obey_mempolicy, h, nid, count, len); 2392 } 2393 2394 static ssize_t nr_hugepages_show(struct kobject *kobj, 2395 struct kobj_attribute *attr, char *buf) 2396 { 2397 return nr_hugepages_show_common(kobj, attr, buf); 2398 } 2399 2400 static ssize_t nr_hugepages_store(struct kobject *kobj, 2401 struct kobj_attribute *attr, const char *buf, size_t len) 2402 { 2403 return nr_hugepages_store_common(false, kobj, buf, len); 2404 } 2405 HSTATE_ATTR(nr_hugepages); 2406 2407 #ifdef CONFIG_NUMA 2408 2409 /* 2410 * hstate attribute for optionally mempolicy-based constraint on persistent 2411 * huge page alloc/free. 2412 */ 2413 static ssize_t nr_hugepages_mempolicy_show(struct kobject *kobj, 2414 struct kobj_attribute *attr, char *buf) 2415 { 2416 return nr_hugepages_show_common(kobj, attr, buf); 2417 } 2418 2419 static ssize_t nr_hugepages_mempolicy_store(struct kobject *kobj, 2420 struct kobj_attribute *attr, const char *buf, size_t len) 2421 { 2422 return nr_hugepages_store_common(true, kobj, buf, len); 2423 } 2424 HSTATE_ATTR(nr_hugepages_mempolicy); 2425 #endif 2426 2427 2428 static ssize_t nr_overcommit_hugepages_show(struct kobject *kobj, 2429 struct kobj_attribute *attr, char *buf) 2430 { 2431 struct hstate *h = kobj_to_hstate(kobj, NULL); 2432 return sprintf(buf, "%lu\n", h->nr_overcommit_huge_pages); 2433 } 2434 2435 static ssize_t nr_overcommit_hugepages_store(struct kobject *kobj, 2436 struct kobj_attribute *attr, const char *buf, size_t count) 2437 { 2438 int err; 2439 unsigned long input; 2440 struct hstate *h = kobj_to_hstate(kobj, NULL); 2441 2442 if (hstate_is_gigantic(h)) 2443 return -EINVAL; 2444 2445 err = kstrtoul(buf, 10, &input); 2446 if (err) 2447 return err; 2448 2449 spin_lock(&hugetlb_lock); 2450 h->nr_overcommit_huge_pages = input; 2451 spin_unlock(&hugetlb_lock); 2452 2453 return count; 2454 } 2455 HSTATE_ATTR(nr_overcommit_hugepages); 2456 2457 static ssize_t free_hugepages_show(struct kobject *kobj, 2458 struct kobj_attribute *attr, char *buf) 2459 { 2460 struct hstate *h; 2461 unsigned long free_huge_pages; 2462 int nid; 2463 2464 h = kobj_to_hstate(kobj, &nid); 2465 if (nid == NUMA_NO_NODE) 2466 free_huge_pages = h->free_huge_pages; 2467 else 2468 free_huge_pages = h->free_huge_pages_node[nid]; 2469 2470 return sprintf(buf, "%lu\n", free_huge_pages); 2471 } 2472 HSTATE_ATTR_RO(free_hugepages); 2473 2474 static ssize_t resv_hugepages_show(struct kobject *kobj, 2475 struct kobj_attribute *attr, char *buf) 2476 { 2477 struct hstate *h = kobj_to_hstate(kobj, NULL); 2478 return sprintf(buf, "%lu\n", h->resv_huge_pages); 2479 } 2480 HSTATE_ATTR_RO(resv_hugepages); 2481 2482 static ssize_t surplus_hugepages_show(struct kobject *kobj, 2483 struct kobj_attribute *attr, char *buf) 2484 { 2485 struct hstate *h; 2486 unsigned long surplus_huge_pages; 2487 int nid; 2488 2489 h = kobj_to_hstate(kobj, &nid); 2490 if (nid == NUMA_NO_NODE) 2491 surplus_huge_pages = h->surplus_huge_pages; 2492 else 2493 surplus_huge_pages = h->surplus_huge_pages_node[nid]; 2494 2495 return sprintf(buf, "%lu\n", surplus_huge_pages); 2496 } 2497 HSTATE_ATTR_RO(surplus_hugepages); 2498 2499 static struct attribute *hstate_attrs[] = { 2500 &nr_hugepages_attr.attr, 2501 &nr_overcommit_hugepages_attr.attr, 2502 &free_hugepages_attr.attr, 2503 &resv_hugepages_attr.attr, 2504 &surplus_hugepages_attr.attr, 2505 #ifdef CONFIG_NUMA 2506 &nr_hugepages_mempolicy_attr.attr, 2507 #endif 2508 NULL, 2509 }; 2510 2511 static struct attribute_group hstate_attr_group = { 2512 .attrs = hstate_attrs, 2513 }; 2514 2515 static int hugetlb_sysfs_add_hstate(struct hstate *h, struct kobject *parent, 2516 struct kobject **hstate_kobjs, 2517 struct attribute_group *hstate_attr_group) 2518 { 2519 int retval; 2520 int hi = hstate_index(h); 2521 2522 hstate_kobjs[hi] = kobject_create_and_add(h->name, parent); 2523 if (!hstate_kobjs[hi]) 2524 return -ENOMEM; 2525 2526 retval = sysfs_create_group(hstate_kobjs[hi], hstate_attr_group); 2527 if (retval) 2528 kobject_put(hstate_kobjs[hi]); 2529 2530 return retval; 2531 } 2532 2533 static void __init hugetlb_sysfs_init(void) 2534 { 2535 struct hstate *h; 2536 int err; 2537 2538 hugepages_kobj = kobject_create_and_add("hugepages", mm_kobj); 2539 if (!hugepages_kobj) 2540 return; 2541 2542 for_each_hstate(h) { 2543 err = hugetlb_sysfs_add_hstate(h, hugepages_kobj, 2544 hstate_kobjs, &hstate_attr_group); 2545 if (err) 2546 pr_err("Hugetlb: Unable to add hstate %s", h->name); 2547 } 2548 } 2549 2550 #ifdef CONFIG_NUMA 2551 2552 /* 2553 * node_hstate/s - associate per node hstate attributes, via their kobjects, 2554 * with node devices in node_devices[] using a parallel array. The array 2555 * index of a node device or _hstate == node id. 2556 * This is here to avoid any static dependency of the node device driver, in 2557 * the base kernel, on the hugetlb module. 2558 */ 2559 struct node_hstate { 2560 struct kobject *hugepages_kobj; 2561 struct kobject *hstate_kobjs[HUGE_MAX_HSTATE]; 2562 }; 2563 static struct node_hstate node_hstates[MAX_NUMNODES]; 2564 2565 /* 2566 * A subset of global hstate attributes for node devices 2567 */ 2568 static struct attribute *per_node_hstate_attrs[] = { 2569 &nr_hugepages_attr.attr, 2570 &free_hugepages_attr.attr, 2571 &surplus_hugepages_attr.attr, 2572 NULL, 2573 }; 2574 2575 static struct attribute_group per_node_hstate_attr_group = { 2576 .attrs = per_node_hstate_attrs, 2577 }; 2578 2579 /* 2580 * kobj_to_node_hstate - lookup global hstate for node device hstate attr kobj. 2581 * Returns node id via non-NULL nidp. 2582 */ 2583 static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp) 2584 { 2585 int nid; 2586 2587 for (nid = 0; nid < nr_node_ids; nid++) { 2588 struct node_hstate *nhs = &node_hstates[nid]; 2589 int i; 2590 for (i = 0; i < HUGE_MAX_HSTATE; i++) 2591 if (nhs->hstate_kobjs[i] == kobj) { 2592 if (nidp) 2593 *nidp = nid; 2594 return &hstates[i]; 2595 } 2596 } 2597 2598 BUG(); 2599 return NULL; 2600 } 2601 2602 /* 2603 * Unregister hstate attributes from a single node device. 2604 * No-op if no hstate attributes attached. 2605 */ 2606 static void hugetlb_unregister_node(struct node *node) 2607 { 2608 struct hstate *h; 2609 struct node_hstate *nhs = &node_hstates[node->dev.id]; 2610 2611 if (!nhs->hugepages_kobj) 2612 return; /* no hstate attributes */ 2613 2614 for_each_hstate(h) { 2615 int idx = hstate_index(h); 2616 if (nhs->hstate_kobjs[idx]) { 2617 kobject_put(nhs->hstate_kobjs[idx]); 2618 nhs->hstate_kobjs[idx] = NULL; 2619 } 2620 } 2621 2622 kobject_put(nhs->hugepages_kobj); 2623 nhs->hugepages_kobj = NULL; 2624 } 2625 2626 2627 /* 2628 * Register hstate attributes for a single node device. 2629 * No-op if attributes already registered. 2630 */ 2631 static void hugetlb_register_node(struct node *node) 2632 { 2633 struct hstate *h; 2634 struct node_hstate *nhs = &node_hstates[node->dev.id]; 2635 int err; 2636 2637 if (nhs->hugepages_kobj) 2638 return; /* already allocated */ 2639 2640 nhs->hugepages_kobj = kobject_create_and_add("hugepages", 2641 &node->dev.kobj); 2642 if (!nhs->hugepages_kobj) 2643 return; 2644 2645 for_each_hstate(h) { 2646 err = hugetlb_sysfs_add_hstate(h, nhs->hugepages_kobj, 2647 nhs->hstate_kobjs, 2648 &per_node_hstate_attr_group); 2649 if (err) { 2650 pr_err("Hugetlb: Unable to add hstate %s for node %d\n", 2651 h->name, node->dev.id); 2652 hugetlb_unregister_node(node); 2653 break; 2654 } 2655 } 2656 } 2657 2658 /* 2659 * hugetlb init time: register hstate attributes for all registered node 2660 * devices of nodes that have memory. All on-line nodes should have 2661 * registered their associated device by this time. 2662 */ 2663 static void __init hugetlb_register_all_nodes(void) 2664 { 2665 int nid; 2666 2667 for_each_node_state(nid, N_MEMORY) { 2668 struct node *node = node_devices[nid]; 2669 if (node->dev.id == nid) 2670 hugetlb_register_node(node); 2671 } 2672 2673 /* 2674 * Let the node device driver know we're here so it can 2675 * [un]register hstate attributes on node hotplug. 2676 */ 2677 register_hugetlbfs_with_node(hugetlb_register_node, 2678 hugetlb_unregister_node); 2679 } 2680 #else /* !CONFIG_NUMA */ 2681 2682 static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp) 2683 { 2684 BUG(); 2685 if (nidp) 2686 *nidp = -1; 2687 return NULL; 2688 } 2689 2690 static void hugetlb_register_all_nodes(void) { } 2691 2692 #endif 2693 2694 static int __init hugetlb_init(void) 2695 { 2696 int i; 2697 2698 if (!hugepages_supported()) 2699 return 0; 2700 2701 if (!size_to_hstate(default_hstate_size)) { 2702 default_hstate_size = HPAGE_SIZE; 2703 if (!size_to_hstate(default_hstate_size)) 2704 hugetlb_add_hstate(HUGETLB_PAGE_ORDER); 2705 } 2706 default_hstate_idx = hstate_index(size_to_hstate(default_hstate_size)); 2707 if (default_hstate_max_huge_pages) { 2708 if (!default_hstate.max_huge_pages) 2709 default_hstate.max_huge_pages = default_hstate_max_huge_pages; 2710 } 2711 2712 hugetlb_init_hstates(); 2713 gather_bootmem_prealloc(); 2714 report_hugepages(); 2715 2716 hugetlb_sysfs_init(); 2717 hugetlb_register_all_nodes(); 2718 hugetlb_cgroup_file_init(); 2719 2720 #ifdef CONFIG_SMP 2721 num_fault_mutexes = roundup_pow_of_two(8 * num_possible_cpus()); 2722 #else 2723 num_fault_mutexes = 1; 2724 #endif 2725 hugetlb_fault_mutex_table = 2726 kmalloc(sizeof(struct mutex) * num_fault_mutexes, GFP_KERNEL); 2727 BUG_ON(!hugetlb_fault_mutex_table); 2728 2729 for (i = 0; i < num_fault_mutexes; i++) 2730 mutex_init(&hugetlb_fault_mutex_table[i]); 2731 return 0; 2732 } 2733 subsys_initcall(hugetlb_init); 2734 2735 /* Should be called on processing a hugepagesz=... option */ 2736 void __init hugetlb_bad_size(void) 2737 { 2738 parsed_valid_hugepagesz = false; 2739 } 2740 2741 void __init hugetlb_add_hstate(unsigned int order) 2742 { 2743 struct hstate *h; 2744 unsigned long i; 2745 2746 if (size_to_hstate(PAGE_SIZE << order)) { 2747 pr_warn("hugepagesz= specified twice, ignoring\n"); 2748 return; 2749 } 2750 BUG_ON(hugetlb_max_hstate >= HUGE_MAX_HSTATE); 2751 BUG_ON(order == 0); 2752 h = &hstates[hugetlb_max_hstate++]; 2753 h->order = order; 2754 h->mask = ~((1ULL << (order + PAGE_SHIFT)) - 1); 2755 h->nr_huge_pages = 0; 2756 h->free_huge_pages = 0; 2757 for (i = 0; i < MAX_NUMNODES; ++i) 2758 INIT_LIST_HEAD(&h->hugepage_freelists[i]); 2759 INIT_LIST_HEAD(&h->hugepage_activelist); 2760 h->next_nid_to_alloc = first_memory_node; 2761 h->next_nid_to_free = first_memory_node; 2762 snprintf(h->name, HSTATE_NAME_LEN, "hugepages-%lukB", 2763 huge_page_size(h)/1024); 2764 2765 parsed_hstate = h; 2766 } 2767 2768 static int __init hugetlb_nrpages_setup(char *s) 2769 { 2770 unsigned long *mhp; 2771 static unsigned long *last_mhp; 2772 2773 if (!parsed_valid_hugepagesz) { 2774 pr_warn("hugepages = %s preceded by " 2775 "an unsupported hugepagesz, ignoring\n", s); 2776 parsed_valid_hugepagesz = true; 2777 return 1; 2778 } 2779 /* 2780 * !hugetlb_max_hstate means we haven't parsed a hugepagesz= parameter yet, 2781 * so this hugepages= parameter goes to the "default hstate". 2782 */ 2783 else if (!hugetlb_max_hstate) 2784 mhp = &default_hstate_max_huge_pages; 2785 else 2786 mhp = &parsed_hstate->max_huge_pages; 2787 2788 if (mhp == last_mhp) { 2789 pr_warn("hugepages= specified twice without interleaving hugepagesz=, ignoring\n"); 2790 return 1; 2791 } 2792 2793 if (sscanf(s, "%lu", mhp) <= 0) 2794 *mhp = 0; 2795 2796 /* 2797 * Global state is always initialized later in hugetlb_init. 2798 * But we need to allocate >= MAX_ORDER hstates here early to still 2799 * use the bootmem allocator. 2800 */ 2801 if (hugetlb_max_hstate && parsed_hstate->order >= MAX_ORDER) 2802 hugetlb_hstate_alloc_pages(parsed_hstate); 2803 2804 last_mhp = mhp; 2805 2806 return 1; 2807 } 2808 __setup("hugepages=", hugetlb_nrpages_setup); 2809 2810 static int __init hugetlb_default_setup(char *s) 2811 { 2812 default_hstate_size = memparse(s, &s); 2813 return 1; 2814 } 2815 __setup("default_hugepagesz=", hugetlb_default_setup); 2816 2817 static unsigned int cpuset_mems_nr(unsigned int *array) 2818 { 2819 int node; 2820 unsigned int nr = 0; 2821 2822 for_each_node_mask(node, cpuset_current_mems_allowed) 2823 nr += array[node]; 2824 2825 return nr; 2826 } 2827 2828 #ifdef CONFIG_SYSCTL 2829 static int hugetlb_sysctl_handler_common(bool obey_mempolicy, 2830 struct ctl_table *table, int write, 2831 void __user *buffer, size_t *length, loff_t *ppos) 2832 { 2833 struct hstate *h = &default_hstate; 2834 unsigned long tmp = h->max_huge_pages; 2835 int ret; 2836 2837 if (!hugepages_supported()) 2838 return -EOPNOTSUPP; 2839 2840 table->data = &tmp; 2841 table->maxlen = sizeof(unsigned long); 2842 ret = proc_doulongvec_minmax(table, write, buffer, length, ppos); 2843 if (ret) 2844 goto out; 2845 2846 if (write) 2847 ret = __nr_hugepages_store_common(obey_mempolicy, h, 2848 NUMA_NO_NODE, tmp, *length); 2849 out: 2850 return ret; 2851 } 2852 2853 int hugetlb_sysctl_handler(struct ctl_table *table, int write, 2854 void __user *buffer, size_t *length, loff_t *ppos) 2855 { 2856 2857 return hugetlb_sysctl_handler_common(false, table, write, 2858 buffer, length, ppos); 2859 } 2860 2861 #ifdef CONFIG_NUMA 2862 int hugetlb_mempolicy_sysctl_handler(struct ctl_table *table, int write, 2863 void __user *buffer, size_t *length, loff_t *ppos) 2864 { 2865 return hugetlb_sysctl_handler_common(true, table, write, 2866 buffer, length, ppos); 2867 } 2868 #endif /* CONFIG_NUMA */ 2869 2870 int hugetlb_overcommit_handler(struct ctl_table *table, int write, 2871 void __user *buffer, 2872 size_t *length, loff_t *ppos) 2873 { 2874 struct hstate *h = &default_hstate; 2875 unsigned long tmp; 2876 int ret; 2877 2878 if (!hugepages_supported()) 2879 return -EOPNOTSUPP; 2880 2881 tmp = h->nr_overcommit_huge_pages; 2882 2883 if (write && hstate_is_gigantic(h)) 2884 return -EINVAL; 2885 2886 table->data = &tmp; 2887 table->maxlen = sizeof(unsigned long); 2888 ret = proc_doulongvec_minmax(table, write, buffer, length, ppos); 2889 if (ret) 2890 goto out; 2891 2892 if (write) { 2893 spin_lock(&hugetlb_lock); 2894 h->nr_overcommit_huge_pages = tmp; 2895 spin_unlock(&hugetlb_lock); 2896 } 2897 out: 2898 return ret; 2899 } 2900 2901 #endif /* CONFIG_SYSCTL */ 2902 2903 void hugetlb_report_meminfo(struct seq_file *m) 2904 { 2905 struct hstate *h = &default_hstate; 2906 if (!hugepages_supported()) 2907 return; 2908 seq_printf(m, 2909 "HugePages_Total: %5lu\n" 2910 "HugePages_Free: %5lu\n" 2911 "HugePages_Rsvd: %5lu\n" 2912 "HugePages_Surp: %5lu\n" 2913 "Hugepagesize: %8lu kB\n", 2914 h->nr_huge_pages, 2915 h->free_huge_pages, 2916 h->resv_huge_pages, 2917 h->surplus_huge_pages, 2918 1UL << (huge_page_order(h) + PAGE_SHIFT - 10)); 2919 } 2920 2921 int hugetlb_report_node_meminfo(int nid, char *buf) 2922 { 2923 struct hstate *h = &default_hstate; 2924 if (!hugepages_supported()) 2925 return 0; 2926 return sprintf(buf, 2927 "Node %d HugePages_Total: %5u\n" 2928 "Node %d HugePages_Free: %5u\n" 2929 "Node %d HugePages_Surp: %5u\n", 2930 nid, h->nr_huge_pages_node[nid], 2931 nid, h->free_huge_pages_node[nid], 2932 nid, h->surplus_huge_pages_node[nid]); 2933 } 2934 2935 void hugetlb_show_meminfo(void) 2936 { 2937 struct hstate *h; 2938 int nid; 2939 2940 if (!hugepages_supported()) 2941 return; 2942 2943 for_each_node_state(nid, N_MEMORY) 2944 for_each_hstate(h) 2945 pr_info("Node %d hugepages_total=%u hugepages_free=%u hugepages_surp=%u hugepages_size=%lukB\n", 2946 nid, 2947 h->nr_huge_pages_node[nid], 2948 h->free_huge_pages_node[nid], 2949 h->surplus_huge_pages_node[nid], 2950 1UL << (huge_page_order(h) + PAGE_SHIFT - 10)); 2951 } 2952 2953 void hugetlb_report_usage(struct seq_file *m, struct mm_struct *mm) 2954 { 2955 seq_printf(m, "HugetlbPages:\t%8lu kB\n", 2956 atomic_long_read(&mm->hugetlb_usage) << (PAGE_SHIFT - 10)); 2957 } 2958 2959 /* Return the number pages of memory we physically have, in PAGE_SIZE units. */ 2960 unsigned long hugetlb_total_pages(void) 2961 { 2962 struct hstate *h; 2963 unsigned long nr_total_pages = 0; 2964 2965 for_each_hstate(h) 2966 nr_total_pages += h->nr_huge_pages * pages_per_huge_page(h); 2967 return nr_total_pages; 2968 } 2969 2970 static int hugetlb_acct_memory(struct hstate *h, long delta) 2971 { 2972 int ret = -ENOMEM; 2973 2974 spin_lock(&hugetlb_lock); 2975 /* 2976 * When cpuset is configured, it breaks the strict hugetlb page 2977 * reservation as the accounting is done on a global variable. Such 2978 * reservation is completely rubbish in the presence of cpuset because 2979 * the reservation is not checked against page availability for the 2980 * current cpuset. Application can still potentially OOM'ed by kernel 2981 * with lack of free htlb page in cpuset that the task is in. 2982 * Attempt to enforce strict accounting with cpuset is almost 2983 * impossible (or too ugly) because cpuset is too fluid that 2984 * task or memory node can be dynamically moved between cpusets. 2985 * 2986 * The change of semantics for shared hugetlb mapping with cpuset is 2987 * undesirable. However, in order to preserve some of the semantics, 2988 * we fall back to check against current free page availability as 2989 * a best attempt and hopefully to minimize the impact of changing 2990 * semantics that cpuset has. 2991 */ 2992 if (delta > 0) { 2993 if (gather_surplus_pages(h, delta) < 0) 2994 goto out; 2995 2996 if (delta > cpuset_mems_nr(h->free_huge_pages_node)) { 2997 return_unused_surplus_pages(h, delta); 2998 goto out; 2999 } 3000 } 3001 3002 ret = 0; 3003 if (delta < 0) 3004 return_unused_surplus_pages(h, (unsigned long) -delta); 3005 3006 out: 3007 spin_unlock(&hugetlb_lock); 3008 return ret; 3009 } 3010 3011 static void hugetlb_vm_op_open(struct vm_area_struct *vma) 3012 { 3013 struct resv_map *resv = vma_resv_map(vma); 3014 3015 /* 3016 * This new VMA should share its siblings reservation map if present. 3017 * The VMA will only ever have a valid reservation map pointer where 3018 * it is being copied for another still existing VMA. As that VMA 3019 * has a reference to the reservation map it cannot disappear until 3020 * after this open call completes. It is therefore safe to take a 3021 * new reference here without additional locking. 3022 */ 3023 if (resv && is_vma_resv_set(vma, HPAGE_RESV_OWNER)) 3024 kref_get(&resv->refs); 3025 } 3026 3027 static void hugetlb_vm_op_close(struct vm_area_struct *vma) 3028 { 3029 struct hstate *h = hstate_vma(vma); 3030 struct resv_map *resv = vma_resv_map(vma); 3031 struct hugepage_subpool *spool = subpool_vma(vma); 3032 unsigned long reserve, start, end; 3033 long gbl_reserve; 3034 3035 if (!resv || !is_vma_resv_set(vma, HPAGE_RESV_OWNER)) 3036 return; 3037 3038 start = vma_hugecache_offset(h, vma, vma->vm_start); 3039 end = vma_hugecache_offset(h, vma, vma->vm_end); 3040 3041 reserve = (end - start) - region_count(resv, start, end); 3042 3043 kref_put(&resv->refs, resv_map_release); 3044 3045 if (reserve) { 3046 /* 3047 * Decrement reserve counts. The global reserve count may be 3048 * adjusted if the subpool has a minimum size. 3049 */ 3050 gbl_reserve = hugepage_subpool_put_pages(spool, reserve); 3051 hugetlb_acct_memory(h, -gbl_reserve); 3052 } 3053 } 3054 3055 /* 3056 * We cannot handle pagefaults against hugetlb pages at all. They cause 3057 * handle_mm_fault() to try to instantiate regular-sized pages in the 3058 * hugegpage VMA. do_page_fault() is supposed to trap this, so BUG is we get 3059 * this far. 3060 */ 3061 static int hugetlb_vm_op_fault(struct vm_area_struct *vma, struct vm_fault *vmf) 3062 { 3063 BUG(); 3064 return 0; 3065 } 3066 3067 const struct vm_operations_struct hugetlb_vm_ops = { 3068 .fault = hugetlb_vm_op_fault, 3069 .open = hugetlb_vm_op_open, 3070 .close = hugetlb_vm_op_close, 3071 }; 3072 3073 static pte_t make_huge_pte(struct vm_area_struct *vma, struct page *page, 3074 int writable) 3075 { 3076 pte_t entry; 3077 3078 if (writable) { 3079 entry = huge_pte_mkwrite(huge_pte_mkdirty(mk_huge_pte(page, 3080 vma->vm_page_prot))); 3081 } else { 3082 entry = huge_pte_wrprotect(mk_huge_pte(page, 3083 vma->vm_page_prot)); 3084 } 3085 entry = pte_mkyoung(entry); 3086 entry = pte_mkhuge(entry); 3087 entry = arch_make_huge_pte(entry, vma, page, writable); 3088 3089 return entry; 3090 } 3091 3092 static void set_huge_ptep_writable(struct vm_area_struct *vma, 3093 unsigned long address, pte_t *ptep) 3094 { 3095 pte_t entry; 3096 3097 entry = huge_pte_mkwrite(huge_pte_mkdirty(huge_ptep_get(ptep))); 3098 if (huge_ptep_set_access_flags(vma, address, ptep, entry, 1)) 3099 update_mmu_cache(vma, address, ptep); 3100 } 3101 3102 static int is_hugetlb_entry_migration(pte_t pte) 3103 { 3104 swp_entry_t swp; 3105 3106 if (huge_pte_none(pte) || pte_present(pte)) 3107 return 0; 3108 swp = pte_to_swp_entry(pte); 3109 if (non_swap_entry(swp) && is_migration_entry(swp)) 3110 return 1; 3111 else 3112 return 0; 3113 } 3114 3115 static int is_hugetlb_entry_hwpoisoned(pte_t pte) 3116 { 3117 swp_entry_t swp; 3118 3119 if (huge_pte_none(pte) || pte_present(pte)) 3120 return 0; 3121 swp = pte_to_swp_entry(pte); 3122 if (non_swap_entry(swp) && is_hwpoison_entry(swp)) 3123 return 1; 3124 else 3125 return 0; 3126 } 3127 3128 int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src, 3129 struct vm_area_struct *vma) 3130 { 3131 pte_t *src_pte, *dst_pte, entry; 3132 struct page *ptepage; 3133 unsigned long addr; 3134 int cow; 3135 struct hstate *h = hstate_vma(vma); 3136 unsigned long sz = huge_page_size(h); 3137 unsigned long mmun_start; /* For mmu_notifiers */ 3138 unsigned long mmun_end; /* For mmu_notifiers */ 3139 int ret = 0; 3140 3141 cow = (vma->vm_flags & (VM_SHARED | VM_MAYWRITE)) == VM_MAYWRITE; 3142 3143 mmun_start = vma->vm_start; 3144 mmun_end = vma->vm_end; 3145 if (cow) 3146 mmu_notifier_invalidate_range_start(src, mmun_start, mmun_end); 3147 3148 for (addr = vma->vm_start; addr < vma->vm_end; addr += sz) { 3149 spinlock_t *src_ptl, *dst_ptl; 3150 src_pte = huge_pte_offset(src, addr); 3151 if (!src_pte) 3152 continue; 3153 dst_pte = huge_pte_alloc(dst, addr, sz); 3154 if (!dst_pte) { 3155 ret = -ENOMEM; 3156 break; 3157 } 3158 3159 /* If the pagetables are shared don't copy or take references */ 3160 if (dst_pte == src_pte) 3161 continue; 3162 3163 dst_ptl = huge_pte_lock(h, dst, dst_pte); 3164 src_ptl = huge_pte_lockptr(h, src, src_pte); 3165 spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING); 3166 entry = huge_ptep_get(src_pte); 3167 if (huge_pte_none(entry)) { /* skip none entry */ 3168 ; 3169 } else if (unlikely(is_hugetlb_entry_migration(entry) || 3170 is_hugetlb_entry_hwpoisoned(entry))) { 3171 swp_entry_t swp_entry = pte_to_swp_entry(entry); 3172 3173 if (is_write_migration_entry(swp_entry) && cow) { 3174 /* 3175 * COW mappings require pages in both 3176 * parent and child to be set to read. 3177 */ 3178 make_migration_entry_read(&swp_entry); 3179 entry = swp_entry_to_pte(swp_entry); 3180 set_huge_pte_at(src, addr, src_pte, entry); 3181 } 3182 set_huge_pte_at(dst, addr, dst_pte, entry); 3183 } else { 3184 if (cow) { 3185 huge_ptep_set_wrprotect(src, addr, src_pte); 3186 mmu_notifier_invalidate_range(src, mmun_start, 3187 mmun_end); 3188 } 3189 entry = huge_ptep_get(src_pte); 3190 ptepage = pte_page(entry); 3191 get_page(ptepage); 3192 page_dup_rmap(ptepage, true); 3193 set_huge_pte_at(dst, addr, dst_pte, entry); 3194 hugetlb_count_add(pages_per_huge_page(h), dst); 3195 } 3196 spin_unlock(src_ptl); 3197 spin_unlock(dst_ptl); 3198 } 3199 3200 if (cow) 3201 mmu_notifier_invalidate_range_end(src, mmun_start, mmun_end); 3202 3203 return ret; 3204 } 3205 3206 void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct *vma, 3207 unsigned long start, unsigned long end, 3208 struct page *ref_page) 3209 { 3210 struct mm_struct *mm = vma->vm_mm; 3211 unsigned long address; 3212 pte_t *ptep; 3213 pte_t pte; 3214 spinlock_t *ptl; 3215 struct page *page; 3216 struct hstate *h = hstate_vma(vma); 3217 unsigned long sz = huge_page_size(h); 3218 const unsigned long mmun_start = start; /* For mmu_notifiers */ 3219 const unsigned long mmun_end = end; /* For mmu_notifiers */ 3220 3221 WARN_ON(!is_vm_hugetlb_page(vma)); 3222 BUG_ON(start & ~huge_page_mask(h)); 3223 BUG_ON(end & ~huge_page_mask(h)); 3224 3225 tlb_start_vma(tlb, vma); 3226 mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end); 3227 address = start; 3228 for (; address < end; address += sz) { 3229 ptep = huge_pte_offset(mm, address); 3230 if (!ptep) 3231 continue; 3232 3233 ptl = huge_pte_lock(h, mm, ptep); 3234 if (huge_pmd_unshare(mm, &address, ptep)) { 3235 spin_unlock(ptl); 3236 continue; 3237 } 3238 3239 pte = huge_ptep_get(ptep); 3240 if (huge_pte_none(pte)) { 3241 spin_unlock(ptl); 3242 continue; 3243 } 3244 3245 /* 3246 * Migrating hugepage or HWPoisoned hugepage is already 3247 * unmapped and its refcount is dropped, so just clear pte here. 3248 */ 3249 if (unlikely(!pte_present(pte))) { 3250 huge_pte_clear(mm, address, ptep); 3251 spin_unlock(ptl); 3252 continue; 3253 } 3254 3255 page = pte_page(pte); 3256 /* 3257 * If a reference page is supplied, it is because a specific 3258 * page is being unmapped, not a range. Ensure the page we 3259 * are about to unmap is the actual page of interest. 3260 */ 3261 if (ref_page) { 3262 if (page != ref_page) { 3263 spin_unlock(ptl); 3264 continue; 3265 } 3266 /* 3267 * Mark the VMA as having unmapped its page so that 3268 * future faults in this VMA will fail rather than 3269 * looking like data was lost 3270 */ 3271 set_vma_resv_flags(vma, HPAGE_RESV_UNMAPPED); 3272 } 3273 3274 pte = huge_ptep_get_and_clear(mm, address, ptep); 3275 tlb_remove_tlb_entry(tlb, ptep, address); 3276 if (huge_pte_dirty(pte)) 3277 set_page_dirty(page); 3278 3279 hugetlb_count_sub(pages_per_huge_page(h), mm); 3280 page_remove_rmap(page, true); 3281 3282 spin_unlock(ptl); 3283 tlb_remove_page_size(tlb, page, huge_page_size(h)); 3284 /* 3285 * Bail out after unmapping reference page if supplied 3286 */ 3287 if (ref_page) 3288 break; 3289 } 3290 mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end); 3291 tlb_end_vma(tlb, vma); 3292 } 3293 3294 void __unmap_hugepage_range_final(struct mmu_gather *tlb, 3295 struct vm_area_struct *vma, unsigned long start, 3296 unsigned long end, struct page *ref_page) 3297 { 3298 __unmap_hugepage_range(tlb, vma, start, end, ref_page); 3299 3300 /* 3301 * Clear this flag so that x86's huge_pmd_share page_table_shareable 3302 * test will fail on a vma being torn down, and not grab a page table 3303 * on its way out. We're lucky that the flag has such an appropriate 3304 * name, and can in fact be safely cleared here. We could clear it 3305 * before the __unmap_hugepage_range above, but all that's necessary 3306 * is to clear it before releasing the i_mmap_rwsem. This works 3307 * because in the context this is called, the VMA is about to be 3308 * destroyed and the i_mmap_rwsem is held. 3309 */ 3310 vma->vm_flags &= ~VM_MAYSHARE; 3311 } 3312 3313 void unmap_hugepage_range(struct vm_area_struct *vma, unsigned long start, 3314 unsigned long end, struct page *ref_page) 3315 { 3316 struct mm_struct *mm; 3317 struct mmu_gather tlb; 3318 3319 mm = vma->vm_mm; 3320 3321 tlb_gather_mmu(&tlb, mm, start, end); 3322 __unmap_hugepage_range(&tlb, vma, start, end, ref_page); 3323 tlb_finish_mmu(&tlb, start, end); 3324 } 3325 3326 /* 3327 * This is called when the original mapper is failing to COW a MAP_PRIVATE 3328 * mappping it owns the reserve page for. The intention is to unmap the page 3329 * from other VMAs and let the children be SIGKILLed if they are faulting the 3330 * same region. 3331 */ 3332 static void unmap_ref_private(struct mm_struct *mm, struct vm_area_struct *vma, 3333 struct page *page, unsigned long address) 3334 { 3335 struct hstate *h = hstate_vma(vma); 3336 struct vm_area_struct *iter_vma; 3337 struct address_space *mapping; 3338 pgoff_t pgoff; 3339 3340 /* 3341 * vm_pgoff is in PAGE_SIZE units, hence the different calculation 3342 * from page cache lookup which is in HPAGE_SIZE units. 3343 */ 3344 address = address & huge_page_mask(h); 3345 pgoff = ((address - vma->vm_start) >> PAGE_SHIFT) + 3346 vma->vm_pgoff; 3347 mapping = vma->vm_file->f_mapping; 3348 3349 /* 3350 * Take the mapping lock for the duration of the table walk. As 3351 * this mapping should be shared between all the VMAs, 3352 * __unmap_hugepage_range() is called as the lock is already held 3353 */ 3354 i_mmap_lock_write(mapping); 3355 vma_interval_tree_foreach(iter_vma, &mapping->i_mmap, pgoff, pgoff) { 3356 /* Do not unmap the current VMA */ 3357 if (iter_vma == vma) 3358 continue; 3359 3360 /* 3361 * Shared VMAs have their own reserves and do not affect 3362 * MAP_PRIVATE accounting but it is possible that a shared 3363 * VMA is using the same page so check and skip such VMAs. 3364 */ 3365 if (iter_vma->vm_flags & VM_MAYSHARE) 3366 continue; 3367 3368 /* 3369 * Unmap the page from other VMAs without their own reserves. 3370 * They get marked to be SIGKILLed if they fault in these 3371 * areas. This is because a future no-page fault on this VMA 3372 * could insert a zeroed page instead of the data existing 3373 * from the time of fork. This would look like data corruption 3374 */ 3375 if (!is_vma_resv_set(iter_vma, HPAGE_RESV_OWNER)) 3376 unmap_hugepage_range(iter_vma, address, 3377 address + huge_page_size(h), page); 3378 } 3379 i_mmap_unlock_write(mapping); 3380 } 3381 3382 /* 3383 * Hugetlb_cow() should be called with page lock of the original hugepage held. 3384 * Called with hugetlb_instantiation_mutex held and pte_page locked so we 3385 * cannot race with other handlers or page migration. 3386 * Keep the pte_same checks anyway to make transition from the mutex easier. 3387 */ 3388 static int hugetlb_cow(struct mm_struct *mm, struct vm_area_struct *vma, 3389 unsigned long address, pte_t *ptep, pte_t pte, 3390 struct page *pagecache_page, spinlock_t *ptl) 3391 { 3392 struct hstate *h = hstate_vma(vma); 3393 struct page *old_page, *new_page; 3394 int ret = 0, outside_reserve = 0; 3395 unsigned long mmun_start; /* For mmu_notifiers */ 3396 unsigned long mmun_end; /* For mmu_notifiers */ 3397 3398 old_page = pte_page(pte); 3399 3400 retry_avoidcopy: 3401 /* If no-one else is actually using this page, avoid the copy 3402 * and just make the page writable */ 3403 if (page_mapcount(old_page) == 1 && PageAnon(old_page)) { 3404 page_move_anon_rmap(old_page, vma); 3405 set_huge_ptep_writable(vma, address, ptep); 3406 return 0; 3407 } 3408 3409 /* 3410 * If the process that created a MAP_PRIVATE mapping is about to 3411 * perform a COW due to a shared page count, attempt to satisfy 3412 * the allocation without using the existing reserves. The pagecache 3413 * page is used to determine if the reserve at this address was 3414 * consumed or not. If reserves were used, a partial faulted mapping 3415 * at the time of fork() could consume its reserves on COW instead 3416 * of the full address range. 3417 */ 3418 if (is_vma_resv_set(vma, HPAGE_RESV_OWNER) && 3419 old_page != pagecache_page) 3420 outside_reserve = 1; 3421 3422 get_page(old_page); 3423 3424 /* 3425 * Drop page table lock as buddy allocator may be called. It will 3426 * be acquired again before returning to the caller, as expected. 3427 */ 3428 spin_unlock(ptl); 3429 new_page = alloc_huge_page(vma, address, outside_reserve); 3430 3431 if (IS_ERR(new_page)) { 3432 /* 3433 * If a process owning a MAP_PRIVATE mapping fails to COW, 3434 * it is due to references held by a child and an insufficient 3435 * huge page pool. To guarantee the original mappers 3436 * reliability, unmap the page from child processes. The child 3437 * may get SIGKILLed if it later faults. 3438 */ 3439 if (outside_reserve) { 3440 put_page(old_page); 3441 BUG_ON(huge_pte_none(pte)); 3442 unmap_ref_private(mm, vma, old_page, address); 3443 BUG_ON(huge_pte_none(pte)); 3444 spin_lock(ptl); 3445 ptep = huge_pte_offset(mm, address & huge_page_mask(h)); 3446 if (likely(ptep && 3447 pte_same(huge_ptep_get(ptep), pte))) 3448 goto retry_avoidcopy; 3449 /* 3450 * race occurs while re-acquiring page table 3451 * lock, and our job is done. 3452 */ 3453 return 0; 3454 } 3455 3456 ret = (PTR_ERR(new_page) == -ENOMEM) ? 3457 VM_FAULT_OOM : VM_FAULT_SIGBUS; 3458 goto out_release_old; 3459 } 3460 3461 /* 3462 * When the original hugepage is shared one, it does not have 3463 * anon_vma prepared. 3464 */ 3465 if (unlikely(anon_vma_prepare(vma))) { 3466 ret = VM_FAULT_OOM; 3467 goto out_release_all; 3468 } 3469 3470 copy_user_huge_page(new_page, old_page, address, vma, 3471 pages_per_huge_page(h)); 3472 __SetPageUptodate(new_page); 3473 set_page_huge_active(new_page); 3474 3475 mmun_start = address & huge_page_mask(h); 3476 mmun_end = mmun_start + huge_page_size(h); 3477 mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end); 3478 3479 /* 3480 * Retake the page table lock to check for racing updates 3481 * before the page tables are altered 3482 */ 3483 spin_lock(ptl); 3484 ptep = huge_pte_offset(mm, address & huge_page_mask(h)); 3485 if (likely(ptep && pte_same(huge_ptep_get(ptep), pte))) { 3486 ClearPagePrivate(new_page); 3487 3488 /* Break COW */ 3489 huge_ptep_clear_flush(vma, address, ptep); 3490 mmu_notifier_invalidate_range(mm, mmun_start, mmun_end); 3491 set_huge_pte_at(mm, address, ptep, 3492 make_huge_pte(vma, new_page, 1)); 3493 page_remove_rmap(old_page, true); 3494 hugepage_add_new_anon_rmap(new_page, vma, address); 3495 /* Make the old page be freed below */ 3496 new_page = old_page; 3497 } 3498 spin_unlock(ptl); 3499 mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end); 3500 out_release_all: 3501 put_page(new_page); 3502 out_release_old: 3503 put_page(old_page); 3504 3505 spin_lock(ptl); /* Caller expects lock to be held */ 3506 return ret; 3507 } 3508 3509 /* Return the pagecache page at a given address within a VMA */ 3510 static struct page *hugetlbfs_pagecache_page(struct hstate *h, 3511 struct vm_area_struct *vma, unsigned long address) 3512 { 3513 struct address_space *mapping; 3514 pgoff_t idx; 3515 3516 mapping = vma->vm_file->f_mapping; 3517 idx = vma_hugecache_offset(h, vma, address); 3518 3519 return find_lock_page(mapping, idx); 3520 } 3521 3522 /* 3523 * Return whether there is a pagecache page to back given address within VMA. 3524 * Caller follow_hugetlb_page() holds page_table_lock so we cannot lock_page. 3525 */ 3526 static bool hugetlbfs_pagecache_present(struct hstate *h, 3527 struct vm_area_struct *vma, unsigned long address) 3528 { 3529 struct address_space *mapping; 3530 pgoff_t idx; 3531 struct page *page; 3532 3533 mapping = vma->vm_file->f_mapping; 3534 idx = vma_hugecache_offset(h, vma, address); 3535 3536 page = find_get_page(mapping, idx); 3537 if (page) 3538 put_page(page); 3539 return page != NULL; 3540 } 3541 3542 int huge_add_to_page_cache(struct page *page, struct address_space *mapping, 3543 pgoff_t idx) 3544 { 3545 struct inode *inode = mapping->host; 3546 struct hstate *h = hstate_inode(inode); 3547 int err = add_to_page_cache(page, mapping, idx, GFP_KERNEL); 3548 3549 if (err) 3550 return err; 3551 ClearPagePrivate(page); 3552 3553 spin_lock(&inode->i_lock); 3554 inode->i_blocks += blocks_per_huge_page(h); 3555 spin_unlock(&inode->i_lock); 3556 return 0; 3557 } 3558 3559 static int hugetlb_no_page(struct mm_struct *mm, struct vm_area_struct *vma, 3560 struct address_space *mapping, pgoff_t idx, 3561 unsigned long address, pte_t *ptep, unsigned int flags) 3562 { 3563 struct hstate *h = hstate_vma(vma); 3564 int ret = VM_FAULT_SIGBUS; 3565 int anon_rmap = 0; 3566 unsigned long size; 3567 struct page *page; 3568 pte_t new_pte; 3569 spinlock_t *ptl; 3570 3571 /* 3572 * Currently, we are forced to kill the process in the event the 3573 * original mapper has unmapped pages from the child due to a failed 3574 * COW. Warn that such a situation has occurred as it may not be obvious 3575 */ 3576 if (is_vma_resv_set(vma, HPAGE_RESV_UNMAPPED)) { 3577 pr_warn_ratelimited("PID %d killed due to inadequate hugepage pool\n", 3578 current->pid); 3579 return ret; 3580 } 3581 3582 /* 3583 * Use page lock to guard against racing truncation 3584 * before we get page_table_lock. 3585 */ 3586 retry: 3587 page = find_lock_page(mapping, idx); 3588 if (!page) { 3589 size = i_size_read(mapping->host) >> huge_page_shift(h); 3590 if (idx >= size) 3591 goto out; 3592 page = alloc_huge_page(vma, address, 0); 3593 if (IS_ERR(page)) { 3594 ret = PTR_ERR(page); 3595 if (ret == -ENOMEM) 3596 ret = VM_FAULT_OOM; 3597 else 3598 ret = VM_FAULT_SIGBUS; 3599 goto out; 3600 } 3601 clear_huge_page(page, address, pages_per_huge_page(h)); 3602 __SetPageUptodate(page); 3603 set_page_huge_active(page); 3604 3605 if (vma->vm_flags & VM_MAYSHARE) { 3606 int err = huge_add_to_page_cache(page, mapping, idx); 3607 if (err) { 3608 put_page(page); 3609 if (err == -EEXIST) 3610 goto retry; 3611 goto out; 3612 } 3613 } else { 3614 lock_page(page); 3615 if (unlikely(anon_vma_prepare(vma))) { 3616 ret = VM_FAULT_OOM; 3617 goto backout_unlocked; 3618 } 3619 anon_rmap = 1; 3620 } 3621 } else { 3622 /* 3623 * If memory error occurs between mmap() and fault, some process 3624 * don't have hwpoisoned swap entry for errored virtual address. 3625 * So we need to block hugepage fault by PG_hwpoison bit check. 3626 */ 3627 if (unlikely(PageHWPoison(page))) { 3628 ret = VM_FAULT_HWPOISON | 3629 VM_FAULT_SET_HINDEX(hstate_index(h)); 3630 goto backout_unlocked; 3631 } 3632 } 3633 3634 /* 3635 * If we are going to COW a private mapping later, we examine the 3636 * pending reservations for this page now. This will ensure that 3637 * any allocations necessary to record that reservation occur outside 3638 * the spinlock. 3639 */ 3640 if ((flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED)) { 3641 if (vma_needs_reservation(h, vma, address) < 0) { 3642 ret = VM_FAULT_OOM; 3643 goto backout_unlocked; 3644 } 3645 /* Just decrements count, does not deallocate */ 3646 vma_end_reservation(h, vma, address); 3647 } 3648 3649 ptl = huge_pte_lockptr(h, mm, ptep); 3650 spin_lock(ptl); 3651 size = i_size_read(mapping->host) >> huge_page_shift(h); 3652 if (idx >= size) 3653 goto backout; 3654 3655 ret = 0; 3656 if (!huge_pte_none(huge_ptep_get(ptep))) 3657 goto backout; 3658 3659 if (anon_rmap) { 3660 ClearPagePrivate(page); 3661 hugepage_add_new_anon_rmap(page, vma, address); 3662 } else 3663 page_dup_rmap(page, true); 3664 new_pte = make_huge_pte(vma, page, ((vma->vm_flags & VM_WRITE) 3665 && (vma->vm_flags & VM_SHARED))); 3666 set_huge_pte_at(mm, address, ptep, new_pte); 3667 3668 hugetlb_count_add(pages_per_huge_page(h), mm); 3669 if ((flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED)) { 3670 /* Optimization, do the COW without a second fault */ 3671 ret = hugetlb_cow(mm, vma, address, ptep, new_pte, page, ptl); 3672 } 3673 3674 spin_unlock(ptl); 3675 unlock_page(page); 3676 out: 3677 return ret; 3678 3679 backout: 3680 spin_unlock(ptl); 3681 backout_unlocked: 3682 unlock_page(page); 3683 put_page(page); 3684 goto out; 3685 } 3686 3687 #ifdef CONFIG_SMP 3688 u32 hugetlb_fault_mutex_hash(struct hstate *h, struct mm_struct *mm, 3689 struct vm_area_struct *vma, 3690 struct address_space *mapping, 3691 pgoff_t idx, unsigned long address) 3692 { 3693 unsigned long key[2]; 3694 u32 hash; 3695 3696 if (vma->vm_flags & VM_SHARED) { 3697 key[0] = (unsigned long) mapping; 3698 key[1] = idx; 3699 } else { 3700 key[0] = (unsigned long) mm; 3701 key[1] = address >> huge_page_shift(h); 3702 } 3703 3704 hash = jhash2((u32 *)&key, sizeof(key)/sizeof(u32), 0); 3705 3706 return hash & (num_fault_mutexes - 1); 3707 } 3708 #else 3709 /* 3710 * For uniprocesor systems we always use a single mutex, so just 3711 * return 0 and avoid the hashing overhead. 3712 */ 3713 u32 hugetlb_fault_mutex_hash(struct hstate *h, struct mm_struct *mm, 3714 struct vm_area_struct *vma, 3715 struct address_space *mapping, 3716 pgoff_t idx, unsigned long address) 3717 { 3718 return 0; 3719 } 3720 #endif 3721 3722 int hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma, 3723 unsigned long address, unsigned int flags) 3724 { 3725 pte_t *ptep, entry; 3726 spinlock_t *ptl; 3727 int ret; 3728 u32 hash; 3729 pgoff_t idx; 3730 struct page *page = NULL; 3731 struct page *pagecache_page = NULL; 3732 struct hstate *h = hstate_vma(vma); 3733 struct address_space *mapping; 3734 int need_wait_lock = 0; 3735 3736 address &= huge_page_mask(h); 3737 3738 ptep = huge_pte_offset(mm, address); 3739 if (ptep) { 3740 entry = huge_ptep_get(ptep); 3741 if (unlikely(is_hugetlb_entry_migration(entry))) { 3742 migration_entry_wait_huge(vma, mm, ptep); 3743 return 0; 3744 } else if (unlikely(is_hugetlb_entry_hwpoisoned(entry))) 3745 return VM_FAULT_HWPOISON_LARGE | 3746 VM_FAULT_SET_HINDEX(hstate_index(h)); 3747 } else { 3748 ptep = huge_pte_alloc(mm, address, huge_page_size(h)); 3749 if (!ptep) 3750 return VM_FAULT_OOM; 3751 } 3752 3753 mapping = vma->vm_file->f_mapping; 3754 idx = vma_hugecache_offset(h, vma, address); 3755 3756 /* 3757 * Serialize hugepage allocation and instantiation, so that we don't 3758 * get spurious allocation failures if two CPUs race to instantiate 3759 * the same page in the page cache. 3760 */ 3761 hash = hugetlb_fault_mutex_hash(h, mm, vma, mapping, idx, address); 3762 mutex_lock(&hugetlb_fault_mutex_table[hash]); 3763 3764 entry = huge_ptep_get(ptep); 3765 if (huge_pte_none(entry)) { 3766 ret = hugetlb_no_page(mm, vma, mapping, idx, address, ptep, flags); 3767 goto out_mutex; 3768 } 3769 3770 ret = 0; 3771 3772 /* 3773 * entry could be a migration/hwpoison entry at this point, so this 3774 * check prevents the kernel from going below assuming that we have 3775 * a active hugepage in pagecache. This goto expects the 2nd page fault, 3776 * and is_hugetlb_entry_(migration|hwpoisoned) check will properly 3777 * handle it. 3778 */ 3779 if (!pte_present(entry)) 3780 goto out_mutex; 3781 3782 /* 3783 * If we are going to COW the mapping later, we examine the pending 3784 * reservations for this page now. This will ensure that any 3785 * allocations necessary to record that reservation occur outside the 3786 * spinlock. For private mappings, we also lookup the pagecache 3787 * page now as it is used to determine if a reservation has been 3788 * consumed. 3789 */ 3790 if ((flags & FAULT_FLAG_WRITE) && !huge_pte_write(entry)) { 3791 if (vma_needs_reservation(h, vma, address) < 0) { 3792 ret = VM_FAULT_OOM; 3793 goto out_mutex; 3794 } 3795 /* Just decrements count, does not deallocate */ 3796 vma_end_reservation(h, vma, address); 3797 3798 if (!(vma->vm_flags & VM_MAYSHARE)) 3799 pagecache_page = hugetlbfs_pagecache_page(h, 3800 vma, address); 3801 } 3802 3803 ptl = huge_pte_lock(h, mm, ptep); 3804 3805 /* Check for a racing update before calling hugetlb_cow */ 3806 if (unlikely(!pte_same(entry, huge_ptep_get(ptep)))) 3807 goto out_ptl; 3808 3809 /* 3810 * hugetlb_cow() requires page locks of pte_page(entry) and 3811 * pagecache_page, so here we need take the former one 3812 * when page != pagecache_page or !pagecache_page. 3813 */ 3814 page = pte_page(entry); 3815 if (page != pagecache_page) 3816 if (!trylock_page(page)) { 3817 need_wait_lock = 1; 3818 goto out_ptl; 3819 } 3820 3821 get_page(page); 3822 3823 if (flags & FAULT_FLAG_WRITE) { 3824 if (!huge_pte_write(entry)) { 3825 ret = hugetlb_cow(mm, vma, address, ptep, entry, 3826 pagecache_page, ptl); 3827 goto out_put_page; 3828 } 3829 entry = huge_pte_mkdirty(entry); 3830 } 3831 entry = pte_mkyoung(entry); 3832 if (huge_ptep_set_access_flags(vma, address, ptep, entry, 3833 flags & FAULT_FLAG_WRITE)) 3834 update_mmu_cache(vma, address, ptep); 3835 out_put_page: 3836 if (page != pagecache_page) 3837 unlock_page(page); 3838 put_page(page); 3839 out_ptl: 3840 spin_unlock(ptl); 3841 3842 if (pagecache_page) { 3843 unlock_page(pagecache_page); 3844 put_page(pagecache_page); 3845 } 3846 out_mutex: 3847 mutex_unlock(&hugetlb_fault_mutex_table[hash]); 3848 /* 3849 * Generally it's safe to hold refcount during waiting page lock. But 3850 * here we just wait to defer the next page fault to avoid busy loop and 3851 * the page is not used after unlocked before returning from the current 3852 * page fault. So we are safe from accessing freed page, even if we wait 3853 * here without taking refcount. 3854 */ 3855 if (need_wait_lock) 3856 wait_on_page_locked(page); 3857 return ret; 3858 } 3859 3860 long follow_hugetlb_page(struct mm_struct *mm, struct vm_area_struct *vma, 3861 struct page **pages, struct vm_area_struct **vmas, 3862 unsigned long *position, unsigned long *nr_pages, 3863 long i, unsigned int flags) 3864 { 3865 unsigned long pfn_offset; 3866 unsigned long vaddr = *position; 3867 unsigned long remainder = *nr_pages; 3868 struct hstate *h = hstate_vma(vma); 3869 3870 while (vaddr < vma->vm_end && remainder) { 3871 pte_t *pte; 3872 spinlock_t *ptl = NULL; 3873 int absent; 3874 struct page *page; 3875 3876 /* 3877 * If we have a pending SIGKILL, don't keep faulting pages and 3878 * potentially allocating memory. 3879 */ 3880 if (unlikely(fatal_signal_pending(current))) { 3881 remainder = 0; 3882 break; 3883 } 3884 3885 /* 3886 * Some archs (sparc64, sh*) have multiple pte_ts to 3887 * each hugepage. We have to make sure we get the 3888 * first, for the page indexing below to work. 3889 * 3890 * Note that page table lock is not held when pte is null. 3891 */ 3892 pte = huge_pte_offset(mm, vaddr & huge_page_mask(h)); 3893 if (pte) 3894 ptl = huge_pte_lock(h, mm, pte); 3895 absent = !pte || huge_pte_none(huge_ptep_get(pte)); 3896 3897 /* 3898 * When coredumping, it suits get_dump_page if we just return 3899 * an error where there's an empty slot with no huge pagecache 3900 * to back it. This way, we avoid allocating a hugepage, and 3901 * the sparse dumpfile avoids allocating disk blocks, but its 3902 * huge holes still show up with zeroes where they need to be. 3903 */ 3904 if (absent && (flags & FOLL_DUMP) && 3905 !hugetlbfs_pagecache_present(h, vma, vaddr)) { 3906 if (pte) 3907 spin_unlock(ptl); 3908 remainder = 0; 3909 break; 3910 } 3911 3912 /* 3913 * We need call hugetlb_fault for both hugepages under migration 3914 * (in which case hugetlb_fault waits for the migration,) and 3915 * hwpoisoned hugepages (in which case we need to prevent the 3916 * caller from accessing to them.) In order to do this, we use 3917 * here is_swap_pte instead of is_hugetlb_entry_migration and 3918 * is_hugetlb_entry_hwpoisoned. This is because it simply covers 3919 * both cases, and because we can't follow correct pages 3920 * directly from any kind of swap entries. 3921 */ 3922 if (absent || is_swap_pte(huge_ptep_get(pte)) || 3923 ((flags & FOLL_WRITE) && 3924 !huge_pte_write(huge_ptep_get(pte)))) { 3925 int ret; 3926 3927 if (pte) 3928 spin_unlock(ptl); 3929 ret = hugetlb_fault(mm, vma, vaddr, 3930 (flags & FOLL_WRITE) ? FAULT_FLAG_WRITE : 0); 3931 if (!(ret & VM_FAULT_ERROR)) 3932 continue; 3933 3934 remainder = 0; 3935 break; 3936 } 3937 3938 pfn_offset = (vaddr & ~huge_page_mask(h)) >> PAGE_SHIFT; 3939 page = pte_page(huge_ptep_get(pte)); 3940 same_page: 3941 if (pages) { 3942 pages[i] = mem_map_offset(page, pfn_offset); 3943 get_page(pages[i]); 3944 } 3945 3946 if (vmas) 3947 vmas[i] = vma; 3948 3949 vaddr += PAGE_SIZE; 3950 ++pfn_offset; 3951 --remainder; 3952 ++i; 3953 if (vaddr < vma->vm_end && remainder && 3954 pfn_offset < pages_per_huge_page(h)) { 3955 /* 3956 * We use pfn_offset to avoid touching the pageframes 3957 * of this compound page. 3958 */ 3959 goto same_page; 3960 } 3961 spin_unlock(ptl); 3962 } 3963 *nr_pages = remainder; 3964 *position = vaddr; 3965 3966 return i ? i : -EFAULT; 3967 } 3968 3969 #ifndef __HAVE_ARCH_FLUSH_HUGETLB_TLB_RANGE 3970 /* 3971 * ARCHes with special requirements for evicting HUGETLB backing TLB entries can 3972 * implement this. 3973 */ 3974 #define flush_hugetlb_tlb_range(vma, addr, end) flush_tlb_range(vma, addr, end) 3975 #endif 3976 3977 unsigned long hugetlb_change_protection(struct vm_area_struct *vma, 3978 unsigned long address, unsigned long end, pgprot_t newprot) 3979 { 3980 struct mm_struct *mm = vma->vm_mm; 3981 unsigned long start = address; 3982 pte_t *ptep; 3983 pte_t pte; 3984 struct hstate *h = hstate_vma(vma); 3985 unsigned long pages = 0; 3986 3987 BUG_ON(address >= end); 3988 flush_cache_range(vma, address, end); 3989 3990 mmu_notifier_invalidate_range_start(mm, start, end); 3991 i_mmap_lock_write(vma->vm_file->f_mapping); 3992 for (; address < end; address += huge_page_size(h)) { 3993 spinlock_t *ptl; 3994 ptep = huge_pte_offset(mm, address); 3995 if (!ptep) 3996 continue; 3997 ptl = huge_pte_lock(h, mm, ptep); 3998 if (huge_pmd_unshare(mm, &address, ptep)) { 3999 pages++; 4000 spin_unlock(ptl); 4001 continue; 4002 } 4003 pte = huge_ptep_get(ptep); 4004 if (unlikely(is_hugetlb_entry_hwpoisoned(pte))) { 4005 spin_unlock(ptl); 4006 continue; 4007 } 4008 if (unlikely(is_hugetlb_entry_migration(pte))) { 4009 swp_entry_t entry = pte_to_swp_entry(pte); 4010 4011 if (is_write_migration_entry(entry)) { 4012 pte_t newpte; 4013 4014 make_migration_entry_read(&entry); 4015 newpte = swp_entry_to_pte(entry); 4016 set_huge_pte_at(mm, address, ptep, newpte); 4017 pages++; 4018 } 4019 spin_unlock(ptl); 4020 continue; 4021 } 4022 if (!huge_pte_none(pte)) { 4023 pte = huge_ptep_get_and_clear(mm, address, ptep); 4024 pte = pte_mkhuge(huge_pte_modify(pte, newprot)); 4025 pte = arch_make_huge_pte(pte, vma, NULL, 0); 4026 set_huge_pte_at(mm, address, ptep, pte); 4027 pages++; 4028 } 4029 spin_unlock(ptl); 4030 } 4031 /* 4032 * Must flush TLB before releasing i_mmap_rwsem: x86's huge_pmd_unshare 4033 * may have cleared our pud entry and done put_page on the page table: 4034 * once we release i_mmap_rwsem, another task can do the final put_page 4035 * and that page table be reused and filled with junk. 4036 */ 4037 flush_hugetlb_tlb_range(vma, start, end); 4038 mmu_notifier_invalidate_range(mm, start, end); 4039 i_mmap_unlock_write(vma->vm_file->f_mapping); 4040 mmu_notifier_invalidate_range_end(mm, start, end); 4041 4042 return pages << h->order; 4043 } 4044 4045 int hugetlb_reserve_pages(struct inode *inode, 4046 long from, long to, 4047 struct vm_area_struct *vma, 4048 vm_flags_t vm_flags) 4049 { 4050 long ret, chg; 4051 struct hstate *h = hstate_inode(inode); 4052 struct hugepage_subpool *spool = subpool_inode(inode); 4053 struct resv_map *resv_map; 4054 long gbl_reserve; 4055 4056 /* 4057 * Only apply hugepage reservation if asked. At fault time, an 4058 * attempt will be made for VM_NORESERVE to allocate a page 4059 * without using reserves 4060 */ 4061 if (vm_flags & VM_NORESERVE) 4062 return 0; 4063 4064 /* 4065 * Shared mappings base their reservation on the number of pages that 4066 * are already allocated on behalf of the file. Private mappings need 4067 * to reserve the full area even if read-only as mprotect() may be 4068 * called to make the mapping read-write. Assume !vma is a shm mapping 4069 */ 4070 if (!vma || vma->vm_flags & VM_MAYSHARE) { 4071 resv_map = inode_resv_map(inode); 4072 4073 chg = region_chg(resv_map, from, to); 4074 4075 } else { 4076 resv_map = resv_map_alloc(); 4077 if (!resv_map) 4078 return -ENOMEM; 4079 4080 chg = to - from; 4081 4082 set_vma_resv_map(vma, resv_map); 4083 set_vma_resv_flags(vma, HPAGE_RESV_OWNER); 4084 } 4085 4086 if (chg < 0) { 4087 ret = chg; 4088 goto out_err; 4089 } 4090 4091 /* 4092 * There must be enough pages in the subpool for the mapping. If 4093 * the subpool has a minimum size, there may be some global 4094 * reservations already in place (gbl_reserve). 4095 */ 4096 gbl_reserve = hugepage_subpool_get_pages(spool, chg); 4097 if (gbl_reserve < 0) { 4098 ret = -ENOSPC; 4099 goto out_err; 4100 } 4101 4102 /* 4103 * Check enough hugepages are available for the reservation. 4104 * Hand the pages back to the subpool if there are not 4105 */ 4106 ret = hugetlb_acct_memory(h, gbl_reserve); 4107 if (ret < 0) { 4108 /* put back original number of pages, chg */ 4109 (void)hugepage_subpool_put_pages(spool, chg); 4110 goto out_err; 4111 } 4112 4113 /* 4114 * Account for the reservations made. Shared mappings record regions 4115 * that have reservations as they are shared by multiple VMAs. 4116 * When the last VMA disappears, the region map says how much 4117 * the reservation was and the page cache tells how much of 4118 * the reservation was consumed. Private mappings are per-VMA and 4119 * only the consumed reservations are tracked. When the VMA 4120 * disappears, the original reservation is the VMA size and the 4121 * consumed reservations are stored in the map. Hence, nothing 4122 * else has to be done for private mappings here 4123 */ 4124 if (!vma || vma->vm_flags & VM_MAYSHARE) { 4125 long add = region_add(resv_map, from, to); 4126 4127 if (unlikely(chg > add)) { 4128 /* 4129 * pages in this range were added to the reserve 4130 * map between region_chg and region_add. This 4131 * indicates a race with alloc_huge_page. Adjust 4132 * the subpool and reserve counts modified above 4133 * based on the difference. 4134 */ 4135 long rsv_adjust; 4136 4137 rsv_adjust = hugepage_subpool_put_pages(spool, 4138 chg - add); 4139 hugetlb_acct_memory(h, -rsv_adjust); 4140 } 4141 } 4142 return 0; 4143 out_err: 4144 if (!vma || vma->vm_flags & VM_MAYSHARE) 4145 region_abort(resv_map, from, to); 4146 if (vma && is_vma_resv_set(vma, HPAGE_RESV_OWNER)) 4147 kref_put(&resv_map->refs, resv_map_release); 4148 return ret; 4149 } 4150 4151 long hugetlb_unreserve_pages(struct inode *inode, long start, long end, 4152 long freed) 4153 { 4154 struct hstate *h = hstate_inode(inode); 4155 struct resv_map *resv_map = inode_resv_map(inode); 4156 long chg = 0; 4157 struct hugepage_subpool *spool = subpool_inode(inode); 4158 long gbl_reserve; 4159 4160 if (resv_map) { 4161 chg = region_del(resv_map, start, end); 4162 /* 4163 * region_del() can fail in the rare case where a region 4164 * must be split and another region descriptor can not be 4165 * allocated. If end == LONG_MAX, it will not fail. 4166 */ 4167 if (chg < 0) 4168 return chg; 4169 } 4170 4171 spin_lock(&inode->i_lock); 4172 inode->i_blocks -= (blocks_per_huge_page(h) * freed); 4173 spin_unlock(&inode->i_lock); 4174 4175 /* 4176 * If the subpool has a minimum size, the number of global 4177 * reservations to be released may be adjusted. 4178 */ 4179 gbl_reserve = hugepage_subpool_put_pages(spool, (chg - freed)); 4180 hugetlb_acct_memory(h, -gbl_reserve); 4181 4182 return 0; 4183 } 4184 4185 #ifdef CONFIG_ARCH_WANT_HUGE_PMD_SHARE 4186 static unsigned long page_table_shareable(struct vm_area_struct *svma, 4187 struct vm_area_struct *vma, 4188 unsigned long addr, pgoff_t idx) 4189 { 4190 unsigned long saddr = ((idx - svma->vm_pgoff) << PAGE_SHIFT) + 4191 svma->vm_start; 4192 unsigned long sbase = saddr & PUD_MASK; 4193 unsigned long s_end = sbase + PUD_SIZE; 4194 4195 /* Allow segments to share if only one is marked locked */ 4196 unsigned long vm_flags = vma->vm_flags & VM_LOCKED_CLEAR_MASK; 4197 unsigned long svm_flags = svma->vm_flags & VM_LOCKED_CLEAR_MASK; 4198 4199 /* 4200 * match the virtual addresses, permission and the alignment of the 4201 * page table page. 4202 */ 4203 if (pmd_index(addr) != pmd_index(saddr) || 4204 vm_flags != svm_flags || 4205 sbase < svma->vm_start || svma->vm_end < s_end) 4206 return 0; 4207 4208 return saddr; 4209 } 4210 4211 static bool vma_shareable(struct vm_area_struct *vma, unsigned long addr) 4212 { 4213 unsigned long base = addr & PUD_MASK; 4214 unsigned long end = base + PUD_SIZE; 4215 4216 /* 4217 * check on proper vm_flags and page table alignment 4218 */ 4219 if (vma->vm_flags & VM_MAYSHARE && 4220 vma->vm_start <= base && end <= vma->vm_end) 4221 return true; 4222 return false; 4223 } 4224 4225 /* 4226 * Search for a shareable pmd page for hugetlb. In any case calls pmd_alloc() 4227 * and returns the corresponding pte. While this is not necessary for the 4228 * !shared pmd case because we can allocate the pmd later as well, it makes the 4229 * code much cleaner. pmd allocation is essential for the shared case because 4230 * pud has to be populated inside the same i_mmap_rwsem section - otherwise 4231 * racing tasks could either miss the sharing (see huge_pte_offset) or select a 4232 * bad pmd for sharing. 4233 */ 4234 pte_t *huge_pmd_share(struct mm_struct *mm, unsigned long addr, pud_t *pud) 4235 { 4236 struct vm_area_struct *vma = find_vma(mm, addr); 4237 struct address_space *mapping = vma->vm_file->f_mapping; 4238 pgoff_t idx = ((addr - vma->vm_start) >> PAGE_SHIFT) + 4239 vma->vm_pgoff; 4240 struct vm_area_struct *svma; 4241 unsigned long saddr; 4242 pte_t *spte = NULL; 4243 pte_t *pte; 4244 spinlock_t *ptl; 4245 4246 if (!vma_shareable(vma, addr)) 4247 return (pte_t *)pmd_alloc(mm, pud, addr); 4248 4249 i_mmap_lock_write(mapping); 4250 vma_interval_tree_foreach(svma, &mapping->i_mmap, idx, idx) { 4251 if (svma == vma) 4252 continue; 4253 4254 saddr = page_table_shareable(svma, vma, addr, idx); 4255 if (saddr) { 4256 spte = huge_pte_offset(svma->vm_mm, saddr); 4257 if (spte) { 4258 get_page(virt_to_page(spte)); 4259 break; 4260 } 4261 } 4262 } 4263 4264 if (!spte) 4265 goto out; 4266 4267 ptl = huge_pte_lockptr(hstate_vma(vma), mm, spte); 4268 spin_lock(ptl); 4269 if (pud_none(*pud)) { 4270 pud_populate(mm, pud, 4271 (pmd_t *)((unsigned long)spte & PAGE_MASK)); 4272 mm_inc_nr_pmds(mm); 4273 } else { 4274 put_page(virt_to_page(spte)); 4275 } 4276 spin_unlock(ptl); 4277 out: 4278 pte = (pte_t *)pmd_alloc(mm, pud, addr); 4279 i_mmap_unlock_write(mapping); 4280 return pte; 4281 } 4282 4283 /* 4284 * unmap huge page backed by shared pte. 4285 * 4286 * Hugetlb pte page is ref counted at the time of mapping. If pte is shared 4287 * indicated by page_count > 1, unmap is achieved by clearing pud and 4288 * decrementing the ref count. If count == 1, the pte page is not shared. 4289 * 4290 * called with page table lock held. 4291 * 4292 * returns: 1 successfully unmapped a shared pte page 4293 * 0 the underlying pte page is not shared, or it is the last user 4294 */ 4295 int huge_pmd_unshare(struct mm_struct *mm, unsigned long *addr, pte_t *ptep) 4296 { 4297 pgd_t *pgd = pgd_offset(mm, *addr); 4298 pud_t *pud = pud_offset(pgd, *addr); 4299 4300 BUG_ON(page_count(virt_to_page(ptep)) == 0); 4301 if (page_count(virt_to_page(ptep)) == 1) 4302 return 0; 4303 4304 pud_clear(pud); 4305 put_page(virt_to_page(ptep)); 4306 mm_dec_nr_pmds(mm); 4307 *addr = ALIGN(*addr, HPAGE_SIZE * PTRS_PER_PTE) - HPAGE_SIZE; 4308 return 1; 4309 } 4310 #define want_pmd_share() (1) 4311 #else /* !CONFIG_ARCH_WANT_HUGE_PMD_SHARE */ 4312 pte_t *huge_pmd_share(struct mm_struct *mm, unsigned long addr, pud_t *pud) 4313 { 4314 return NULL; 4315 } 4316 4317 int huge_pmd_unshare(struct mm_struct *mm, unsigned long *addr, pte_t *ptep) 4318 { 4319 return 0; 4320 } 4321 #define want_pmd_share() (0) 4322 #endif /* CONFIG_ARCH_WANT_HUGE_PMD_SHARE */ 4323 4324 #ifdef CONFIG_ARCH_WANT_GENERAL_HUGETLB 4325 pte_t *huge_pte_alloc(struct mm_struct *mm, 4326 unsigned long addr, unsigned long sz) 4327 { 4328 pgd_t *pgd; 4329 pud_t *pud; 4330 pte_t *pte = NULL; 4331 4332 pgd = pgd_offset(mm, addr); 4333 pud = pud_alloc(mm, pgd, addr); 4334 if (pud) { 4335 if (sz == PUD_SIZE) { 4336 pte = (pte_t *)pud; 4337 } else { 4338 BUG_ON(sz != PMD_SIZE); 4339 if (want_pmd_share() && pud_none(*pud)) 4340 pte = huge_pmd_share(mm, addr, pud); 4341 else 4342 pte = (pte_t *)pmd_alloc(mm, pud, addr); 4343 } 4344 } 4345 BUG_ON(pte && pte_present(*pte) && !pte_huge(*pte)); 4346 4347 return pte; 4348 } 4349 4350 pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr) 4351 { 4352 pgd_t *pgd; 4353 pud_t *pud; 4354 pmd_t *pmd = NULL; 4355 4356 pgd = pgd_offset(mm, addr); 4357 if (pgd_present(*pgd)) { 4358 pud = pud_offset(pgd, addr); 4359 if (pud_present(*pud)) { 4360 if (pud_huge(*pud)) 4361 return (pte_t *)pud; 4362 pmd = pmd_offset(pud, addr); 4363 } 4364 } 4365 return (pte_t *) pmd; 4366 } 4367 4368 #endif /* CONFIG_ARCH_WANT_GENERAL_HUGETLB */ 4369 4370 /* 4371 * These functions are overwritable if your architecture needs its own 4372 * behavior. 4373 */ 4374 struct page * __weak 4375 follow_huge_addr(struct mm_struct *mm, unsigned long address, 4376 int write) 4377 { 4378 return ERR_PTR(-EINVAL); 4379 } 4380 4381 struct page * __weak 4382 follow_huge_pmd(struct mm_struct *mm, unsigned long address, 4383 pmd_t *pmd, int flags) 4384 { 4385 struct page *page = NULL; 4386 spinlock_t *ptl; 4387 retry: 4388 ptl = pmd_lockptr(mm, pmd); 4389 spin_lock(ptl); 4390 /* 4391 * make sure that the address range covered by this pmd is not 4392 * unmapped from other threads. 4393 */ 4394 if (!pmd_huge(*pmd)) 4395 goto out; 4396 if (pmd_present(*pmd)) { 4397 page = pmd_page(*pmd) + ((address & ~PMD_MASK) >> PAGE_SHIFT); 4398 if (flags & FOLL_GET) 4399 get_page(page); 4400 } else { 4401 if (is_hugetlb_entry_migration(huge_ptep_get((pte_t *)pmd))) { 4402 spin_unlock(ptl); 4403 __migration_entry_wait(mm, (pte_t *)pmd, ptl); 4404 goto retry; 4405 } 4406 /* 4407 * hwpoisoned entry is treated as no_page_table in 4408 * follow_page_mask(). 4409 */ 4410 } 4411 out: 4412 spin_unlock(ptl); 4413 return page; 4414 } 4415 4416 struct page * __weak 4417 follow_huge_pud(struct mm_struct *mm, unsigned long address, 4418 pud_t *pud, int flags) 4419 { 4420 if (flags & FOLL_GET) 4421 return NULL; 4422 4423 return pte_page(*(pte_t *)pud) + ((address & ~PUD_MASK) >> PAGE_SHIFT); 4424 } 4425 4426 #ifdef CONFIG_MEMORY_FAILURE 4427 4428 /* 4429 * This function is called from memory failure code. 4430 */ 4431 int dequeue_hwpoisoned_huge_page(struct page *hpage) 4432 { 4433 struct hstate *h = page_hstate(hpage); 4434 int nid = page_to_nid(hpage); 4435 int ret = -EBUSY; 4436 4437 spin_lock(&hugetlb_lock); 4438 /* 4439 * Just checking !page_huge_active is not enough, because that could be 4440 * an isolated/hwpoisoned hugepage (which have >0 refcount). 4441 */ 4442 if (!page_huge_active(hpage) && !page_count(hpage)) { 4443 /* 4444 * Hwpoisoned hugepage isn't linked to activelist or freelist, 4445 * but dangling hpage->lru can trigger list-debug warnings 4446 * (this happens when we call unpoison_memory() on it), 4447 * so let it point to itself with list_del_init(). 4448 */ 4449 list_del_init(&hpage->lru); 4450 set_page_refcounted(hpage); 4451 h->free_huge_pages--; 4452 h->free_huge_pages_node[nid]--; 4453 ret = 0; 4454 } 4455 spin_unlock(&hugetlb_lock); 4456 return ret; 4457 } 4458 #endif 4459 4460 bool isolate_huge_page(struct page *page, struct list_head *list) 4461 { 4462 bool ret = true; 4463 4464 VM_BUG_ON_PAGE(!PageHead(page), page); 4465 spin_lock(&hugetlb_lock); 4466 if (!page_huge_active(page) || !get_page_unless_zero(page)) { 4467 ret = false; 4468 goto unlock; 4469 } 4470 clear_page_huge_active(page); 4471 list_move_tail(&page->lru, list); 4472 unlock: 4473 spin_unlock(&hugetlb_lock); 4474 return ret; 4475 } 4476 4477 void putback_active_hugepage(struct page *page) 4478 { 4479 VM_BUG_ON_PAGE(!PageHead(page), page); 4480 spin_lock(&hugetlb_lock); 4481 set_page_huge_active(page); 4482 list_move_tail(&page->lru, &(page_hstate(page))->hugepage_activelist); 4483 spin_unlock(&hugetlb_lock); 4484 put_page(page); 4485 } 4486