1 /* 2 * Generic hugetlb support. 3 * (C) Nadia Yvette Chambers, April 2004 4 */ 5 #include <linux/list.h> 6 #include <linux/init.h> 7 #include <linux/mm.h> 8 #include <linux/seq_file.h> 9 #include <linux/sysctl.h> 10 #include <linux/highmem.h> 11 #include <linux/mmu_notifier.h> 12 #include <linux/nodemask.h> 13 #include <linux/pagemap.h> 14 #include <linux/mempolicy.h> 15 #include <linux/compiler.h> 16 #include <linux/cpuset.h> 17 #include <linux/mutex.h> 18 #include <linux/bootmem.h> 19 #include <linux/sysfs.h> 20 #include <linux/slab.h> 21 #include <linux/rmap.h> 22 #include <linux/swap.h> 23 #include <linux/swapops.h> 24 #include <linux/page-isolation.h> 25 #include <linux/jhash.h> 26 27 #include <asm/page.h> 28 #include <asm/pgtable.h> 29 #include <asm/tlb.h> 30 31 #include <linux/io.h> 32 #include <linux/hugetlb.h> 33 #include <linux/hugetlb_cgroup.h> 34 #include <linux/node.h> 35 #include "internal.h" 36 37 int hugepages_treat_as_movable; 38 39 int hugetlb_max_hstate __read_mostly; 40 unsigned int default_hstate_idx; 41 struct hstate hstates[HUGE_MAX_HSTATE]; 42 /* 43 * Minimum page order among possible hugepage sizes, set to a proper value 44 * at boot time. 45 */ 46 static unsigned int minimum_order __read_mostly = UINT_MAX; 47 48 __initdata LIST_HEAD(huge_boot_pages); 49 50 /* for command line parsing */ 51 static struct hstate * __initdata parsed_hstate; 52 static unsigned long __initdata default_hstate_max_huge_pages; 53 static unsigned long __initdata default_hstate_size; 54 static bool __initdata parsed_valid_hugepagesz = true; 55 56 /* 57 * Protects updates to hugepage_freelists, hugepage_activelist, nr_huge_pages, 58 * free_huge_pages, and surplus_huge_pages. 59 */ 60 DEFINE_SPINLOCK(hugetlb_lock); 61 62 /* 63 * Serializes faults on the same logical page. This is used to 64 * prevent spurious OOMs when the hugepage pool is fully utilized. 65 */ 66 static int num_fault_mutexes; 67 struct mutex *hugetlb_fault_mutex_table ____cacheline_aligned_in_smp; 68 69 /* Forward declaration */ 70 static int hugetlb_acct_memory(struct hstate *h, long delta); 71 72 static inline void unlock_or_release_subpool(struct hugepage_subpool *spool) 73 { 74 bool free = (spool->count == 0) && (spool->used_hpages == 0); 75 76 spin_unlock(&spool->lock); 77 78 /* If no pages are used, and no other handles to the subpool 79 * remain, give up any reservations mased on minimum size and 80 * free the subpool */ 81 if (free) { 82 if (spool->min_hpages != -1) 83 hugetlb_acct_memory(spool->hstate, 84 -spool->min_hpages); 85 kfree(spool); 86 } 87 } 88 89 struct hugepage_subpool *hugepage_new_subpool(struct hstate *h, long max_hpages, 90 long min_hpages) 91 { 92 struct hugepage_subpool *spool; 93 94 spool = kzalloc(sizeof(*spool), GFP_KERNEL); 95 if (!spool) 96 return NULL; 97 98 spin_lock_init(&spool->lock); 99 spool->count = 1; 100 spool->max_hpages = max_hpages; 101 spool->hstate = h; 102 spool->min_hpages = min_hpages; 103 104 if (min_hpages != -1 && hugetlb_acct_memory(h, min_hpages)) { 105 kfree(spool); 106 return NULL; 107 } 108 spool->rsv_hpages = min_hpages; 109 110 return spool; 111 } 112 113 void hugepage_put_subpool(struct hugepage_subpool *spool) 114 { 115 spin_lock(&spool->lock); 116 BUG_ON(!spool->count); 117 spool->count--; 118 unlock_or_release_subpool(spool); 119 } 120 121 /* 122 * Subpool accounting for allocating and reserving pages. 123 * Return -ENOMEM if there are not enough resources to satisfy the 124 * the request. Otherwise, return the number of pages by which the 125 * global pools must be adjusted (upward). The returned value may 126 * only be different than the passed value (delta) in the case where 127 * a subpool minimum size must be manitained. 128 */ 129 static long hugepage_subpool_get_pages(struct hugepage_subpool *spool, 130 long delta) 131 { 132 long ret = delta; 133 134 if (!spool) 135 return ret; 136 137 spin_lock(&spool->lock); 138 139 if (spool->max_hpages != -1) { /* maximum size accounting */ 140 if ((spool->used_hpages + delta) <= spool->max_hpages) 141 spool->used_hpages += delta; 142 else { 143 ret = -ENOMEM; 144 goto unlock_ret; 145 } 146 } 147 148 /* minimum size accounting */ 149 if (spool->min_hpages != -1 && spool->rsv_hpages) { 150 if (delta > spool->rsv_hpages) { 151 /* 152 * Asking for more reserves than those already taken on 153 * behalf of subpool. Return difference. 154 */ 155 ret = delta - spool->rsv_hpages; 156 spool->rsv_hpages = 0; 157 } else { 158 ret = 0; /* reserves already accounted for */ 159 spool->rsv_hpages -= delta; 160 } 161 } 162 163 unlock_ret: 164 spin_unlock(&spool->lock); 165 return ret; 166 } 167 168 /* 169 * Subpool accounting for freeing and unreserving pages. 170 * Return the number of global page reservations that must be dropped. 171 * The return value may only be different than the passed value (delta) 172 * in the case where a subpool minimum size must be maintained. 173 */ 174 static long hugepage_subpool_put_pages(struct hugepage_subpool *spool, 175 long delta) 176 { 177 long ret = delta; 178 179 if (!spool) 180 return delta; 181 182 spin_lock(&spool->lock); 183 184 if (spool->max_hpages != -1) /* maximum size accounting */ 185 spool->used_hpages -= delta; 186 187 /* minimum size accounting */ 188 if (spool->min_hpages != -1 && spool->used_hpages < spool->min_hpages) { 189 if (spool->rsv_hpages + delta <= spool->min_hpages) 190 ret = 0; 191 else 192 ret = spool->rsv_hpages + delta - spool->min_hpages; 193 194 spool->rsv_hpages += delta; 195 if (spool->rsv_hpages > spool->min_hpages) 196 spool->rsv_hpages = spool->min_hpages; 197 } 198 199 /* 200 * If hugetlbfs_put_super couldn't free spool due to an outstanding 201 * quota reference, free it now. 202 */ 203 unlock_or_release_subpool(spool); 204 205 return ret; 206 } 207 208 static inline struct hugepage_subpool *subpool_inode(struct inode *inode) 209 { 210 return HUGETLBFS_SB(inode->i_sb)->spool; 211 } 212 213 static inline struct hugepage_subpool *subpool_vma(struct vm_area_struct *vma) 214 { 215 return subpool_inode(file_inode(vma->vm_file)); 216 } 217 218 /* 219 * Region tracking -- allows tracking of reservations and instantiated pages 220 * across the pages in a mapping. 221 * 222 * The region data structures are embedded into a resv_map and protected 223 * by a resv_map's lock. The set of regions within the resv_map represent 224 * reservations for huge pages, or huge pages that have already been 225 * instantiated within the map. The from and to elements are huge page 226 * indicies into the associated mapping. from indicates the starting index 227 * of the region. to represents the first index past the end of the region. 228 * 229 * For example, a file region structure with from == 0 and to == 4 represents 230 * four huge pages in a mapping. It is important to note that the to element 231 * represents the first element past the end of the region. This is used in 232 * arithmetic as 4(to) - 0(from) = 4 huge pages in the region. 233 * 234 * Interval notation of the form [from, to) will be used to indicate that 235 * the endpoint from is inclusive and to is exclusive. 236 */ 237 struct file_region { 238 struct list_head link; 239 long from; 240 long to; 241 }; 242 243 /* 244 * Add the huge page range represented by [f, t) to the reserve 245 * map. In the normal case, existing regions will be expanded 246 * to accommodate the specified range. Sufficient regions should 247 * exist for expansion due to the previous call to region_chg 248 * with the same range. However, it is possible that region_del 249 * could have been called after region_chg and modifed the map 250 * in such a way that no region exists to be expanded. In this 251 * case, pull a region descriptor from the cache associated with 252 * the map and use that for the new range. 253 * 254 * Return the number of new huge pages added to the map. This 255 * number is greater than or equal to zero. 256 */ 257 static long region_add(struct resv_map *resv, long f, long t) 258 { 259 struct list_head *head = &resv->regions; 260 struct file_region *rg, *nrg, *trg; 261 long add = 0; 262 263 spin_lock(&resv->lock); 264 /* Locate the region we are either in or before. */ 265 list_for_each_entry(rg, head, link) 266 if (f <= rg->to) 267 break; 268 269 /* 270 * If no region exists which can be expanded to include the 271 * specified range, the list must have been modified by an 272 * interleving call to region_del(). Pull a region descriptor 273 * from the cache and use it for this range. 274 */ 275 if (&rg->link == head || t < rg->from) { 276 VM_BUG_ON(resv->region_cache_count <= 0); 277 278 resv->region_cache_count--; 279 nrg = list_first_entry(&resv->region_cache, struct file_region, 280 link); 281 list_del(&nrg->link); 282 283 nrg->from = f; 284 nrg->to = t; 285 list_add(&nrg->link, rg->link.prev); 286 287 add += t - f; 288 goto out_locked; 289 } 290 291 /* Round our left edge to the current segment if it encloses us. */ 292 if (f > rg->from) 293 f = rg->from; 294 295 /* Check for and consume any regions we now overlap with. */ 296 nrg = rg; 297 list_for_each_entry_safe(rg, trg, rg->link.prev, link) { 298 if (&rg->link == head) 299 break; 300 if (rg->from > t) 301 break; 302 303 /* If this area reaches higher then extend our area to 304 * include it completely. If this is not the first area 305 * which we intend to reuse, free it. */ 306 if (rg->to > t) 307 t = rg->to; 308 if (rg != nrg) { 309 /* Decrement return value by the deleted range. 310 * Another range will span this area so that by 311 * end of routine add will be >= zero 312 */ 313 add -= (rg->to - rg->from); 314 list_del(&rg->link); 315 kfree(rg); 316 } 317 } 318 319 add += (nrg->from - f); /* Added to beginning of region */ 320 nrg->from = f; 321 add += t - nrg->to; /* Added to end of region */ 322 nrg->to = t; 323 324 out_locked: 325 resv->adds_in_progress--; 326 spin_unlock(&resv->lock); 327 VM_BUG_ON(add < 0); 328 return add; 329 } 330 331 /* 332 * Examine the existing reserve map and determine how many 333 * huge pages in the specified range [f, t) are NOT currently 334 * represented. This routine is called before a subsequent 335 * call to region_add that will actually modify the reserve 336 * map to add the specified range [f, t). region_chg does 337 * not change the number of huge pages represented by the 338 * map. However, if the existing regions in the map can not 339 * be expanded to represent the new range, a new file_region 340 * structure is added to the map as a placeholder. This is 341 * so that the subsequent region_add call will have all the 342 * regions it needs and will not fail. 343 * 344 * Upon entry, region_chg will also examine the cache of region descriptors 345 * associated with the map. If there are not enough descriptors cached, one 346 * will be allocated for the in progress add operation. 347 * 348 * Returns the number of huge pages that need to be added to the existing 349 * reservation map for the range [f, t). This number is greater or equal to 350 * zero. -ENOMEM is returned if a new file_region structure or cache entry 351 * is needed and can not be allocated. 352 */ 353 static long region_chg(struct resv_map *resv, long f, long t) 354 { 355 struct list_head *head = &resv->regions; 356 struct file_region *rg, *nrg = NULL; 357 long chg = 0; 358 359 retry: 360 spin_lock(&resv->lock); 361 retry_locked: 362 resv->adds_in_progress++; 363 364 /* 365 * Check for sufficient descriptors in the cache to accommodate 366 * the number of in progress add operations. 367 */ 368 if (resv->adds_in_progress > resv->region_cache_count) { 369 struct file_region *trg; 370 371 VM_BUG_ON(resv->adds_in_progress - resv->region_cache_count > 1); 372 /* Must drop lock to allocate a new descriptor. */ 373 resv->adds_in_progress--; 374 spin_unlock(&resv->lock); 375 376 trg = kmalloc(sizeof(*trg), GFP_KERNEL); 377 if (!trg) { 378 kfree(nrg); 379 return -ENOMEM; 380 } 381 382 spin_lock(&resv->lock); 383 list_add(&trg->link, &resv->region_cache); 384 resv->region_cache_count++; 385 goto retry_locked; 386 } 387 388 /* Locate the region we are before or in. */ 389 list_for_each_entry(rg, head, link) 390 if (f <= rg->to) 391 break; 392 393 /* If we are below the current region then a new region is required. 394 * Subtle, allocate a new region at the position but make it zero 395 * size such that we can guarantee to record the reservation. */ 396 if (&rg->link == head || t < rg->from) { 397 if (!nrg) { 398 resv->adds_in_progress--; 399 spin_unlock(&resv->lock); 400 nrg = kmalloc(sizeof(*nrg), GFP_KERNEL); 401 if (!nrg) 402 return -ENOMEM; 403 404 nrg->from = f; 405 nrg->to = f; 406 INIT_LIST_HEAD(&nrg->link); 407 goto retry; 408 } 409 410 list_add(&nrg->link, rg->link.prev); 411 chg = t - f; 412 goto out_nrg; 413 } 414 415 /* Round our left edge to the current segment if it encloses us. */ 416 if (f > rg->from) 417 f = rg->from; 418 chg = t - f; 419 420 /* Check for and consume any regions we now overlap with. */ 421 list_for_each_entry(rg, rg->link.prev, link) { 422 if (&rg->link == head) 423 break; 424 if (rg->from > t) 425 goto out; 426 427 /* We overlap with this area, if it extends further than 428 * us then we must extend ourselves. Account for its 429 * existing reservation. */ 430 if (rg->to > t) { 431 chg += rg->to - t; 432 t = rg->to; 433 } 434 chg -= rg->to - rg->from; 435 } 436 437 out: 438 spin_unlock(&resv->lock); 439 /* We already know we raced and no longer need the new region */ 440 kfree(nrg); 441 return chg; 442 out_nrg: 443 spin_unlock(&resv->lock); 444 return chg; 445 } 446 447 /* 448 * Abort the in progress add operation. The adds_in_progress field 449 * of the resv_map keeps track of the operations in progress between 450 * calls to region_chg and region_add. Operations are sometimes 451 * aborted after the call to region_chg. In such cases, region_abort 452 * is called to decrement the adds_in_progress counter. 453 * 454 * NOTE: The range arguments [f, t) are not needed or used in this 455 * routine. They are kept to make reading the calling code easier as 456 * arguments will match the associated region_chg call. 457 */ 458 static void region_abort(struct resv_map *resv, long f, long t) 459 { 460 spin_lock(&resv->lock); 461 VM_BUG_ON(!resv->region_cache_count); 462 resv->adds_in_progress--; 463 spin_unlock(&resv->lock); 464 } 465 466 /* 467 * Delete the specified range [f, t) from the reserve map. If the 468 * t parameter is LONG_MAX, this indicates that ALL regions after f 469 * should be deleted. Locate the regions which intersect [f, t) 470 * and either trim, delete or split the existing regions. 471 * 472 * Returns the number of huge pages deleted from the reserve map. 473 * In the normal case, the return value is zero or more. In the 474 * case where a region must be split, a new region descriptor must 475 * be allocated. If the allocation fails, -ENOMEM will be returned. 476 * NOTE: If the parameter t == LONG_MAX, then we will never split 477 * a region and possibly return -ENOMEM. Callers specifying 478 * t == LONG_MAX do not need to check for -ENOMEM error. 479 */ 480 static long region_del(struct resv_map *resv, long f, long t) 481 { 482 struct list_head *head = &resv->regions; 483 struct file_region *rg, *trg; 484 struct file_region *nrg = NULL; 485 long del = 0; 486 487 retry: 488 spin_lock(&resv->lock); 489 list_for_each_entry_safe(rg, trg, head, link) { 490 /* 491 * Skip regions before the range to be deleted. file_region 492 * ranges are normally of the form [from, to). However, there 493 * may be a "placeholder" entry in the map which is of the form 494 * (from, to) with from == to. Check for placeholder entries 495 * at the beginning of the range to be deleted. 496 */ 497 if (rg->to <= f && (rg->to != rg->from || rg->to != f)) 498 continue; 499 500 if (rg->from >= t) 501 break; 502 503 if (f > rg->from && t < rg->to) { /* Must split region */ 504 /* 505 * Check for an entry in the cache before dropping 506 * lock and attempting allocation. 507 */ 508 if (!nrg && 509 resv->region_cache_count > resv->adds_in_progress) { 510 nrg = list_first_entry(&resv->region_cache, 511 struct file_region, 512 link); 513 list_del(&nrg->link); 514 resv->region_cache_count--; 515 } 516 517 if (!nrg) { 518 spin_unlock(&resv->lock); 519 nrg = kmalloc(sizeof(*nrg), GFP_KERNEL); 520 if (!nrg) 521 return -ENOMEM; 522 goto retry; 523 } 524 525 del += t - f; 526 527 /* New entry for end of split region */ 528 nrg->from = t; 529 nrg->to = rg->to; 530 INIT_LIST_HEAD(&nrg->link); 531 532 /* Original entry is trimmed */ 533 rg->to = f; 534 535 list_add(&nrg->link, &rg->link); 536 nrg = NULL; 537 break; 538 } 539 540 if (f <= rg->from && t >= rg->to) { /* Remove entire region */ 541 del += rg->to - rg->from; 542 list_del(&rg->link); 543 kfree(rg); 544 continue; 545 } 546 547 if (f <= rg->from) { /* Trim beginning of region */ 548 del += t - rg->from; 549 rg->from = t; 550 } else { /* Trim end of region */ 551 del += rg->to - f; 552 rg->to = f; 553 } 554 } 555 556 spin_unlock(&resv->lock); 557 kfree(nrg); 558 return del; 559 } 560 561 /* 562 * A rare out of memory error was encountered which prevented removal of 563 * the reserve map region for a page. The huge page itself was free'ed 564 * and removed from the page cache. This routine will adjust the subpool 565 * usage count, and the global reserve count if needed. By incrementing 566 * these counts, the reserve map entry which could not be deleted will 567 * appear as a "reserved" entry instead of simply dangling with incorrect 568 * counts. 569 */ 570 void hugetlb_fix_reserve_counts(struct inode *inode) 571 { 572 struct hugepage_subpool *spool = subpool_inode(inode); 573 long rsv_adjust; 574 575 rsv_adjust = hugepage_subpool_get_pages(spool, 1); 576 if (rsv_adjust) { 577 struct hstate *h = hstate_inode(inode); 578 579 hugetlb_acct_memory(h, 1); 580 } 581 } 582 583 /* 584 * Count and return the number of huge pages in the reserve map 585 * that intersect with the range [f, t). 586 */ 587 static long region_count(struct resv_map *resv, long f, long t) 588 { 589 struct list_head *head = &resv->regions; 590 struct file_region *rg; 591 long chg = 0; 592 593 spin_lock(&resv->lock); 594 /* Locate each segment we overlap with, and count that overlap. */ 595 list_for_each_entry(rg, head, link) { 596 long seg_from; 597 long seg_to; 598 599 if (rg->to <= f) 600 continue; 601 if (rg->from >= t) 602 break; 603 604 seg_from = max(rg->from, f); 605 seg_to = min(rg->to, t); 606 607 chg += seg_to - seg_from; 608 } 609 spin_unlock(&resv->lock); 610 611 return chg; 612 } 613 614 /* 615 * Convert the address within this vma to the page offset within 616 * the mapping, in pagecache page units; huge pages here. 617 */ 618 static pgoff_t vma_hugecache_offset(struct hstate *h, 619 struct vm_area_struct *vma, unsigned long address) 620 { 621 return ((address - vma->vm_start) >> huge_page_shift(h)) + 622 (vma->vm_pgoff >> huge_page_order(h)); 623 } 624 625 pgoff_t linear_hugepage_index(struct vm_area_struct *vma, 626 unsigned long address) 627 { 628 return vma_hugecache_offset(hstate_vma(vma), vma, address); 629 } 630 EXPORT_SYMBOL_GPL(linear_hugepage_index); 631 632 /* 633 * Return the size of the pages allocated when backing a VMA. In the majority 634 * cases this will be same size as used by the page table entries. 635 */ 636 unsigned long vma_kernel_pagesize(struct vm_area_struct *vma) 637 { 638 struct hstate *hstate; 639 640 if (!is_vm_hugetlb_page(vma)) 641 return PAGE_SIZE; 642 643 hstate = hstate_vma(vma); 644 645 return 1UL << huge_page_shift(hstate); 646 } 647 EXPORT_SYMBOL_GPL(vma_kernel_pagesize); 648 649 /* 650 * Return the page size being used by the MMU to back a VMA. In the majority 651 * of cases, the page size used by the kernel matches the MMU size. On 652 * architectures where it differs, an architecture-specific version of this 653 * function is required. 654 */ 655 #ifndef vma_mmu_pagesize 656 unsigned long vma_mmu_pagesize(struct vm_area_struct *vma) 657 { 658 return vma_kernel_pagesize(vma); 659 } 660 #endif 661 662 /* 663 * Flags for MAP_PRIVATE reservations. These are stored in the bottom 664 * bits of the reservation map pointer, which are always clear due to 665 * alignment. 666 */ 667 #define HPAGE_RESV_OWNER (1UL << 0) 668 #define HPAGE_RESV_UNMAPPED (1UL << 1) 669 #define HPAGE_RESV_MASK (HPAGE_RESV_OWNER | HPAGE_RESV_UNMAPPED) 670 671 /* 672 * These helpers are used to track how many pages are reserved for 673 * faults in a MAP_PRIVATE mapping. Only the process that called mmap() 674 * is guaranteed to have their future faults succeed. 675 * 676 * With the exception of reset_vma_resv_huge_pages() which is called at fork(), 677 * the reserve counters are updated with the hugetlb_lock held. It is safe 678 * to reset the VMA at fork() time as it is not in use yet and there is no 679 * chance of the global counters getting corrupted as a result of the values. 680 * 681 * The private mapping reservation is represented in a subtly different 682 * manner to a shared mapping. A shared mapping has a region map associated 683 * with the underlying file, this region map represents the backing file 684 * pages which have ever had a reservation assigned which this persists even 685 * after the page is instantiated. A private mapping has a region map 686 * associated with the original mmap which is attached to all VMAs which 687 * reference it, this region map represents those offsets which have consumed 688 * reservation ie. where pages have been instantiated. 689 */ 690 static unsigned long get_vma_private_data(struct vm_area_struct *vma) 691 { 692 return (unsigned long)vma->vm_private_data; 693 } 694 695 static void set_vma_private_data(struct vm_area_struct *vma, 696 unsigned long value) 697 { 698 vma->vm_private_data = (void *)value; 699 } 700 701 struct resv_map *resv_map_alloc(void) 702 { 703 struct resv_map *resv_map = kmalloc(sizeof(*resv_map), GFP_KERNEL); 704 struct file_region *rg = kmalloc(sizeof(*rg), GFP_KERNEL); 705 706 if (!resv_map || !rg) { 707 kfree(resv_map); 708 kfree(rg); 709 return NULL; 710 } 711 712 kref_init(&resv_map->refs); 713 spin_lock_init(&resv_map->lock); 714 INIT_LIST_HEAD(&resv_map->regions); 715 716 resv_map->adds_in_progress = 0; 717 718 INIT_LIST_HEAD(&resv_map->region_cache); 719 list_add(&rg->link, &resv_map->region_cache); 720 resv_map->region_cache_count = 1; 721 722 return resv_map; 723 } 724 725 void resv_map_release(struct kref *ref) 726 { 727 struct resv_map *resv_map = container_of(ref, struct resv_map, refs); 728 struct list_head *head = &resv_map->region_cache; 729 struct file_region *rg, *trg; 730 731 /* Clear out any active regions before we release the map. */ 732 region_del(resv_map, 0, LONG_MAX); 733 734 /* ... and any entries left in the cache */ 735 list_for_each_entry_safe(rg, trg, head, link) { 736 list_del(&rg->link); 737 kfree(rg); 738 } 739 740 VM_BUG_ON(resv_map->adds_in_progress); 741 742 kfree(resv_map); 743 } 744 745 static inline struct resv_map *inode_resv_map(struct inode *inode) 746 { 747 return inode->i_mapping->private_data; 748 } 749 750 static struct resv_map *vma_resv_map(struct vm_area_struct *vma) 751 { 752 VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma); 753 if (vma->vm_flags & VM_MAYSHARE) { 754 struct address_space *mapping = vma->vm_file->f_mapping; 755 struct inode *inode = mapping->host; 756 757 return inode_resv_map(inode); 758 759 } else { 760 return (struct resv_map *)(get_vma_private_data(vma) & 761 ~HPAGE_RESV_MASK); 762 } 763 } 764 765 static void set_vma_resv_map(struct vm_area_struct *vma, struct resv_map *map) 766 { 767 VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma); 768 VM_BUG_ON_VMA(vma->vm_flags & VM_MAYSHARE, vma); 769 770 set_vma_private_data(vma, (get_vma_private_data(vma) & 771 HPAGE_RESV_MASK) | (unsigned long)map); 772 } 773 774 static void set_vma_resv_flags(struct vm_area_struct *vma, unsigned long flags) 775 { 776 VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma); 777 VM_BUG_ON_VMA(vma->vm_flags & VM_MAYSHARE, vma); 778 779 set_vma_private_data(vma, get_vma_private_data(vma) | flags); 780 } 781 782 static int is_vma_resv_set(struct vm_area_struct *vma, unsigned long flag) 783 { 784 VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma); 785 786 return (get_vma_private_data(vma) & flag) != 0; 787 } 788 789 /* Reset counters to 0 and clear all HPAGE_RESV_* flags */ 790 void reset_vma_resv_huge_pages(struct vm_area_struct *vma) 791 { 792 VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma); 793 if (!(vma->vm_flags & VM_MAYSHARE)) 794 vma->vm_private_data = (void *)0; 795 } 796 797 /* Returns true if the VMA has associated reserve pages */ 798 static bool vma_has_reserves(struct vm_area_struct *vma, long chg) 799 { 800 if (vma->vm_flags & VM_NORESERVE) { 801 /* 802 * This address is already reserved by other process(chg == 0), 803 * so, we should decrement reserved count. Without decrementing, 804 * reserve count remains after releasing inode, because this 805 * allocated page will go into page cache and is regarded as 806 * coming from reserved pool in releasing step. Currently, we 807 * don't have any other solution to deal with this situation 808 * properly, so add work-around here. 809 */ 810 if (vma->vm_flags & VM_MAYSHARE && chg == 0) 811 return true; 812 else 813 return false; 814 } 815 816 /* Shared mappings always use reserves */ 817 if (vma->vm_flags & VM_MAYSHARE) { 818 /* 819 * We know VM_NORESERVE is not set. Therefore, there SHOULD 820 * be a region map for all pages. The only situation where 821 * there is no region map is if a hole was punched via 822 * fallocate. In this case, there really are no reverves to 823 * use. This situation is indicated if chg != 0. 824 */ 825 if (chg) 826 return false; 827 else 828 return true; 829 } 830 831 /* 832 * Only the process that called mmap() has reserves for 833 * private mappings. 834 */ 835 if (is_vma_resv_set(vma, HPAGE_RESV_OWNER)) { 836 /* 837 * Like the shared case above, a hole punch or truncate 838 * could have been performed on the private mapping. 839 * Examine the value of chg to determine if reserves 840 * actually exist or were previously consumed. 841 * Very Subtle - The value of chg comes from a previous 842 * call to vma_needs_reserves(). The reserve map for 843 * private mappings has different (opposite) semantics 844 * than that of shared mappings. vma_needs_reserves() 845 * has already taken this difference in semantics into 846 * account. Therefore, the meaning of chg is the same 847 * as in the shared case above. Code could easily be 848 * combined, but keeping it separate draws attention to 849 * subtle differences. 850 */ 851 if (chg) 852 return false; 853 else 854 return true; 855 } 856 857 return false; 858 } 859 860 static void enqueue_huge_page(struct hstate *h, struct page *page) 861 { 862 int nid = page_to_nid(page); 863 list_move(&page->lru, &h->hugepage_freelists[nid]); 864 h->free_huge_pages++; 865 h->free_huge_pages_node[nid]++; 866 } 867 868 static struct page *dequeue_huge_page_node(struct hstate *h, int nid) 869 { 870 struct page *page; 871 872 list_for_each_entry(page, &h->hugepage_freelists[nid], lru) 873 if (!is_migrate_isolate_page(page)) 874 break; 875 /* 876 * if 'non-isolated free hugepage' not found on the list, 877 * the allocation fails. 878 */ 879 if (&h->hugepage_freelists[nid] == &page->lru) 880 return NULL; 881 list_move(&page->lru, &h->hugepage_activelist); 882 set_page_refcounted(page); 883 h->free_huge_pages--; 884 h->free_huge_pages_node[nid]--; 885 return page; 886 } 887 888 /* Movability of hugepages depends on migration support. */ 889 static inline gfp_t htlb_alloc_mask(struct hstate *h) 890 { 891 if (hugepages_treat_as_movable || hugepage_migration_supported(h)) 892 return GFP_HIGHUSER_MOVABLE; 893 else 894 return GFP_HIGHUSER; 895 } 896 897 static struct page *dequeue_huge_page_vma(struct hstate *h, 898 struct vm_area_struct *vma, 899 unsigned long address, int avoid_reserve, 900 long chg) 901 { 902 struct page *page = NULL; 903 struct mempolicy *mpol; 904 nodemask_t *nodemask; 905 struct zonelist *zonelist; 906 struct zone *zone; 907 struct zoneref *z; 908 unsigned int cpuset_mems_cookie; 909 910 /* 911 * A child process with MAP_PRIVATE mappings created by their parent 912 * have no page reserves. This check ensures that reservations are 913 * not "stolen". The child may still get SIGKILLed 914 */ 915 if (!vma_has_reserves(vma, chg) && 916 h->free_huge_pages - h->resv_huge_pages == 0) 917 goto err; 918 919 /* If reserves cannot be used, ensure enough pages are in the pool */ 920 if (avoid_reserve && h->free_huge_pages - h->resv_huge_pages == 0) 921 goto err; 922 923 retry_cpuset: 924 cpuset_mems_cookie = read_mems_allowed_begin(); 925 zonelist = huge_zonelist(vma, address, 926 htlb_alloc_mask(h), &mpol, &nodemask); 927 928 for_each_zone_zonelist_nodemask(zone, z, zonelist, 929 MAX_NR_ZONES - 1, nodemask) { 930 if (cpuset_zone_allowed(zone, htlb_alloc_mask(h))) { 931 page = dequeue_huge_page_node(h, zone_to_nid(zone)); 932 if (page) { 933 if (avoid_reserve) 934 break; 935 if (!vma_has_reserves(vma, chg)) 936 break; 937 938 SetPagePrivate(page); 939 h->resv_huge_pages--; 940 break; 941 } 942 } 943 } 944 945 mpol_cond_put(mpol); 946 if (unlikely(!page && read_mems_allowed_retry(cpuset_mems_cookie))) 947 goto retry_cpuset; 948 return page; 949 950 err: 951 return NULL; 952 } 953 954 /* 955 * common helper functions for hstate_next_node_to_{alloc|free}. 956 * We may have allocated or freed a huge page based on a different 957 * nodes_allowed previously, so h->next_node_to_{alloc|free} might 958 * be outside of *nodes_allowed. Ensure that we use an allowed 959 * node for alloc or free. 960 */ 961 static int next_node_allowed(int nid, nodemask_t *nodes_allowed) 962 { 963 nid = next_node_in(nid, *nodes_allowed); 964 VM_BUG_ON(nid >= MAX_NUMNODES); 965 966 return nid; 967 } 968 969 static int get_valid_node_allowed(int nid, nodemask_t *nodes_allowed) 970 { 971 if (!node_isset(nid, *nodes_allowed)) 972 nid = next_node_allowed(nid, nodes_allowed); 973 return nid; 974 } 975 976 /* 977 * returns the previously saved node ["this node"] from which to 978 * allocate a persistent huge page for the pool and advance the 979 * next node from which to allocate, handling wrap at end of node 980 * mask. 981 */ 982 static int hstate_next_node_to_alloc(struct hstate *h, 983 nodemask_t *nodes_allowed) 984 { 985 int nid; 986 987 VM_BUG_ON(!nodes_allowed); 988 989 nid = get_valid_node_allowed(h->next_nid_to_alloc, nodes_allowed); 990 h->next_nid_to_alloc = next_node_allowed(nid, nodes_allowed); 991 992 return nid; 993 } 994 995 /* 996 * helper for free_pool_huge_page() - return the previously saved 997 * node ["this node"] from which to free a huge page. Advance the 998 * next node id whether or not we find a free huge page to free so 999 * that the next attempt to free addresses the next node. 1000 */ 1001 static int hstate_next_node_to_free(struct hstate *h, nodemask_t *nodes_allowed) 1002 { 1003 int nid; 1004 1005 VM_BUG_ON(!nodes_allowed); 1006 1007 nid = get_valid_node_allowed(h->next_nid_to_free, nodes_allowed); 1008 h->next_nid_to_free = next_node_allowed(nid, nodes_allowed); 1009 1010 return nid; 1011 } 1012 1013 #define for_each_node_mask_to_alloc(hs, nr_nodes, node, mask) \ 1014 for (nr_nodes = nodes_weight(*mask); \ 1015 nr_nodes > 0 && \ 1016 ((node = hstate_next_node_to_alloc(hs, mask)) || 1); \ 1017 nr_nodes--) 1018 1019 #define for_each_node_mask_to_free(hs, nr_nodes, node, mask) \ 1020 for (nr_nodes = nodes_weight(*mask); \ 1021 nr_nodes > 0 && \ 1022 ((node = hstate_next_node_to_free(hs, mask)) || 1); \ 1023 nr_nodes--) 1024 1025 #if defined(CONFIG_ARCH_HAS_GIGANTIC_PAGE) && \ 1026 ((defined(CONFIG_MEMORY_ISOLATION) && defined(CONFIG_COMPACTION)) || \ 1027 defined(CONFIG_CMA)) 1028 static void destroy_compound_gigantic_page(struct page *page, 1029 unsigned int order) 1030 { 1031 int i; 1032 int nr_pages = 1 << order; 1033 struct page *p = page + 1; 1034 1035 atomic_set(compound_mapcount_ptr(page), 0); 1036 for (i = 1; i < nr_pages; i++, p = mem_map_next(p, page, i)) { 1037 clear_compound_head(p); 1038 set_page_refcounted(p); 1039 } 1040 1041 set_compound_order(page, 0); 1042 __ClearPageHead(page); 1043 } 1044 1045 static void free_gigantic_page(struct page *page, unsigned int order) 1046 { 1047 free_contig_range(page_to_pfn(page), 1 << order); 1048 } 1049 1050 static int __alloc_gigantic_page(unsigned long start_pfn, 1051 unsigned long nr_pages) 1052 { 1053 unsigned long end_pfn = start_pfn + nr_pages; 1054 return alloc_contig_range(start_pfn, end_pfn, MIGRATE_MOVABLE); 1055 } 1056 1057 static bool pfn_range_valid_gigantic(struct zone *z, 1058 unsigned long start_pfn, unsigned long nr_pages) 1059 { 1060 unsigned long i, end_pfn = start_pfn + nr_pages; 1061 struct page *page; 1062 1063 for (i = start_pfn; i < end_pfn; i++) { 1064 if (!pfn_valid(i)) 1065 return false; 1066 1067 page = pfn_to_page(i); 1068 1069 if (page_zone(page) != z) 1070 return false; 1071 1072 if (PageReserved(page)) 1073 return false; 1074 1075 if (page_count(page) > 0) 1076 return false; 1077 1078 if (PageHuge(page)) 1079 return false; 1080 } 1081 1082 return true; 1083 } 1084 1085 static bool zone_spans_last_pfn(const struct zone *zone, 1086 unsigned long start_pfn, unsigned long nr_pages) 1087 { 1088 unsigned long last_pfn = start_pfn + nr_pages - 1; 1089 return zone_spans_pfn(zone, last_pfn); 1090 } 1091 1092 static struct page *alloc_gigantic_page(int nid, unsigned int order) 1093 { 1094 unsigned long nr_pages = 1 << order; 1095 unsigned long ret, pfn, flags; 1096 struct zone *z; 1097 1098 z = NODE_DATA(nid)->node_zones; 1099 for (; z - NODE_DATA(nid)->node_zones < MAX_NR_ZONES; z++) { 1100 spin_lock_irqsave(&z->lock, flags); 1101 1102 pfn = ALIGN(z->zone_start_pfn, nr_pages); 1103 while (zone_spans_last_pfn(z, pfn, nr_pages)) { 1104 if (pfn_range_valid_gigantic(z, pfn, nr_pages)) { 1105 /* 1106 * We release the zone lock here because 1107 * alloc_contig_range() will also lock the zone 1108 * at some point. If there's an allocation 1109 * spinning on this lock, it may win the race 1110 * and cause alloc_contig_range() to fail... 1111 */ 1112 spin_unlock_irqrestore(&z->lock, flags); 1113 ret = __alloc_gigantic_page(pfn, nr_pages); 1114 if (!ret) 1115 return pfn_to_page(pfn); 1116 spin_lock_irqsave(&z->lock, flags); 1117 } 1118 pfn += nr_pages; 1119 } 1120 1121 spin_unlock_irqrestore(&z->lock, flags); 1122 } 1123 1124 return NULL; 1125 } 1126 1127 static void prep_new_huge_page(struct hstate *h, struct page *page, int nid); 1128 static void prep_compound_gigantic_page(struct page *page, unsigned int order); 1129 1130 static struct page *alloc_fresh_gigantic_page_node(struct hstate *h, int nid) 1131 { 1132 struct page *page; 1133 1134 page = alloc_gigantic_page(nid, huge_page_order(h)); 1135 if (page) { 1136 prep_compound_gigantic_page(page, huge_page_order(h)); 1137 prep_new_huge_page(h, page, nid); 1138 } 1139 1140 return page; 1141 } 1142 1143 static int alloc_fresh_gigantic_page(struct hstate *h, 1144 nodemask_t *nodes_allowed) 1145 { 1146 struct page *page = NULL; 1147 int nr_nodes, node; 1148 1149 for_each_node_mask_to_alloc(h, nr_nodes, node, nodes_allowed) { 1150 page = alloc_fresh_gigantic_page_node(h, node); 1151 if (page) 1152 return 1; 1153 } 1154 1155 return 0; 1156 } 1157 1158 static inline bool gigantic_page_supported(void) { return true; } 1159 #else 1160 static inline bool gigantic_page_supported(void) { return false; } 1161 static inline void free_gigantic_page(struct page *page, unsigned int order) { } 1162 static inline void destroy_compound_gigantic_page(struct page *page, 1163 unsigned int order) { } 1164 static inline int alloc_fresh_gigantic_page(struct hstate *h, 1165 nodemask_t *nodes_allowed) { return 0; } 1166 #endif 1167 1168 static void update_and_free_page(struct hstate *h, struct page *page) 1169 { 1170 int i; 1171 1172 if (hstate_is_gigantic(h) && !gigantic_page_supported()) 1173 return; 1174 1175 h->nr_huge_pages--; 1176 h->nr_huge_pages_node[page_to_nid(page)]--; 1177 for (i = 0; i < pages_per_huge_page(h); i++) { 1178 page[i].flags &= ~(1 << PG_locked | 1 << PG_error | 1179 1 << PG_referenced | 1 << PG_dirty | 1180 1 << PG_active | 1 << PG_private | 1181 1 << PG_writeback); 1182 } 1183 VM_BUG_ON_PAGE(hugetlb_cgroup_from_page(page), page); 1184 set_compound_page_dtor(page, NULL_COMPOUND_DTOR); 1185 set_page_refcounted(page); 1186 if (hstate_is_gigantic(h)) { 1187 destroy_compound_gigantic_page(page, huge_page_order(h)); 1188 free_gigantic_page(page, huge_page_order(h)); 1189 } else { 1190 __free_pages(page, huge_page_order(h)); 1191 } 1192 } 1193 1194 struct hstate *size_to_hstate(unsigned long size) 1195 { 1196 struct hstate *h; 1197 1198 for_each_hstate(h) { 1199 if (huge_page_size(h) == size) 1200 return h; 1201 } 1202 return NULL; 1203 } 1204 1205 /* 1206 * Test to determine whether the hugepage is "active/in-use" (i.e. being linked 1207 * to hstate->hugepage_activelist.) 1208 * 1209 * This function can be called for tail pages, but never returns true for them. 1210 */ 1211 bool page_huge_active(struct page *page) 1212 { 1213 VM_BUG_ON_PAGE(!PageHuge(page), page); 1214 return PageHead(page) && PagePrivate(&page[1]); 1215 } 1216 1217 /* never called for tail page */ 1218 static void set_page_huge_active(struct page *page) 1219 { 1220 VM_BUG_ON_PAGE(!PageHeadHuge(page), page); 1221 SetPagePrivate(&page[1]); 1222 } 1223 1224 static void clear_page_huge_active(struct page *page) 1225 { 1226 VM_BUG_ON_PAGE(!PageHeadHuge(page), page); 1227 ClearPagePrivate(&page[1]); 1228 } 1229 1230 void free_huge_page(struct page *page) 1231 { 1232 /* 1233 * Can't pass hstate in here because it is called from the 1234 * compound page destructor. 1235 */ 1236 struct hstate *h = page_hstate(page); 1237 int nid = page_to_nid(page); 1238 struct hugepage_subpool *spool = 1239 (struct hugepage_subpool *)page_private(page); 1240 bool restore_reserve; 1241 1242 set_page_private(page, 0); 1243 page->mapping = NULL; 1244 VM_BUG_ON_PAGE(page_count(page), page); 1245 VM_BUG_ON_PAGE(page_mapcount(page), page); 1246 restore_reserve = PagePrivate(page); 1247 ClearPagePrivate(page); 1248 1249 /* 1250 * A return code of zero implies that the subpool will be under its 1251 * minimum size if the reservation is not restored after page is free. 1252 * Therefore, force restore_reserve operation. 1253 */ 1254 if (hugepage_subpool_put_pages(spool, 1) == 0) 1255 restore_reserve = true; 1256 1257 spin_lock(&hugetlb_lock); 1258 clear_page_huge_active(page); 1259 hugetlb_cgroup_uncharge_page(hstate_index(h), 1260 pages_per_huge_page(h), page); 1261 if (restore_reserve) 1262 h->resv_huge_pages++; 1263 1264 if (h->surplus_huge_pages_node[nid]) { 1265 /* remove the page from active list */ 1266 list_del(&page->lru); 1267 update_and_free_page(h, page); 1268 h->surplus_huge_pages--; 1269 h->surplus_huge_pages_node[nid]--; 1270 } else { 1271 arch_clear_hugepage_flags(page); 1272 enqueue_huge_page(h, page); 1273 } 1274 spin_unlock(&hugetlb_lock); 1275 } 1276 1277 static void prep_new_huge_page(struct hstate *h, struct page *page, int nid) 1278 { 1279 INIT_LIST_HEAD(&page->lru); 1280 set_compound_page_dtor(page, HUGETLB_PAGE_DTOR); 1281 spin_lock(&hugetlb_lock); 1282 set_hugetlb_cgroup(page, NULL); 1283 h->nr_huge_pages++; 1284 h->nr_huge_pages_node[nid]++; 1285 spin_unlock(&hugetlb_lock); 1286 put_page(page); /* free it into the hugepage allocator */ 1287 } 1288 1289 static void prep_compound_gigantic_page(struct page *page, unsigned int order) 1290 { 1291 int i; 1292 int nr_pages = 1 << order; 1293 struct page *p = page + 1; 1294 1295 /* we rely on prep_new_huge_page to set the destructor */ 1296 set_compound_order(page, order); 1297 __ClearPageReserved(page); 1298 __SetPageHead(page); 1299 for (i = 1; i < nr_pages; i++, p = mem_map_next(p, page, i)) { 1300 /* 1301 * For gigantic hugepages allocated through bootmem at 1302 * boot, it's safer to be consistent with the not-gigantic 1303 * hugepages and clear the PG_reserved bit from all tail pages 1304 * too. Otherwse drivers using get_user_pages() to access tail 1305 * pages may get the reference counting wrong if they see 1306 * PG_reserved set on a tail page (despite the head page not 1307 * having PG_reserved set). Enforcing this consistency between 1308 * head and tail pages allows drivers to optimize away a check 1309 * on the head page when they need know if put_page() is needed 1310 * after get_user_pages(). 1311 */ 1312 __ClearPageReserved(p); 1313 set_page_count(p, 0); 1314 set_compound_head(p, page); 1315 } 1316 atomic_set(compound_mapcount_ptr(page), -1); 1317 } 1318 1319 /* 1320 * PageHuge() only returns true for hugetlbfs pages, but not for normal or 1321 * transparent huge pages. See the PageTransHuge() documentation for more 1322 * details. 1323 */ 1324 int PageHuge(struct page *page) 1325 { 1326 if (!PageCompound(page)) 1327 return 0; 1328 1329 page = compound_head(page); 1330 return page[1].compound_dtor == HUGETLB_PAGE_DTOR; 1331 } 1332 EXPORT_SYMBOL_GPL(PageHuge); 1333 1334 /* 1335 * PageHeadHuge() only returns true for hugetlbfs head page, but not for 1336 * normal or transparent huge pages. 1337 */ 1338 int PageHeadHuge(struct page *page_head) 1339 { 1340 if (!PageHead(page_head)) 1341 return 0; 1342 1343 return get_compound_page_dtor(page_head) == free_huge_page; 1344 } 1345 1346 pgoff_t __basepage_index(struct page *page) 1347 { 1348 struct page *page_head = compound_head(page); 1349 pgoff_t index = page_index(page_head); 1350 unsigned long compound_idx; 1351 1352 if (!PageHuge(page_head)) 1353 return page_index(page); 1354 1355 if (compound_order(page_head) >= MAX_ORDER) 1356 compound_idx = page_to_pfn(page) - page_to_pfn(page_head); 1357 else 1358 compound_idx = page - page_head; 1359 1360 return (index << compound_order(page_head)) + compound_idx; 1361 } 1362 1363 static struct page *alloc_fresh_huge_page_node(struct hstate *h, int nid) 1364 { 1365 struct page *page; 1366 1367 page = __alloc_pages_node(nid, 1368 htlb_alloc_mask(h)|__GFP_COMP|__GFP_THISNODE| 1369 __GFP_REPEAT|__GFP_NOWARN, 1370 huge_page_order(h)); 1371 if (page) { 1372 prep_new_huge_page(h, page, nid); 1373 } 1374 1375 return page; 1376 } 1377 1378 static int alloc_fresh_huge_page(struct hstate *h, nodemask_t *nodes_allowed) 1379 { 1380 struct page *page; 1381 int nr_nodes, node; 1382 int ret = 0; 1383 1384 for_each_node_mask_to_alloc(h, nr_nodes, node, nodes_allowed) { 1385 page = alloc_fresh_huge_page_node(h, node); 1386 if (page) { 1387 ret = 1; 1388 break; 1389 } 1390 } 1391 1392 if (ret) 1393 count_vm_event(HTLB_BUDDY_PGALLOC); 1394 else 1395 count_vm_event(HTLB_BUDDY_PGALLOC_FAIL); 1396 1397 return ret; 1398 } 1399 1400 /* 1401 * Free huge page from pool from next node to free. 1402 * Attempt to keep persistent huge pages more or less 1403 * balanced over allowed nodes. 1404 * Called with hugetlb_lock locked. 1405 */ 1406 static int free_pool_huge_page(struct hstate *h, nodemask_t *nodes_allowed, 1407 bool acct_surplus) 1408 { 1409 int nr_nodes, node; 1410 int ret = 0; 1411 1412 for_each_node_mask_to_free(h, nr_nodes, node, nodes_allowed) { 1413 /* 1414 * If we're returning unused surplus pages, only examine 1415 * nodes with surplus pages. 1416 */ 1417 if ((!acct_surplus || h->surplus_huge_pages_node[node]) && 1418 !list_empty(&h->hugepage_freelists[node])) { 1419 struct page *page = 1420 list_entry(h->hugepage_freelists[node].next, 1421 struct page, lru); 1422 list_del(&page->lru); 1423 h->free_huge_pages--; 1424 h->free_huge_pages_node[node]--; 1425 if (acct_surplus) { 1426 h->surplus_huge_pages--; 1427 h->surplus_huge_pages_node[node]--; 1428 } 1429 update_and_free_page(h, page); 1430 ret = 1; 1431 break; 1432 } 1433 } 1434 1435 return ret; 1436 } 1437 1438 /* 1439 * Dissolve a given free hugepage into free buddy pages. This function does 1440 * nothing for in-use (including surplus) hugepages. Returns -EBUSY if the 1441 * number of free hugepages would be reduced below the number of reserved 1442 * hugepages. 1443 */ 1444 static int dissolve_free_huge_page(struct page *page) 1445 { 1446 int rc = 0; 1447 1448 spin_lock(&hugetlb_lock); 1449 if (PageHuge(page) && !page_count(page)) { 1450 struct page *head = compound_head(page); 1451 struct hstate *h = page_hstate(head); 1452 int nid = page_to_nid(head); 1453 if (h->free_huge_pages - h->resv_huge_pages == 0) { 1454 rc = -EBUSY; 1455 goto out; 1456 } 1457 list_del(&head->lru); 1458 h->free_huge_pages--; 1459 h->free_huge_pages_node[nid]--; 1460 h->max_huge_pages--; 1461 update_and_free_page(h, head); 1462 } 1463 out: 1464 spin_unlock(&hugetlb_lock); 1465 return rc; 1466 } 1467 1468 /* 1469 * Dissolve free hugepages in a given pfn range. Used by memory hotplug to 1470 * make specified memory blocks removable from the system. 1471 * Note that this will dissolve a free gigantic hugepage completely, if any 1472 * part of it lies within the given range. 1473 * Also note that if dissolve_free_huge_page() returns with an error, all 1474 * free hugepages that were dissolved before that error are lost. 1475 */ 1476 int dissolve_free_huge_pages(unsigned long start_pfn, unsigned long end_pfn) 1477 { 1478 unsigned long pfn; 1479 struct page *page; 1480 int rc = 0; 1481 1482 if (!hugepages_supported()) 1483 return rc; 1484 1485 for (pfn = start_pfn; pfn < end_pfn; pfn += 1 << minimum_order) { 1486 page = pfn_to_page(pfn); 1487 if (PageHuge(page) && !page_count(page)) { 1488 rc = dissolve_free_huge_page(page); 1489 if (rc) 1490 break; 1491 } 1492 } 1493 1494 return rc; 1495 } 1496 1497 /* 1498 * There are 3 ways this can get called: 1499 * 1. With vma+addr: we use the VMA's memory policy 1500 * 2. With !vma, but nid=NUMA_NO_NODE: We try to allocate a huge 1501 * page from any node, and let the buddy allocator itself figure 1502 * it out. 1503 * 3. With !vma, but nid!=NUMA_NO_NODE. We allocate a huge page 1504 * strictly from 'nid' 1505 */ 1506 static struct page *__hugetlb_alloc_buddy_huge_page(struct hstate *h, 1507 struct vm_area_struct *vma, unsigned long addr, int nid) 1508 { 1509 int order = huge_page_order(h); 1510 gfp_t gfp = htlb_alloc_mask(h)|__GFP_COMP|__GFP_REPEAT|__GFP_NOWARN; 1511 unsigned int cpuset_mems_cookie; 1512 1513 /* 1514 * We need a VMA to get a memory policy. If we do not 1515 * have one, we use the 'nid' argument. 1516 * 1517 * The mempolicy stuff below has some non-inlined bits 1518 * and calls ->vm_ops. That makes it hard to optimize at 1519 * compile-time, even when NUMA is off and it does 1520 * nothing. This helps the compiler optimize it out. 1521 */ 1522 if (!IS_ENABLED(CONFIG_NUMA) || !vma) { 1523 /* 1524 * If a specific node is requested, make sure to 1525 * get memory from there, but only when a node 1526 * is explicitly specified. 1527 */ 1528 if (nid != NUMA_NO_NODE) 1529 gfp |= __GFP_THISNODE; 1530 /* 1531 * Make sure to call something that can handle 1532 * nid=NUMA_NO_NODE 1533 */ 1534 return alloc_pages_node(nid, gfp, order); 1535 } 1536 1537 /* 1538 * OK, so we have a VMA. Fetch the mempolicy and try to 1539 * allocate a huge page with it. We will only reach this 1540 * when CONFIG_NUMA=y. 1541 */ 1542 do { 1543 struct page *page; 1544 struct mempolicy *mpol; 1545 struct zonelist *zl; 1546 nodemask_t *nodemask; 1547 1548 cpuset_mems_cookie = read_mems_allowed_begin(); 1549 zl = huge_zonelist(vma, addr, gfp, &mpol, &nodemask); 1550 mpol_cond_put(mpol); 1551 page = __alloc_pages_nodemask(gfp, order, zl, nodemask); 1552 if (page) 1553 return page; 1554 } while (read_mems_allowed_retry(cpuset_mems_cookie)); 1555 1556 return NULL; 1557 } 1558 1559 /* 1560 * There are two ways to allocate a huge page: 1561 * 1. When you have a VMA and an address (like a fault) 1562 * 2. When you have no VMA (like when setting /proc/.../nr_hugepages) 1563 * 1564 * 'vma' and 'addr' are only for (1). 'nid' is always NUMA_NO_NODE in 1565 * this case which signifies that the allocation should be done with 1566 * respect for the VMA's memory policy. 1567 * 1568 * For (2), we ignore 'vma' and 'addr' and use 'nid' exclusively. This 1569 * implies that memory policies will not be taken in to account. 1570 */ 1571 static struct page *__alloc_buddy_huge_page(struct hstate *h, 1572 struct vm_area_struct *vma, unsigned long addr, int nid) 1573 { 1574 struct page *page; 1575 unsigned int r_nid; 1576 1577 if (hstate_is_gigantic(h)) 1578 return NULL; 1579 1580 /* 1581 * Make sure that anyone specifying 'nid' is not also specifying a VMA. 1582 * This makes sure the caller is picking _one_ of the modes with which 1583 * we can call this function, not both. 1584 */ 1585 if (vma || (addr != -1)) { 1586 VM_WARN_ON_ONCE(addr == -1); 1587 VM_WARN_ON_ONCE(nid != NUMA_NO_NODE); 1588 } 1589 /* 1590 * Assume we will successfully allocate the surplus page to 1591 * prevent racing processes from causing the surplus to exceed 1592 * overcommit 1593 * 1594 * This however introduces a different race, where a process B 1595 * tries to grow the static hugepage pool while alloc_pages() is 1596 * called by process A. B will only examine the per-node 1597 * counters in determining if surplus huge pages can be 1598 * converted to normal huge pages in adjust_pool_surplus(). A 1599 * won't be able to increment the per-node counter, until the 1600 * lock is dropped by B, but B doesn't drop hugetlb_lock until 1601 * no more huge pages can be converted from surplus to normal 1602 * state (and doesn't try to convert again). Thus, we have a 1603 * case where a surplus huge page exists, the pool is grown, and 1604 * the surplus huge page still exists after, even though it 1605 * should just have been converted to a normal huge page. This 1606 * does not leak memory, though, as the hugepage will be freed 1607 * once it is out of use. It also does not allow the counters to 1608 * go out of whack in adjust_pool_surplus() as we don't modify 1609 * the node values until we've gotten the hugepage and only the 1610 * per-node value is checked there. 1611 */ 1612 spin_lock(&hugetlb_lock); 1613 if (h->surplus_huge_pages >= h->nr_overcommit_huge_pages) { 1614 spin_unlock(&hugetlb_lock); 1615 return NULL; 1616 } else { 1617 h->nr_huge_pages++; 1618 h->surplus_huge_pages++; 1619 } 1620 spin_unlock(&hugetlb_lock); 1621 1622 page = __hugetlb_alloc_buddy_huge_page(h, vma, addr, nid); 1623 1624 spin_lock(&hugetlb_lock); 1625 if (page) { 1626 INIT_LIST_HEAD(&page->lru); 1627 r_nid = page_to_nid(page); 1628 set_compound_page_dtor(page, HUGETLB_PAGE_DTOR); 1629 set_hugetlb_cgroup(page, NULL); 1630 /* 1631 * We incremented the global counters already 1632 */ 1633 h->nr_huge_pages_node[r_nid]++; 1634 h->surplus_huge_pages_node[r_nid]++; 1635 __count_vm_event(HTLB_BUDDY_PGALLOC); 1636 } else { 1637 h->nr_huge_pages--; 1638 h->surplus_huge_pages--; 1639 __count_vm_event(HTLB_BUDDY_PGALLOC_FAIL); 1640 } 1641 spin_unlock(&hugetlb_lock); 1642 1643 return page; 1644 } 1645 1646 /* 1647 * Allocate a huge page from 'nid'. Note, 'nid' may be 1648 * NUMA_NO_NODE, which means that it may be allocated 1649 * anywhere. 1650 */ 1651 static 1652 struct page *__alloc_buddy_huge_page_no_mpol(struct hstate *h, int nid) 1653 { 1654 unsigned long addr = -1; 1655 1656 return __alloc_buddy_huge_page(h, NULL, addr, nid); 1657 } 1658 1659 /* 1660 * Use the VMA's mpolicy to allocate a huge page from the buddy. 1661 */ 1662 static 1663 struct page *__alloc_buddy_huge_page_with_mpol(struct hstate *h, 1664 struct vm_area_struct *vma, unsigned long addr) 1665 { 1666 return __alloc_buddy_huge_page(h, vma, addr, NUMA_NO_NODE); 1667 } 1668 1669 /* 1670 * This allocation function is useful in the context where vma is irrelevant. 1671 * E.g. soft-offlining uses this function because it only cares physical 1672 * address of error page. 1673 */ 1674 struct page *alloc_huge_page_node(struct hstate *h, int nid) 1675 { 1676 struct page *page = NULL; 1677 1678 spin_lock(&hugetlb_lock); 1679 if (h->free_huge_pages - h->resv_huge_pages > 0) 1680 page = dequeue_huge_page_node(h, nid); 1681 spin_unlock(&hugetlb_lock); 1682 1683 if (!page) 1684 page = __alloc_buddy_huge_page_no_mpol(h, nid); 1685 1686 return page; 1687 } 1688 1689 /* 1690 * Increase the hugetlb pool such that it can accommodate a reservation 1691 * of size 'delta'. 1692 */ 1693 static int gather_surplus_pages(struct hstate *h, int delta) 1694 { 1695 struct list_head surplus_list; 1696 struct page *page, *tmp; 1697 int ret, i; 1698 int needed, allocated; 1699 bool alloc_ok = true; 1700 1701 needed = (h->resv_huge_pages + delta) - h->free_huge_pages; 1702 if (needed <= 0) { 1703 h->resv_huge_pages += delta; 1704 return 0; 1705 } 1706 1707 allocated = 0; 1708 INIT_LIST_HEAD(&surplus_list); 1709 1710 ret = -ENOMEM; 1711 retry: 1712 spin_unlock(&hugetlb_lock); 1713 for (i = 0; i < needed; i++) { 1714 page = __alloc_buddy_huge_page_no_mpol(h, NUMA_NO_NODE); 1715 if (!page) { 1716 alloc_ok = false; 1717 break; 1718 } 1719 list_add(&page->lru, &surplus_list); 1720 } 1721 allocated += i; 1722 1723 /* 1724 * After retaking hugetlb_lock, we need to recalculate 'needed' 1725 * because either resv_huge_pages or free_huge_pages may have changed. 1726 */ 1727 spin_lock(&hugetlb_lock); 1728 needed = (h->resv_huge_pages + delta) - 1729 (h->free_huge_pages + allocated); 1730 if (needed > 0) { 1731 if (alloc_ok) 1732 goto retry; 1733 /* 1734 * We were not able to allocate enough pages to 1735 * satisfy the entire reservation so we free what 1736 * we've allocated so far. 1737 */ 1738 goto free; 1739 } 1740 /* 1741 * The surplus_list now contains _at_least_ the number of extra pages 1742 * needed to accommodate the reservation. Add the appropriate number 1743 * of pages to the hugetlb pool and free the extras back to the buddy 1744 * allocator. Commit the entire reservation here to prevent another 1745 * process from stealing the pages as they are added to the pool but 1746 * before they are reserved. 1747 */ 1748 needed += allocated; 1749 h->resv_huge_pages += delta; 1750 ret = 0; 1751 1752 /* Free the needed pages to the hugetlb pool */ 1753 list_for_each_entry_safe(page, tmp, &surplus_list, lru) { 1754 if ((--needed) < 0) 1755 break; 1756 /* 1757 * This page is now managed by the hugetlb allocator and has 1758 * no users -- drop the buddy allocator's reference. 1759 */ 1760 put_page_testzero(page); 1761 VM_BUG_ON_PAGE(page_count(page), page); 1762 enqueue_huge_page(h, page); 1763 } 1764 free: 1765 spin_unlock(&hugetlb_lock); 1766 1767 /* Free unnecessary surplus pages to the buddy allocator */ 1768 list_for_each_entry_safe(page, tmp, &surplus_list, lru) 1769 put_page(page); 1770 spin_lock(&hugetlb_lock); 1771 1772 return ret; 1773 } 1774 1775 /* 1776 * When releasing a hugetlb pool reservation, any surplus pages that were 1777 * allocated to satisfy the reservation must be explicitly freed if they were 1778 * never used. 1779 * Called with hugetlb_lock held. 1780 */ 1781 static void return_unused_surplus_pages(struct hstate *h, 1782 unsigned long unused_resv_pages) 1783 { 1784 unsigned long nr_pages; 1785 1786 /* Uncommit the reservation */ 1787 h->resv_huge_pages -= unused_resv_pages; 1788 1789 /* Cannot return gigantic pages currently */ 1790 if (hstate_is_gigantic(h)) 1791 return; 1792 1793 nr_pages = min(unused_resv_pages, h->surplus_huge_pages); 1794 1795 /* 1796 * We want to release as many surplus pages as possible, spread 1797 * evenly across all nodes with memory. Iterate across these nodes 1798 * until we can no longer free unreserved surplus pages. This occurs 1799 * when the nodes with surplus pages have no free pages. 1800 * free_pool_huge_page() will balance the the freed pages across the 1801 * on-line nodes with memory and will handle the hstate accounting. 1802 */ 1803 while (nr_pages--) { 1804 if (!free_pool_huge_page(h, &node_states[N_MEMORY], 1)) 1805 break; 1806 cond_resched_lock(&hugetlb_lock); 1807 } 1808 } 1809 1810 1811 /* 1812 * vma_needs_reservation, vma_commit_reservation and vma_end_reservation 1813 * are used by the huge page allocation routines to manage reservations. 1814 * 1815 * vma_needs_reservation is called to determine if the huge page at addr 1816 * within the vma has an associated reservation. If a reservation is 1817 * needed, the value 1 is returned. The caller is then responsible for 1818 * managing the global reservation and subpool usage counts. After 1819 * the huge page has been allocated, vma_commit_reservation is called 1820 * to add the page to the reservation map. If the page allocation fails, 1821 * the reservation must be ended instead of committed. vma_end_reservation 1822 * is called in such cases. 1823 * 1824 * In the normal case, vma_commit_reservation returns the same value 1825 * as the preceding vma_needs_reservation call. The only time this 1826 * is not the case is if a reserve map was changed between calls. It 1827 * is the responsibility of the caller to notice the difference and 1828 * take appropriate action. 1829 * 1830 * vma_add_reservation is used in error paths where a reservation must 1831 * be restored when a newly allocated huge page must be freed. It is 1832 * to be called after calling vma_needs_reservation to determine if a 1833 * reservation exists. 1834 */ 1835 enum vma_resv_mode { 1836 VMA_NEEDS_RESV, 1837 VMA_COMMIT_RESV, 1838 VMA_END_RESV, 1839 VMA_ADD_RESV, 1840 }; 1841 static long __vma_reservation_common(struct hstate *h, 1842 struct vm_area_struct *vma, unsigned long addr, 1843 enum vma_resv_mode mode) 1844 { 1845 struct resv_map *resv; 1846 pgoff_t idx; 1847 long ret; 1848 1849 resv = vma_resv_map(vma); 1850 if (!resv) 1851 return 1; 1852 1853 idx = vma_hugecache_offset(h, vma, addr); 1854 switch (mode) { 1855 case VMA_NEEDS_RESV: 1856 ret = region_chg(resv, idx, idx + 1); 1857 break; 1858 case VMA_COMMIT_RESV: 1859 ret = region_add(resv, idx, idx + 1); 1860 break; 1861 case VMA_END_RESV: 1862 region_abort(resv, idx, idx + 1); 1863 ret = 0; 1864 break; 1865 case VMA_ADD_RESV: 1866 if (vma->vm_flags & VM_MAYSHARE) 1867 ret = region_add(resv, idx, idx + 1); 1868 else { 1869 region_abort(resv, idx, idx + 1); 1870 ret = region_del(resv, idx, idx + 1); 1871 } 1872 break; 1873 default: 1874 BUG(); 1875 } 1876 1877 if (vma->vm_flags & VM_MAYSHARE) 1878 return ret; 1879 else if (is_vma_resv_set(vma, HPAGE_RESV_OWNER) && ret >= 0) { 1880 /* 1881 * In most cases, reserves always exist for private mappings. 1882 * However, a file associated with mapping could have been 1883 * hole punched or truncated after reserves were consumed. 1884 * As subsequent fault on such a range will not use reserves. 1885 * Subtle - The reserve map for private mappings has the 1886 * opposite meaning than that of shared mappings. If NO 1887 * entry is in the reserve map, it means a reservation exists. 1888 * If an entry exists in the reserve map, it means the 1889 * reservation has already been consumed. As a result, the 1890 * return value of this routine is the opposite of the 1891 * value returned from reserve map manipulation routines above. 1892 */ 1893 if (ret) 1894 return 0; 1895 else 1896 return 1; 1897 } 1898 else 1899 return ret < 0 ? ret : 0; 1900 } 1901 1902 static long vma_needs_reservation(struct hstate *h, 1903 struct vm_area_struct *vma, unsigned long addr) 1904 { 1905 return __vma_reservation_common(h, vma, addr, VMA_NEEDS_RESV); 1906 } 1907 1908 static long vma_commit_reservation(struct hstate *h, 1909 struct vm_area_struct *vma, unsigned long addr) 1910 { 1911 return __vma_reservation_common(h, vma, addr, VMA_COMMIT_RESV); 1912 } 1913 1914 static void vma_end_reservation(struct hstate *h, 1915 struct vm_area_struct *vma, unsigned long addr) 1916 { 1917 (void)__vma_reservation_common(h, vma, addr, VMA_END_RESV); 1918 } 1919 1920 static long vma_add_reservation(struct hstate *h, 1921 struct vm_area_struct *vma, unsigned long addr) 1922 { 1923 return __vma_reservation_common(h, vma, addr, VMA_ADD_RESV); 1924 } 1925 1926 /* 1927 * This routine is called to restore a reservation on error paths. In the 1928 * specific error paths, a huge page was allocated (via alloc_huge_page) 1929 * and is about to be freed. If a reservation for the page existed, 1930 * alloc_huge_page would have consumed the reservation and set PagePrivate 1931 * in the newly allocated page. When the page is freed via free_huge_page, 1932 * the global reservation count will be incremented if PagePrivate is set. 1933 * However, free_huge_page can not adjust the reserve map. Adjust the 1934 * reserve map here to be consistent with global reserve count adjustments 1935 * to be made by free_huge_page. 1936 */ 1937 static void restore_reserve_on_error(struct hstate *h, 1938 struct vm_area_struct *vma, unsigned long address, 1939 struct page *page) 1940 { 1941 if (unlikely(PagePrivate(page))) { 1942 long rc = vma_needs_reservation(h, vma, address); 1943 1944 if (unlikely(rc < 0)) { 1945 /* 1946 * Rare out of memory condition in reserve map 1947 * manipulation. Clear PagePrivate so that 1948 * global reserve count will not be incremented 1949 * by free_huge_page. This will make it appear 1950 * as though the reservation for this page was 1951 * consumed. This may prevent the task from 1952 * faulting in the page at a later time. This 1953 * is better than inconsistent global huge page 1954 * accounting of reserve counts. 1955 */ 1956 ClearPagePrivate(page); 1957 } else if (rc) { 1958 rc = vma_add_reservation(h, vma, address); 1959 if (unlikely(rc < 0)) 1960 /* 1961 * See above comment about rare out of 1962 * memory condition. 1963 */ 1964 ClearPagePrivate(page); 1965 } else 1966 vma_end_reservation(h, vma, address); 1967 } 1968 } 1969 1970 struct page *alloc_huge_page(struct vm_area_struct *vma, 1971 unsigned long addr, int avoid_reserve) 1972 { 1973 struct hugepage_subpool *spool = subpool_vma(vma); 1974 struct hstate *h = hstate_vma(vma); 1975 struct page *page; 1976 long map_chg, map_commit; 1977 long gbl_chg; 1978 int ret, idx; 1979 struct hugetlb_cgroup *h_cg; 1980 1981 idx = hstate_index(h); 1982 /* 1983 * Examine the region/reserve map to determine if the process 1984 * has a reservation for the page to be allocated. A return 1985 * code of zero indicates a reservation exists (no change). 1986 */ 1987 map_chg = gbl_chg = vma_needs_reservation(h, vma, addr); 1988 if (map_chg < 0) 1989 return ERR_PTR(-ENOMEM); 1990 1991 /* 1992 * Processes that did not create the mapping will have no 1993 * reserves as indicated by the region/reserve map. Check 1994 * that the allocation will not exceed the subpool limit. 1995 * Allocations for MAP_NORESERVE mappings also need to be 1996 * checked against any subpool limit. 1997 */ 1998 if (map_chg || avoid_reserve) { 1999 gbl_chg = hugepage_subpool_get_pages(spool, 1); 2000 if (gbl_chg < 0) { 2001 vma_end_reservation(h, vma, addr); 2002 return ERR_PTR(-ENOSPC); 2003 } 2004 2005 /* 2006 * Even though there was no reservation in the region/reserve 2007 * map, there could be reservations associated with the 2008 * subpool that can be used. This would be indicated if the 2009 * return value of hugepage_subpool_get_pages() is zero. 2010 * However, if avoid_reserve is specified we still avoid even 2011 * the subpool reservations. 2012 */ 2013 if (avoid_reserve) 2014 gbl_chg = 1; 2015 } 2016 2017 ret = hugetlb_cgroup_charge_cgroup(idx, pages_per_huge_page(h), &h_cg); 2018 if (ret) 2019 goto out_subpool_put; 2020 2021 spin_lock(&hugetlb_lock); 2022 /* 2023 * glb_chg is passed to indicate whether or not a page must be taken 2024 * from the global free pool (global change). gbl_chg == 0 indicates 2025 * a reservation exists for the allocation. 2026 */ 2027 page = dequeue_huge_page_vma(h, vma, addr, avoid_reserve, gbl_chg); 2028 if (!page) { 2029 spin_unlock(&hugetlb_lock); 2030 page = __alloc_buddy_huge_page_with_mpol(h, vma, addr); 2031 if (!page) 2032 goto out_uncharge_cgroup; 2033 if (!avoid_reserve && vma_has_reserves(vma, gbl_chg)) { 2034 SetPagePrivate(page); 2035 h->resv_huge_pages--; 2036 } 2037 spin_lock(&hugetlb_lock); 2038 list_move(&page->lru, &h->hugepage_activelist); 2039 /* Fall through */ 2040 } 2041 hugetlb_cgroup_commit_charge(idx, pages_per_huge_page(h), h_cg, page); 2042 spin_unlock(&hugetlb_lock); 2043 2044 set_page_private(page, (unsigned long)spool); 2045 2046 map_commit = vma_commit_reservation(h, vma, addr); 2047 if (unlikely(map_chg > map_commit)) { 2048 /* 2049 * The page was added to the reservation map between 2050 * vma_needs_reservation and vma_commit_reservation. 2051 * This indicates a race with hugetlb_reserve_pages. 2052 * Adjust for the subpool count incremented above AND 2053 * in hugetlb_reserve_pages for the same page. Also, 2054 * the reservation count added in hugetlb_reserve_pages 2055 * no longer applies. 2056 */ 2057 long rsv_adjust; 2058 2059 rsv_adjust = hugepage_subpool_put_pages(spool, 1); 2060 hugetlb_acct_memory(h, -rsv_adjust); 2061 } 2062 return page; 2063 2064 out_uncharge_cgroup: 2065 hugetlb_cgroup_uncharge_cgroup(idx, pages_per_huge_page(h), h_cg); 2066 out_subpool_put: 2067 if (map_chg || avoid_reserve) 2068 hugepage_subpool_put_pages(spool, 1); 2069 vma_end_reservation(h, vma, addr); 2070 return ERR_PTR(-ENOSPC); 2071 } 2072 2073 /* 2074 * alloc_huge_page()'s wrapper which simply returns the page if allocation 2075 * succeeds, otherwise NULL. This function is called from new_vma_page(), 2076 * where no ERR_VALUE is expected to be returned. 2077 */ 2078 struct page *alloc_huge_page_noerr(struct vm_area_struct *vma, 2079 unsigned long addr, int avoid_reserve) 2080 { 2081 struct page *page = alloc_huge_page(vma, addr, avoid_reserve); 2082 if (IS_ERR(page)) 2083 page = NULL; 2084 return page; 2085 } 2086 2087 int __weak alloc_bootmem_huge_page(struct hstate *h) 2088 { 2089 struct huge_bootmem_page *m; 2090 int nr_nodes, node; 2091 2092 for_each_node_mask_to_alloc(h, nr_nodes, node, &node_states[N_MEMORY]) { 2093 void *addr; 2094 2095 addr = memblock_virt_alloc_try_nid_nopanic( 2096 huge_page_size(h), huge_page_size(h), 2097 0, BOOTMEM_ALLOC_ACCESSIBLE, node); 2098 if (addr) { 2099 /* 2100 * Use the beginning of the huge page to store the 2101 * huge_bootmem_page struct (until gather_bootmem 2102 * puts them into the mem_map). 2103 */ 2104 m = addr; 2105 goto found; 2106 } 2107 } 2108 return 0; 2109 2110 found: 2111 BUG_ON(!IS_ALIGNED(virt_to_phys(m), huge_page_size(h))); 2112 /* Put them into a private list first because mem_map is not up yet */ 2113 list_add(&m->list, &huge_boot_pages); 2114 m->hstate = h; 2115 return 1; 2116 } 2117 2118 static void __init prep_compound_huge_page(struct page *page, 2119 unsigned int order) 2120 { 2121 if (unlikely(order > (MAX_ORDER - 1))) 2122 prep_compound_gigantic_page(page, order); 2123 else 2124 prep_compound_page(page, order); 2125 } 2126 2127 /* Put bootmem huge pages into the standard lists after mem_map is up */ 2128 static void __init gather_bootmem_prealloc(void) 2129 { 2130 struct huge_bootmem_page *m; 2131 2132 list_for_each_entry(m, &huge_boot_pages, list) { 2133 struct hstate *h = m->hstate; 2134 struct page *page; 2135 2136 #ifdef CONFIG_HIGHMEM 2137 page = pfn_to_page(m->phys >> PAGE_SHIFT); 2138 memblock_free_late(__pa(m), 2139 sizeof(struct huge_bootmem_page)); 2140 #else 2141 page = virt_to_page(m); 2142 #endif 2143 WARN_ON(page_count(page) != 1); 2144 prep_compound_huge_page(page, h->order); 2145 WARN_ON(PageReserved(page)); 2146 prep_new_huge_page(h, page, page_to_nid(page)); 2147 /* 2148 * If we had gigantic hugepages allocated at boot time, we need 2149 * to restore the 'stolen' pages to totalram_pages in order to 2150 * fix confusing memory reports from free(1) and another 2151 * side-effects, like CommitLimit going negative. 2152 */ 2153 if (hstate_is_gigantic(h)) 2154 adjust_managed_page_count(page, 1 << h->order); 2155 } 2156 } 2157 2158 static void __init hugetlb_hstate_alloc_pages(struct hstate *h) 2159 { 2160 unsigned long i; 2161 2162 for (i = 0; i < h->max_huge_pages; ++i) { 2163 if (hstate_is_gigantic(h)) { 2164 if (!alloc_bootmem_huge_page(h)) 2165 break; 2166 } else if (!alloc_fresh_huge_page(h, 2167 &node_states[N_MEMORY])) 2168 break; 2169 } 2170 h->max_huge_pages = i; 2171 } 2172 2173 static void __init hugetlb_init_hstates(void) 2174 { 2175 struct hstate *h; 2176 2177 for_each_hstate(h) { 2178 if (minimum_order > huge_page_order(h)) 2179 minimum_order = huge_page_order(h); 2180 2181 /* oversize hugepages were init'ed in early boot */ 2182 if (!hstate_is_gigantic(h)) 2183 hugetlb_hstate_alloc_pages(h); 2184 } 2185 VM_BUG_ON(minimum_order == UINT_MAX); 2186 } 2187 2188 static char * __init memfmt(char *buf, unsigned long n) 2189 { 2190 if (n >= (1UL << 30)) 2191 sprintf(buf, "%lu GB", n >> 30); 2192 else if (n >= (1UL << 20)) 2193 sprintf(buf, "%lu MB", n >> 20); 2194 else 2195 sprintf(buf, "%lu KB", n >> 10); 2196 return buf; 2197 } 2198 2199 static void __init report_hugepages(void) 2200 { 2201 struct hstate *h; 2202 2203 for_each_hstate(h) { 2204 char buf[32]; 2205 pr_info("HugeTLB registered %s page size, pre-allocated %ld pages\n", 2206 memfmt(buf, huge_page_size(h)), 2207 h->free_huge_pages); 2208 } 2209 } 2210 2211 #ifdef CONFIG_HIGHMEM 2212 static void try_to_free_low(struct hstate *h, unsigned long count, 2213 nodemask_t *nodes_allowed) 2214 { 2215 int i; 2216 2217 if (hstate_is_gigantic(h)) 2218 return; 2219 2220 for_each_node_mask(i, *nodes_allowed) { 2221 struct page *page, *next; 2222 struct list_head *freel = &h->hugepage_freelists[i]; 2223 list_for_each_entry_safe(page, next, freel, lru) { 2224 if (count >= h->nr_huge_pages) 2225 return; 2226 if (PageHighMem(page)) 2227 continue; 2228 list_del(&page->lru); 2229 update_and_free_page(h, page); 2230 h->free_huge_pages--; 2231 h->free_huge_pages_node[page_to_nid(page)]--; 2232 } 2233 } 2234 } 2235 #else 2236 static inline void try_to_free_low(struct hstate *h, unsigned long count, 2237 nodemask_t *nodes_allowed) 2238 { 2239 } 2240 #endif 2241 2242 /* 2243 * Increment or decrement surplus_huge_pages. Keep node-specific counters 2244 * balanced by operating on them in a round-robin fashion. 2245 * Returns 1 if an adjustment was made. 2246 */ 2247 static int adjust_pool_surplus(struct hstate *h, nodemask_t *nodes_allowed, 2248 int delta) 2249 { 2250 int nr_nodes, node; 2251 2252 VM_BUG_ON(delta != -1 && delta != 1); 2253 2254 if (delta < 0) { 2255 for_each_node_mask_to_alloc(h, nr_nodes, node, nodes_allowed) { 2256 if (h->surplus_huge_pages_node[node]) 2257 goto found; 2258 } 2259 } else { 2260 for_each_node_mask_to_free(h, nr_nodes, node, nodes_allowed) { 2261 if (h->surplus_huge_pages_node[node] < 2262 h->nr_huge_pages_node[node]) 2263 goto found; 2264 } 2265 } 2266 return 0; 2267 2268 found: 2269 h->surplus_huge_pages += delta; 2270 h->surplus_huge_pages_node[node] += delta; 2271 return 1; 2272 } 2273 2274 #define persistent_huge_pages(h) (h->nr_huge_pages - h->surplus_huge_pages) 2275 static unsigned long set_max_huge_pages(struct hstate *h, unsigned long count, 2276 nodemask_t *nodes_allowed) 2277 { 2278 unsigned long min_count, ret; 2279 2280 if (hstate_is_gigantic(h) && !gigantic_page_supported()) 2281 return h->max_huge_pages; 2282 2283 /* 2284 * Increase the pool size 2285 * First take pages out of surplus state. Then make up the 2286 * remaining difference by allocating fresh huge pages. 2287 * 2288 * We might race with __alloc_buddy_huge_page() here and be unable 2289 * to convert a surplus huge page to a normal huge page. That is 2290 * not critical, though, it just means the overall size of the 2291 * pool might be one hugepage larger than it needs to be, but 2292 * within all the constraints specified by the sysctls. 2293 */ 2294 spin_lock(&hugetlb_lock); 2295 while (h->surplus_huge_pages && count > persistent_huge_pages(h)) { 2296 if (!adjust_pool_surplus(h, nodes_allowed, -1)) 2297 break; 2298 } 2299 2300 while (count > persistent_huge_pages(h)) { 2301 /* 2302 * If this allocation races such that we no longer need the 2303 * page, free_huge_page will handle it by freeing the page 2304 * and reducing the surplus. 2305 */ 2306 spin_unlock(&hugetlb_lock); 2307 2308 /* yield cpu to avoid soft lockup */ 2309 cond_resched(); 2310 2311 if (hstate_is_gigantic(h)) 2312 ret = alloc_fresh_gigantic_page(h, nodes_allowed); 2313 else 2314 ret = alloc_fresh_huge_page(h, nodes_allowed); 2315 spin_lock(&hugetlb_lock); 2316 if (!ret) 2317 goto out; 2318 2319 /* Bail for signals. Probably ctrl-c from user */ 2320 if (signal_pending(current)) 2321 goto out; 2322 } 2323 2324 /* 2325 * Decrease the pool size 2326 * First return free pages to the buddy allocator (being careful 2327 * to keep enough around to satisfy reservations). Then place 2328 * pages into surplus state as needed so the pool will shrink 2329 * to the desired size as pages become free. 2330 * 2331 * By placing pages into the surplus state independent of the 2332 * overcommit value, we are allowing the surplus pool size to 2333 * exceed overcommit. There are few sane options here. Since 2334 * __alloc_buddy_huge_page() is checking the global counter, 2335 * though, we'll note that we're not allowed to exceed surplus 2336 * and won't grow the pool anywhere else. Not until one of the 2337 * sysctls are changed, or the surplus pages go out of use. 2338 */ 2339 min_count = h->resv_huge_pages + h->nr_huge_pages - h->free_huge_pages; 2340 min_count = max(count, min_count); 2341 try_to_free_low(h, min_count, nodes_allowed); 2342 while (min_count < persistent_huge_pages(h)) { 2343 if (!free_pool_huge_page(h, nodes_allowed, 0)) 2344 break; 2345 cond_resched_lock(&hugetlb_lock); 2346 } 2347 while (count < persistent_huge_pages(h)) { 2348 if (!adjust_pool_surplus(h, nodes_allowed, 1)) 2349 break; 2350 } 2351 out: 2352 ret = persistent_huge_pages(h); 2353 spin_unlock(&hugetlb_lock); 2354 return ret; 2355 } 2356 2357 #define HSTATE_ATTR_RO(_name) \ 2358 static struct kobj_attribute _name##_attr = __ATTR_RO(_name) 2359 2360 #define HSTATE_ATTR(_name) \ 2361 static struct kobj_attribute _name##_attr = \ 2362 __ATTR(_name, 0644, _name##_show, _name##_store) 2363 2364 static struct kobject *hugepages_kobj; 2365 static struct kobject *hstate_kobjs[HUGE_MAX_HSTATE]; 2366 2367 static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp); 2368 2369 static struct hstate *kobj_to_hstate(struct kobject *kobj, int *nidp) 2370 { 2371 int i; 2372 2373 for (i = 0; i < HUGE_MAX_HSTATE; i++) 2374 if (hstate_kobjs[i] == kobj) { 2375 if (nidp) 2376 *nidp = NUMA_NO_NODE; 2377 return &hstates[i]; 2378 } 2379 2380 return kobj_to_node_hstate(kobj, nidp); 2381 } 2382 2383 static ssize_t nr_hugepages_show_common(struct kobject *kobj, 2384 struct kobj_attribute *attr, char *buf) 2385 { 2386 struct hstate *h; 2387 unsigned long nr_huge_pages; 2388 int nid; 2389 2390 h = kobj_to_hstate(kobj, &nid); 2391 if (nid == NUMA_NO_NODE) 2392 nr_huge_pages = h->nr_huge_pages; 2393 else 2394 nr_huge_pages = h->nr_huge_pages_node[nid]; 2395 2396 return sprintf(buf, "%lu\n", nr_huge_pages); 2397 } 2398 2399 static ssize_t __nr_hugepages_store_common(bool obey_mempolicy, 2400 struct hstate *h, int nid, 2401 unsigned long count, size_t len) 2402 { 2403 int err; 2404 NODEMASK_ALLOC(nodemask_t, nodes_allowed, GFP_KERNEL | __GFP_NORETRY); 2405 2406 if (hstate_is_gigantic(h) && !gigantic_page_supported()) { 2407 err = -EINVAL; 2408 goto out; 2409 } 2410 2411 if (nid == NUMA_NO_NODE) { 2412 /* 2413 * global hstate attribute 2414 */ 2415 if (!(obey_mempolicy && 2416 init_nodemask_of_mempolicy(nodes_allowed))) { 2417 NODEMASK_FREE(nodes_allowed); 2418 nodes_allowed = &node_states[N_MEMORY]; 2419 } 2420 } else if (nodes_allowed) { 2421 /* 2422 * per node hstate attribute: adjust count to global, 2423 * but restrict alloc/free to the specified node. 2424 */ 2425 count += h->nr_huge_pages - h->nr_huge_pages_node[nid]; 2426 init_nodemask_of_node(nodes_allowed, nid); 2427 } else 2428 nodes_allowed = &node_states[N_MEMORY]; 2429 2430 h->max_huge_pages = set_max_huge_pages(h, count, nodes_allowed); 2431 2432 if (nodes_allowed != &node_states[N_MEMORY]) 2433 NODEMASK_FREE(nodes_allowed); 2434 2435 return len; 2436 out: 2437 NODEMASK_FREE(nodes_allowed); 2438 return err; 2439 } 2440 2441 static ssize_t nr_hugepages_store_common(bool obey_mempolicy, 2442 struct kobject *kobj, const char *buf, 2443 size_t len) 2444 { 2445 struct hstate *h; 2446 unsigned long count; 2447 int nid; 2448 int err; 2449 2450 err = kstrtoul(buf, 10, &count); 2451 if (err) 2452 return err; 2453 2454 h = kobj_to_hstate(kobj, &nid); 2455 return __nr_hugepages_store_common(obey_mempolicy, h, nid, count, len); 2456 } 2457 2458 static ssize_t nr_hugepages_show(struct kobject *kobj, 2459 struct kobj_attribute *attr, char *buf) 2460 { 2461 return nr_hugepages_show_common(kobj, attr, buf); 2462 } 2463 2464 static ssize_t nr_hugepages_store(struct kobject *kobj, 2465 struct kobj_attribute *attr, const char *buf, size_t len) 2466 { 2467 return nr_hugepages_store_common(false, kobj, buf, len); 2468 } 2469 HSTATE_ATTR(nr_hugepages); 2470 2471 #ifdef CONFIG_NUMA 2472 2473 /* 2474 * hstate attribute for optionally mempolicy-based constraint on persistent 2475 * huge page alloc/free. 2476 */ 2477 static ssize_t nr_hugepages_mempolicy_show(struct kobject *kobj, 2478 struct kobj_attribute *attr, char *buf) 2479 { 2480 return nr_hugepages_show_common(kobj, attr, buf); 2481 } 2482 2483 static ssize_t nr_hugepages_mempolicy_store(struct kobject *kobj, 2484 struct kobj_attribute *attr, const char *buf, size_t len) 2485 { 2486 return nr_hugepages_store_common(true, kobj, buf, len); 2487 } 2488 HSTATE_ATTR(nr_hugepages_mempolicy); 2489 #endif 2490 2491 2492 static ssize_t nr_overcommit_hugepages_show(struct kobject *kobj, 2493 struct kobj_attribute *attr, char *buf) 2494 { 2495 struct hstate *h = kobj_to_hstate(kobj, NULL); 2496 return sprintf(buf, "%lu\n", h->nr_overcommit_huge_pages); 2497 } 2498 2499 static ssize_t nr_overcommit_hugepages_store(struct kobject *kobj, 2500 struct kobj_attribute *attr, const char *buf, size_t count) 2501 { 2502 int err; 2503 unsigned long input; 2504 struct hstate *h = kobj_to_hstate(kobj, NULL); 2505 2506 if (hstate_is_gigantic(h)) 2507 return -EINVAL; 2508 2509 err = kstrtoul(buf, 10, &input); 2510 if (err) 2511 return err; 2512 2513 spin_lock(&hugetlb_lock); 2514 h->nr_overcommit_huge_pages = input; 2515 spin_unlock(&hugetlb_lock); 2516 2517 return count; 2518 } 2519 HSTATE_ATTR(nr_overcommit_hugepages); 2520 2521 static ssize_t free_hugepages_show(struct kobject *kobj, 2522 struct kobj_attribute *attr, char *buf) 2523 { 2524 struct hstate *h; 2525 unsigned long free_huge_pages; 2526 int nid; 2527 2528 h = kobj_to_hstate(kobj, &nid); 2529 if (nid == NUMA_NO_NODE) 2530 free_huge_pages = h->free_huge_pages; 2531 else 2532 free_huge_pages = h->free_huge_pages_node[nid]; 2533 2534 return sprintf(buf, "%lu\n", free_huge_pages); 2535 } 2536 HSTATE_ATTR_RO(free_hugepages); 2537 2538 static ssize_t resv_hugepages_show(struct kobject *kobj, 2539 struct kobj_attribute *attr, char *buf) 2540 { 2541 struct hstate *h = kobj_to_hstate(kobj, NULL); 2542 return sprintf(buf, "%lu\n", h->resv_huge_pages); 2543 } 2544 HSTATE_ATTR_RO(resv_hugepages); 2545 2546 static ssize_t surplus_hugepages_show(struct kobject *kobj, 2547 struct kobj_attribute *attr, char *buf) 2548 { 2549 struct hstate *h; 2550 unsigned long surplus_huge_pages; 2551 int nid; 2552 2553 h = kobj_to_hstate(kobj, &nid); 2554 if (nid == NUMA_NO_NODE) 2555 surplus_huge_pages = h->surplus_huge_pages; 2556 else 2557 surplus_huge_pages = h->surplus_huge_pages_node[nid]; 2558 2559 return sprintf(buf, "%lu\n", surplus_huge_pages); 2560 } 2561 HSTATE_ATTR_RO(surplus_hugepages); 2562 2563 static struct attribute *hstate_attrs[] = { 2564 &nr_hugepages_attr.attr, 2565 &nr_overcommit_hugepages_attr.attr, 2566 &free_hugepages_attr.attr, 2567 &resv_hugepages_attr.attr, 2568 &surplus_hugepages_attr.attr, 2569 #ifdef CONFIG_NUMA 2570 &nr_hugepages_mempolicy_attr.attr, 2571 #endif 2572 NULL, 2573 }; 2574 2575 static struct attribute_group hstate_attr_group = { 2576 .attrs = hstate_attrs, 2577 }; 2578 2579 static int hugetlb_sysfs_add_hstate(struct hstate *h, struct kobject *parent, 2580 struct kobject **hstate_kobjs, 2581 struct attribute_group *hstate_attr_group) 2582 { 2583 int retval; 2584 int hi = hstate_index(h); 2585 2586 hstate_kobjs[hi] = kobject_create_and_add(h->name, parent); 2587 if (!hstate_kobjs[hi]) 2588 return -ENOMEM; 2589 2590 retval = sysfs_create_group(hstate_kobjs[hi], hstate_attr_group); 2591 if (retval) 2592 kobject_put(hstate_kobjs[hi]); 2593 2594 return retval; 2595 } 2596 2597 static void __init hugetlb_sysfs_init(void) 2598 { 2599 struct hstate *h; 2600 int err; 2601 2602 hugepages_kobj = kobject_create_and_add("hugepages", mm_kobj); 2603 if (!hugepages_kobj) 2604 return; 2605 2606 for_each_hstate(h) { 2607 err = hugetlb_sysfs_add_hstate(h, hugepages_kobj, 2608 hstate_kobjs, &hstate_attr_group); 2609 if (err) 2610 pr_err("Hugetlb: Unable to add hstate %s", h->name); 2611 } 2612 } 2613 2614 #ifdef CONFIG_NUMA 2615 2616 /* 2617 * node_hstate/s - associate per node hstate attributes, via their kobjects, 2618 * with node devices in node_devices[] using a parallel array. The array 2619 * index of a node device or _hstate == node id. 2620 * This is here to avoid any static dependency of the node device driver, in 2621 * the base kernel, on the hugetlb module. 2622 */ 2623 struct node_hstate { 2624 struct kobject *hugepages_kobj; 2625 struct kobject *hstate_kobjs[HUGE_MAX_HSTATE]; 2626 }; 2627 static struct node_hstate node_hstates[MAX_NUMNODES]; 2628 2629 /* 2630 * A subset of global hstate attributes for node devices 2631 */ 2632 static struct attribute *per_node_hstate_attrs[] = { 2633 &nr_hugepages_attr.attr, 2634 &free_hugepages_attr.attr, 2635 &surplus_hugepages_attr.attr, 2636 NULL, 2637 }; 2638 2639 static struct attribute_group per_node_hstate_attr_group = { 2640 .attrs = per_node_hstate_attrs, 2641 }; 2642 2643 /* 2644 * kobj_to_node_hstate - lookup global hstate for node device hstate attr kobj. 2645 * Returns node id via non-NULL nidp. 2646 */ 2647 static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp) 2648 { 2649 int nid; 2650 2651 for (nid = 0; nid < nr_node_ids; nid++) { 2652 struct node_hstate *nhs = &node_hstates[nid]; 2653 int i; 2654 for (i = 0; i < HUGE_MAX_HSTATE; i++) 2655 if (nhs->hstate_kobjs[i] == kobj) { 2656 if (nidp) 2657 *nidp = nid; 2658 return &hstates[i]; 2659 } 2660 } 2661 2662 BUG(); 2663 return NULL; 2664 } 2665 2666 /* 2667 * Unregister hstate attributes from a single node device. 2668 * No-op if no hstate attributes attached. 2669 */ 2670 static void hugetlb_unregister_node(struct node *node) 2671 { 2672 struct hstate *h; 2673 struct node_hstate *nhs = &node_hstates[node->dev.id]; 2674 2675 if (!nhs->hugepages_kobj) 2676 return; /* no hstate attributes */ 2677 2678 for_each_hstate(h) { 2679 int idx = hstate_index(h); 2680 if (nhs->hstate_kobjs[idx]) { 2681 kobject_put(nhs->hstate_kobjs[idx]); 2682 nhs->hstate_kobjs[idx] = NULL; 2683 } 2684 } 2685 2686 kobject_put(nhs->hugepages_kobj); 2687 nhs->hugepages_kobj = NULL; 2688 } 2689 2690 2691 /* 2692 * Register hstate attributes for a single node device. 2693 * No-op if attributes already registered. 2694 */ 2695 static void hugetlb_register_node(struct node *node) 2696 { 2697 struct hstate *h; 2698 struct node_hstate *nhs = &node_hstates[node->dev.id]; 2699 int err; 2700 2701 if (nhs->hugepages_kobj) 2702 return; /* already allocated */ 2703 2704 nhs->hugepages_kobj = kobject_create_and_add("hugepages", 2705 &node->dev.kobj); 2706 if (!nhs->hugepages_kobj) 2707 return; 2708 2709 for_each_hstate(h) { 2710 err = hugetlb_sysfs_add_hstate(h, nhs->hugepages_kobj, 2711 nhs->hstate_kobjs, 2712 &per_node_hstate_attr_group); 2713 if (err) { 2714 pr_err("Hugetlb: Unable to add hstate %s for node %d\n", 2715 h->name, node->dev.id); 2716 hugetlb_unregister_node(node); 2717 break; 2718 } 2719 } 2720 } 2721 2722 /* 2723 * hugetlb init time: register hstate attributes for all registered node 2724 * devices of nodes that have memory. All on-line nodes should have 2725 * registered their associated device by this time. 2726 */ 2727 static void __init hugetlb_register_all_nodes(void) 2728 { 2729 int nid; 2730 2731 for_each_node_state(nid, N_MEMORY) { 2732 struct node *node = node_devices[nid]; 2733 if (node->dev.id == nid) 2734 hugetlb_register_node(node); 2735 } 2736 2737 /* 2738 * Let the node device driver know we're here so it can 2739 * [un]register hstate attributes on node hotplug. 2740 */ 2741 register_hugetlbfs_with_node(hugetlb_register_node, 2742 hugetlb_unregister_node); 2743 } 2744 #else /* !CONFIG_NUMA */ 2745 2746 static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp) 2747 { 2748 BUG(); 2749 if (nidp) 2750 *nidp = -1; 2751 return NULL; 2752 } 2753 2754 static void hugetlb_register_all_nodes(void) { } 2755 2756 #endif 2757 2758 static int __init hugetlb_init(void) 2759 { 2760 int i; 2761 2762 if (!hugepages_supported()) 2763 return 0; 2764 2765 if (!size_to_hstate(default_hstate_size)) { 2766 default_hstate_size = HPAGE_SIZE; 2767 if (!size_to_hstate(default_hstate_size)) 2768 hugetlb_add_hstate(HUGETLB_PAGE_ORDER); 2769 } 2770 default_hstate_idx = hstate_index(size_to_hstate(default_hstate_size)); 2771 if (default_hstate_max_huge_pages) { 2772 if (!default_hstate.max_huge_pages) 2773 default_hstate.max_huge_pages = default_hstate_max_huge_pages; 2774 } 2775 2776 hugetlb_init_hstates(); 2777 gather_bootmem_prealloc(); 2778 report_hugepages(); 2779 2780 hugetlb_sysfs_init(); 2781 hugetlb_register_all_nodes(); 2782 hugetlb_cgroup_file_init(); 2783 2784 #ifdef CONFIG_SMP 2785 num_fault_mutexes = roundup_pow_of_two(8 * num_possible_cpus()); 2786 #else 2787 num_fault_mutexes = 1; 2788 #endif 2789 hugetlb_fault_mutex_table = 2790 kmalloc(sizeof(struct mutex) * num_fault_mutexes, GFP_KERNEL); 2791 BUG_ON(!hugetlb_fault_mutex_table); 2792 2793 for (i = 0; i < num_fault_mutexes; i++) 2794 mutex_init(&hugetlb_fault_mutex_table[i]); 2795 return 0; 2796 } 2797 subsys_initcall(hugetlb_init); 2798 2799 /* Should be called on processing a hugepagesz=... option */ 2800 void __init hugetlb_bad_size(void) 2801 { 2802 parsed_valid_hugepagesz = false; 2803 } 2804 2805 void __init hugetlb_add_hstate(unsigned int order) 2806 { 2807 struct hstate *h; 2808 unsigned long i; 2809 2810 if (size_to_hstate(PAGE_SIZE << order)) { 2811 pr_warn("hugepagesz= specified twice, ignoring\n"); 2812 return; 2813 } 2814 BUG_ON(hugetlb_max_hstate >= HUGE_MAX_HSTATE); 2815 BUG_ON(order == 0); 2816 h = &hstates[hugetlb_max_hstate++]; 2817 h->order = order; 2818 h->mask = ~((1ULL << (order + PAGE_SHIFT)) - 1); 2819 h->nr_huge_pages = 0; 2820 h->free_huge_pages = 0; 2821 for (i = 0; i < MAX_NUMNODES; ++i) 2822 INIT_LIST_HEAD(&h->hugepage_freelists[i]); 2823 INIT_LIST_HEAD(&h->hugepage_activelist); 2824 h->next_nid_to_alloc = first_memory_node; 2825 h->next_nid_to_free = first_memory_node; 2826 snprintf(h->name, HSTATE_NAME_LEN, "hugepages-%lukB", 2827 huge_page_size(h)/1024); 2828 2829 parsed_hstate = h; 2830 } 2831 2832 static int __init hugetlb_nrpages_setup(char *s) 2833 { 2834 unsigned long *mhp; 2835 static unsigned long *last_mhp; 2836 2837 if (!parsed_valid_hugepagesz) { 2838 pr_warn("hugepages = %s preceded by " 2839 "an unsupported hugepagesz, ignoring\n", s); 2840 parsed_valid_hugepagesz = true; 2841 return 1; 2842 } 2843 /* 2844 * !hugetlb_max_hstate means we haven't parsed a hugepagesz= parameter yet, 2845 * so this hugepages= parameter goes to the "default hstate". 2846 */ 2847 else if (!hugetlb_max_hstate) 2848 mhp = &default_hstate_max_huge_pages; 2849 else 2850 mhp = &parsed_hstate->max_huge_pages; 2851 2852 if (mhp == last_mhp) { 2853 pr_warn("hugepages= specified twice without interleaving hugepagesz=, ignoring\n"); 2854 return 1; 2855 } 2856 2857 if (sscanf(s, "%lu", mhp) <= 0) 2858 *mhp = 0; 2859 2860 /* 2861 * Global state is always initialized later in hugetlb_init. 2862 * But we need to allocate >= MAX_ORDER hstates here early to still 2863 * use the bootmem allocator. 2864 */ 2865 if (hugetlb_max_hstate && parsed_hstate->order >= MAX_ORDER) 2866 hugetlb_hstate_alloc_pages(parsed_hstate); 2867 2868 last_mhp = mhp; 2869 2870 return 1; 2871 } 2872 __setup("hugepages=", hugetlb_nrpages_setup); 2873 2874 static int __init hugetlb_default_setup(char *s) 2875 { 2876 default_hstate_size = memparse(s, &s); 2877 return 1; 2878 } 2879 __setup("default_hugepagesz=", hugetlb_default_setup); 2880 2881 static unsigned int cpuset_mems_nr(unsigned int *array) 2882 { 2883 int node; 2884 unsigned int nr = 0; 2885 2886 for_each_node_mask(node, cpuset_current_mems_allowed) 2887 nr += array[node]; 2888 2889 return nr; 2890 } 2891 2892 #ifdef CONFIG_SYSCTL 2893 static int hugetlb_sysctl_handler_common(bool obey_mempolicy, 2894 struct ctl_table *table, int write, 2895 void __user *buffer, size_t *length, loff_t *ppos) 2896 { 2897 struct hstate *h = &default_hstate; 2898 unsigned long tmp = h->max_huge_pages; 2899 int ret; 2900 2901 if (!hugepages_supported()) 2902 return -EOPNOTSUPP; 2903 2904 table->data = &tmp; 2905 table->maxlen = sizeof(unsigned long); 2906 ret = proc_doulongvec_minmax(table, write, buffer, length, ppos); 2907 if (ret) 2908 goto out; 2909 2910 if (write) 2911 ret = __nr_hugepages_store_common(obey_mempolicy, h, 2912 NUMA_NO_NODE, tmp, *length); 2913 out: 2914 return ret; 2915 } 2916 2917 int hugetlb_sysctl_handler(struct ctl_table *table, int write, 2918 void __user *buffer, size_t *length, loff_t *ppos) 2919 { 2920 2921 return hugetlb_sysctl_handler_common(false, table, write, 2922 buffer, length, ppos); 2923 } 2924 2925 #ifdef CONFIG_NUMA 2926 int hugetlb_mempolicy_sysctl_handler(struct ctl_table *table, int write, 2927 void __user *buffer, size_t *length, loff_t *ppos) 2928 { 2929 return hugetlb_sysctl_handler_common(true, table, write, 2930 buffer, length, ppos); 2931 } 2932 #endif /* CONFIG_NUMA */ 2933 2934 int hugetlb_overcommit_handler(struct ctl_table *table, int write, 2935 void __user *buffer, 2936 size_t *length, loff_t *ppos) 2937 { 2938 struct hstate *h = &default_hstate; 2939 unsigned long tmp; 2940 int ret; 2941 2942 if (!hugepages_supported()) 2943 return -EOPNOTSUPP; 2944 2945 tmp = h->nr_overcommit_huge_pages; 2946 2947 if (write && hstate_is_gigantic(h)) 2948 return -EINVAL; 2949 2950 table->data = &tmp; 2951 table->maxlen = sizeof(unsigned long); 2952 ret = proc_doulongvec_minmax(table, write, buffer, length, ppos); 2953 if (ret) 2954 goto out; 2955 2956 if (write) { 2957 spin_lock(&hugetlb_lock); 2958 h->nr_overcommit_huge_pages = tmp; 2959 spin_unlock(&hugetlb_lock); 2960 } 2961 out: 2962 return ret; 2963 } 2964 2965 #endif /* CONFIG_SYSCTL */ 2966 2967 void hugetlb_report_meminfo(struct seq_file *m) 2968 { 2969 struct hstate *h = &default_hstate; 2970 if (!hugepages_supported()) 2971 return; 2972 seq_printf(m, 2973 "HugePages_Total: %5lu\n" 2974 "HugePages_Free: %5lu\n" 2975 "HugePages_Rsvd: %5lu\n" 2976 "HugePages_Surp: %5lu\n" 2977 "Hugepagesize: %8lu kB\n", 2978 h->nr_huge_pages, 2979 h->free_huge_pages, 2980 h->resv_huge_pages, 2981 h->surplus_huge_pages, 2982 1UL << (huge_page_order(h) + PAGE_SHIFT - 10)); 2983 } 2984 2985 int hugetlb_report_node_meminfo(int nid, char *buf) 2986 { 2987 struct hstate *h = &default_hstate; 2988 if (!hugepages_supported()) 2989 return 0; 2990 return sprintf(buf, 2991 "Node %d HugePages_Total: %5u\n" 2992 "Node %d HugePages_Free: %5u\n" 2993 "Node %d HugePages_Surp: %5u\n", 2994 nid, h->nr_huge_pages_node[nid], 2995 nid, h->free_huge_pages_node[nid], 2996 nid, h->surplus_huge_pages_node[nid]); 2997 } 2998 2999 void hugetlb_show_meminfo(void) 3000 { 3001 struct hstate *h; 3002 int nid; 3003 3004 if (!hugepages_supported()) 3005 return; 3006 3007 for_each_node_state(nid, N_MEMORY) 3008 for_each_hstate(h) 3009 pr_info("Node %d hugepages_total=%u hugepages_free=%u hugepages_surp=%u hugepages_size=%lukB\n", 3010 nid, 3011 h->nr_huge_pages_node[nid], 3012 h->free_huge_pages_node[nid], 3013 h->surplus_huge_pages_node[nid], 3014 1UL << (huge_page_order(h) + PAGE_SHIFT - 10)); 3015 } 3016 3017 void hugetlb_report_usage(struct seq_file *m, struct mm_struct *mm) 3018 { 3019 seq_printf(m, "HugetlbPages:\t%8lu kB\n", 3020 atomic_long_read(&mm->hugetlb_usage) << (PAGE_SHIFT - 10)); 3021 } 3022 3023 /* Return the number pages of memory we physically have, in PAGE_SIZE units. */ 3024 unsigned long hugetlb_total_pages(void) 3025 { 3026 struct hstate *h; 3027 unsigned long nr_total_pages = 0; 3028 3029 for_each_hstate(h) 3030 nr_total_pages += h->nr_huge_pages * pages_per_huge_page(h); 3031 return nr_total_pages; 3032 } 3033 3034 static int hugetlb_acct_memory(struct hstate *h, long delta) 3035 { 3036 int ret = -ENOMEM; 3037 3038 spin_lock(&hugetlb_lock); 3039 /* 3040 * When cpuset is configured, it breaks the strict hugetlb page 3041 * reservation as the accounting is done on a global variable. Such 3042 * reservation is completely rubbish in the presence of cpuset because 3043 * the reservation is not checked against page availability for the 3044 * current cpuset. Application can still potentially OOM'ed by kernel 3045 * with lack of free htlb page in cpuset that the task is in. 3046 * Attempt to enforce strict accounting with cpuset is almost 3047 * impossible (or too ugly) because cpuset is too fluid that 3048 * task or memory node can be dynamically moved between cpusets. 3049 * 3050 * The change of semantics for shared hugetlb mapping with cpuset is 3051 * undesirable. However, in order to preserve some of the semantics, 3052 * we fall back to check against current free page availability as 3053 * a best attempt and hopefully to minimize the impact of changing 3054 * semantics that cpuset has. 3055 */ 3056 if (delta > 0) { 3057 if (gather_surplus_pages(h, delta) < 0) 3058 goto out; 3059 3060 if (delta > cpuset_mems_nr(h->free_huge_pages_node)) { 3061 return_unused_surplus_pages(h, delta); 3062 goto out; 3063 } 3064 } 3065 3066 ret = 0; 3067 if (delta < 0) 3068 return_unused_surplus_pages(h, (unsigned long) -delta); 3069 3070 out: 3071 spin_unlock(&hugetlb_lock); 3072 return ret; 3073 } 3074 3075 static void hugetlb_vm_op_open(struct vm_area_struct *vma) 3076 { 3077 struct resv_map *resv = vma_resv_map(vma); 3078 3079 /* 3080 * This new VMA should share its siblings reservation map if present. 3081 * The VMA will only ever have a valid reservation map pointer where 3082 * it is being copied for another still existing VMA. As that VMA 3083 * has a reference to the reservation map it cannot disappear until 3084 * after this open call completes. It is therefore safe to take a 3085 * new reference here without additional locking. 3086 */ 3087 if (resv && is_vma_resv_set(vma, HPAGE_RESV_OWNER)) 3088 kref_get(&resv->refs); 3089 } 3090 3091 static void hugetlb_vm_op_close(struct vm_area_struct *vma) 3092 { 3093 struct hstate *h = hstate_vma(vma); 3094 struct resv_map *resv = vma_resv_map(vma); 3095 struct hugepage_subpool *spool = subpool_vma(vma); 3096 unsigned long reserve, start, end; 3097 long gbl_reserve; 3098 3099 if (!resv || !is_vma_resv_set(vma, HPAGE_RESV_OWNER)) 3100 return; 3101 3102 start = vma_hugecache_offset(h, vma, vma->vm_start); 3103 end = vma_hugecache_offset(h, vma, vma->vm_end); 3104 3105 reserve = (end - start) - region_count(resv, start, end); 3106 3107 kref_put(&resv->refs, resv_map_release); 3108 3109 if (reserve) { 3110 /* 3111 * Decrement reserve counts. The global reserve count may be 3112 * adjusted if the subpool has a minimum size. 3113 */ 3114 gbl_reserve = hugepage_subpool_put_pages(spool, reserve); 3115 hugetlb_acct_memory(h, -gbl_reserve); 3116 } 3117 } 3118 3119 /* 3120 * We cannot handle pagefaults against hugetlb pages at all. They cause 3121 * handle_mm_fault() to try to instantiate regular-sized pages in the 3122 * hugegpage VMA. do_page_fault() is supposed to trap this, so BUG is we get 3123 * this far. 3124 */ 3125 static int hugetlb_vm_op_fault(struct vm_area_struct *vma, struct vm_fault *vmf) 3126 { 3127 BUG(); 3128 return 0; 3129 } 3130 3131 const struct vm_operations_struct hugetlb_vm_ops = { 3132 .fault = hugetlb_vm_op_fault, 3133 .open = hugetlb_vm_op_open, 3134 .close = hugetlb_vm_op_close, 3135 }; 3136 3137 static pte_t make_huge_pte(struct vm_area_struct *vma, struct page *page, 3138 int writable) 3139 { 3140 pte_t entry; 3141 3142 if (writable) { 3143 entry = huge_pte_mkwrite(huge_pte_mkdirty(mk_huge_pte(page, 3144 vma->vm_page_prot))); 3145 } else { 3146 entry = huge_pte_wrprotect(mk_huge_pte(page, 3147 vma->vm_page_prot)); 3148 } 3149 entry = pte_mkyoung(entry); 3150 entry = pte_mkhuge(entry); 3151 entry = arch_make_huge_pte(entry, vma, page, writable); 3152 3153 return entry; 3154 } 3155 3156 static void set_huge_ptep_writable(struct vm_area_struct *vma, 3157 unsigned long address, pte_t *ptep) 3158 { 3159 pte_t entry; 3160 3161 entry = huge_pte_mkwrite(huge_pte_mkdirty(huge_ptep_get(ptep))); 3162 if (huge_ptep_set_access_flags(vma, address, ptep, entry, 1)) 3163 update_mmu_cache(vma, address, ptep); 3164 } 3165 3166 static int is_hugetlb_entry_migration(pte_t pte) 3167 { 3168 swp_entry_t swp; 3169 3170 if (huge_pte_none(pte) || pte_present(pte)) 3171 return 0; 3172 swp = pte_to_swp_entry(pte); 3173 if (non_swap_entry(swp) && is_migration_entry(swp)) 3174 return 1; 3175 else 3176 return 0; 3177 } 3178 3179 static int is_hugetlb_entry_hwpoisoned(pte_t pte) 3180 { 3181 swp_entry_t swp; 3182 3183 if (huge_pte_none(pte) || pte_present(pte)) 3184 return 0; 3185 swp = pte_to_swp_entry(pte); 3186 if (non_swap_entry(swp) && is_hwpoison_entry(swp)) 3187 return 1; 3188 else 3189 return 0; 3190 } 3191 3192 int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src, 3193 struct vm_area_struct *vma) 3194 { 3195 pte_t *src_pte, *dst_pte, entry; 3196 struct page *ptepage; 3197 unsigned long addr; 3198 int cow; 3199 struct hstate *h = hstate_vma(vma); 3200 unsigned long sz = huge_page_size(h); 3201 unsigned long mmun_start; /* For mmu_notifiers */ 3202 unsigned long mmun_end; /* For mmu_notifiers */ 3203 int ret = 0; 3204 3205 cow = (vma->vm_flags & (VM_SHARED | VM_MAYWRITE)) == VM_MAYWRITE; 3206 3207 mmun_start = vma->vm_start; 3208 mmun_end = vma->vm_end; 3209 if (cow) 3210 mmu_notifier_invalidate_range_start(src, mmun_start, mmun_end); 3211 3212 for (addr = vma->vm_start; addr < vma->vm_end; addr += sz) { 3213 spinlock_t *src_ptl, *dst_ptl; 3214 src_pte = huge_pte_offset(src, addr); 3215 if (!src_pte) 3216 continue; 3217 dst_pte = huge_pte_alloc(dst, addr, sz); 3218 if (!dst_pte) { 3219 ret = -ENOMEM; 3220 break; 3221 } 3222 3223 /* If the pagetables are shared don't copy or take references */ 3224 if (dst_pte == src_pte) 3225 continue; 3226 3227 dst_ptl = huge_pte_lock(h, dst, dst_pte); 3228 src_ptl = huge_pte_lockptr(h, src, src_pte); 3229 spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING); 3230 entry = huge_ptep_get(src_pte); 3231 if (huge_pte_none(entry)) { /* skip none entry */ 3232 ; 3233 } else if (unlikely(is_hugetlb_entry_migration(entry) || 3234 is_hugetlb_entry_hwpoisoned(entry))) { 3235 swp_entry_t swp_entry = pte_to_swp_entry(entry); 3236 3237 if (is_write_migration_entry(swp_entry) && cow) { 3238 /* 3239 * COW mappings require pages in both 3240 * parent and child to be set to read. 3241 */ 3242 make_migration_entry_read(&swp_entry); 3243 entry = swp_entry_to_pte(swp_entry); 3244 set_huge_pte_at(src, addr, src_pte, entry); 3245 } 3246 set_huge_pte_at(dst, addr, dst_pte, entry); 3247 } else { 3248 if (cow) { 3249 huge_ptep_set_wrprotect(src, addr, src_pte); 3250 mmu_notifier_invalidate_range(src, mmun_start, 3251 mmun_end); 3252 } 3253 entry = huge_ptep_get(src_pte); 3254 ptepage = pte_page(entry); 3255 get_page(ptepage); 3256 page_dup_rmap(ptepage, true); 3257 set_huge_pte_at(dst, addr, dst_pte, entry); 3258 hugetlb_count_add(pages_per_huge_page(h), dst); 3259 } 3260 spin_unlock(src_ptl); 3261 spin_unlock(dst_ptl); 3262 } 3263 3264 if (cow) 3265 mmu_notifier_invalidate_range_end(src, mmun_start, mmun_end); 3266 3267 return ret; 3268 } 3269 3270 void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct *vma, 3271 unsigned long start, unsigned long end, 3272 struct page *ref_page) 3273 { 3274 struct mm_struct *mm = vma->vm_mm; 3275 unsigned long address; 3276 pte_t *ptep; 3277 pte_t pte; 3278 spinlock_t *ptl; 3279 struct page *page; 3280 struct hstate *h = hstate_vma(vma); 3281 unsigned long sz = huge_page_size(h); 3282 const unsigned long mmun_start = start; /* For mmu_notifiers */ 3283 const unsigned long mmun_end = end; /* For mmu_notifiers */ 3284 3285 WARN_ON(!is_vm_hugetlb_page(vma)); 3286 BUG_ON(start & ~huge_page_mask(h)); 3287 BUG_ON(end & ~huge_page_mask(h)); 3288 3289 /* 3290 * This is a hugetlb vma, all the pte entries should point 3291 * to huge page. 3292 */ 3293 tlb_remove_check_page_size_change(tlb, sz); 3294 tlb_start_vma(tlb, vma); 3295 mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end); 3296 address = start; 3297 for (; address < end; address += sz) { 3298 ptep = huge_pte_offset(mm, address); 3299 if (!ptep) 3300 continue; 3301 3302 ptl = huge_pte_lock(h, mm, ptep); 3303 if (huge_pmd_unshare(mm, &address, ptep)) { 3304 spin_unlock(ptl); 3305 continue; 3306 } 3307 3308 pte = huge_ptep_get(ptep); 3309 if (huge_pte_none(pte)) { 3310 spin_unlock(ptl); 3311 continue; 3312 } 3313 3314 /* 3315 * Migrating hugepage or HWPoisoned hugepage is already 3316 * unmapped and its refcount is dropped, so just clear pte here. 3317 */ 3318 if (unlikely(!pte_present(pte))) { 3319 huge_pte_clear(mm, address, ptep); 3320 spin_unlock(ptl); 3321 continue; 3322 } 3323 3324 page = pte_page(pte); 3325 /* 3326 * If a reference page is supplied, it is because a specific 3327 * page is being unmapped, not a range. Ensure the page we 3328 * are about to unmap is the actual page of interest. 3329 */ 3330 if (ref_page) { 3331 if (page != ref_page) { 3332 spin_unlock(ptl); 3333 continue; 3334 } 3335 /* 3336 * Mark the VMA as having unmapped its page so that 3337 * future faults in this VMA will fail rather than 3338 * looking like data was lost 3339 */ 3340 set_vma_resv_flags(vma, HPAGE_RESV_UNMAPPED); 3341 } 3342 3343 pte = huge_ptep_get_and_clear(mm, address, ptep); 3344 tlb_remove_huge_tlb_entry(h, tlb, ptep, address); 3345 if (huge_pte_dirty(pte)) 3346 set_page_dirty(page); 3347 3348 hugetlb_count_sub(pages_per_huge_page(h), mm); 3349 page_remove_rmap(page, true); 3350 3351 spin_unlock(ptl); 3352 tlb_remove_page_size(tlb, page, huge_page_size(h)); 3353 /* 3354 * Bail out after unmapping reference page if supplied 3355 */ 3356 if (ref_page) 3357 break; 3358 } 3359 mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end); 3360 tlb_end_vma(tlb, vma); 3361 } 3362 3363 void __unmap_hugepage_range_final(struct mmu_gather *tlb, 3364 struct vm_area_struct *vma, unsigned long start, 3365 unsigned long end, struct page *ref_page) 3366 { 3367 __unmap_hugepage_range(tlb, vma, start, end, ref_page); 3368 3369 /* 3370 * Clear this flag so that x86's huge_pmd_share page_table_shareable 3371 * test will fail on a vma being torn down, and not grab a page table 3372 * on its way out. We're lucky that the flag has such an appropriate 3373 * name, and can in fact be safely cleared here. We could clear it 3374 * before the __unmap_hugepage_range above, but all that's necessary 3375 * is to clear it before releasing the i_mmap_rwsem. This works 3376 * because in the context this is called, the VMA is about to be 3377 * destroyed and the i_mmap_rwsem is held. 3378 */ 3379 vma->vm_flags &= ~VM_MAYSHARE; 3380 } 3381 3382 void unmap_hugepage_range(struct vm_area_struct *vma, unsigned long start, 3383 unsigned long end, struct page *ref_page) 3384 { 3385 struct mm_struct *mm; 3386 struct mmu_gather tlb; 3387 3388 mm = vma->vm_mm; 3389 3390 tlb_gather_mmu(&tlb, mm, start, end); 3391 __unmap_hugepage_range(&tlb, vma, start, end, ref_page); 3392 tlb_finish_mmu(&tlb, start, end); 3393 } 3394 3395 /* 3396 * This is called when the original mapper is failing to COW a MAP_PRIVATE 3397 * mappping it owns the reserve page for. The intention is to unmap the page 3398 * from other VMAs and let the children be SIGKILLed if they are faulting the 3399 * same region. 3400 */ 3401 static void unmap_ref_private(struct mm_struct *mm, struct vm_area_struct *vma, 3402 struct page *page, unsigned long address) 3403 { 3404 struct hstate *h = hstate_vma(vma); 3405 struct vm_area_struct *iter_vma; 3406 struct address_space *mapping; 3407 pgoff_t pgoff; 3408 3409 /* 3410 * vm_pgoff is in PAGE_SIZE units, hence the different calculation 3411 * from page cache lookup which is in HPAGE_SIZE units. 3412 */ 3413 address = address & huge_page_mask(h); 3414 pgoff = ((address - vma->vm_start) >> PAGE_SHIFT) + 3415 vma->vm_pgoff; 3416 mapping = vma->vm_file->f_mapping; 3417 3418 /* 3419 * Take the mapping lock for the duration of the table walk. As 3420 * this mapping should be shared between all the VMAs, 3421 * __unmap_hugepage_range() is called as the lock is already held 3422 */ 3423 i_mmap_lock_write(mapping); 3424 vma_interval_tree_foreach(iter_vma, &mapping->i_mmap, pgoff, pgoff) { 3425 /* Do not unmap the current VMA */ 3426 if (iter_vma == vma) 3427 continue; 3428 3429 /* 3430 * Shared VMAs have their own reserves and do not affect 3431 * MAP_PRIVATE accounting but it is possible that a shared 3432 * VMA is using the same page so check and skip such VMAs. 3433 */ 3434 if (iter_vma->vm_flags & VM_MAYSHARE) 3435 continue; 3436 3437 /* 3438 * Unmap the page from other VMAs without their own reserves. 3439 * They get marked to be SIGKILLed if they fault in these 3440 * areas. This is because a future no-page fault on this VMA 3441 * could insert a zeroed page instead of the data existing 3442 * from the time of fork. This would look like data corruption 3443 */ 3444 if (!is_vma_resv_set(iter_vma, HPAGE_RESV_OWNER)) 3445 unmap_hugepage_range(iter_vma, address, 3446 address + huge_page_size(h), page); 3447 } 3448 i_mmap_unlock_write(mapping); 3449 } 3450 3451 /* 3452 * Hugetlb_cow() should be called with page lock of the original hugepage held. 3453 * Called with hugetlb_instantiation_mutex held and pte_page locked so we 3454 * cannot race with other handlers or page migration. 3455 * Keep the pte_same checks anyway to make transition from the mutex easier. 3456 */ 3457 static int hugetlb_cow(struct mm_struct *mm, struct vm_area_struct *vma, 3458 unsigned long address, pte_t *ptep, 3459 struct page *pagecache_page, spinlock_t *ptl) 3460 { 3461 pte_t pte; 3462 struct hstate *h = hstate_vma(vma); 3463 struct page *old_page, *new_page; 3464 int ret = 0, outside_reserve = 0; 3465 unsigned long mmun_start; /* For mmu_notifiers */ 3466 unsigned long mmun_end; /* For mmu_notifiers */ 3467 3468 pte = huge_ptep_get(ptep); 3469 old_page = pte_page(pte); 3470 3471 retry_avoidcopy: 3472 /* If no-one else is actually using this page, avoid the copy 3473 * and just make the page writable */ 3474 if (page_mapcount(old_page) == 1 && PageAnon(old_page)) { 3475 page_move_anon_rmap(old_page, vma); 3476 set_huge_ptep_writable(vma, address, ptep); 3477 return 0; 3478 } 3479 3480 /* 3481 * If the process that created a MAP_PRIVATE mapping is about to 3482 * perform a COW due to a shared page count, attempt to satisfy 3483 * the allocation without using the existing reserves. The pagecache 3484 * page is used to determine if the reserve at this address was 3485 * consumed or not. If reserves were used, a partial faulted mapping 3486 * at the time of fork() could consume its reserves on COW instead 3487 * of the full address range. 3488 */ 3489 if (is_vma_resv_set(vma, HPAGE_RESV_OWNER) && 3490 old_page != pagecache_page) 3491 outside_reserve = 1; 3492 3493 get_page(old_page); 3494 3495 /* 3496 * Drop page table lock as buddy allocator may be called. It will 3497 * be acquired again before returning to the caller, as expected. 3498 */ 3499 spin_unlock(ptl); 3500 new_page = alloc_huge_page(vma, address, outside_reserve); 3501 3502 if (IS_ERR(new_page)) { 3503 /* 3504 * If a process owning a MAP_PRIVATE mapping fails to COW, 3505 * it is due to references held by a child and an insufficient 3506 * huge page pool. To guarantee the original mappers 3507 * reliability, unmap the page from child processes. The child 3508 * may get SIGKILLed if it later faults. 3509 */ 3510 if (outside_reserve) { 3511 put_page(old_page); 3512 BUG_ON(huge_pte_none(pte)); 3513 unmap_ref_private(mm, vma, old_page, address); 3514 BUG_ON(huge_pte_none(pte)); 3515 spin_lock(ptl); 3516 ptep = huge_pte_offset(mm, address & huge_page_mask(h)); 3517 if (likely(ptep && 3518 pte_same(huge_ptep_get(ptep), pte))) 3519 goto retry_avoidcopy; 3520 /* 3521 * race occurs while re-acquiring page table 3522 * lock, and our job is done. 3523 */ 3524 return 0; 3525 } 3526 3527 ret = (PTR_ERR(new_page) == -ENOMEM) ? 3528 VM_FAULT_OOM : VM_FAULT_SIGBUS; 3529 goto out_release_old; 3530 } 3531 3532 /* 3533 * When the original hugepage is shared one, it does not have 3534 * anon_vma prepared. 3535 */ 3536 if (unlikely(anon_vma_prepare(vma))) { 3537 ret = VM_FAULT_OOM; 3538 goto out_release_all; 3539 } 3540 3541 copy_user_huge_page(new_page, old_page, address, vma, 3542 pages_per_huge_page(h)); 3543 __SetPageUptodate(new_page); 3544 set_page_huge_active(new_page); 3545 3546 mmun_start = address & huge_page_mask(h); 3547 mmun_end = mmun_start + huge_page_size(h); 3548 mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end); 3549 3550 /* 3551 * Retake the page table lock to check for racing updates 3552 * before the page tables are altered 3553 */ 3554 spin_lock(ptl); 3555 ptep = huge_pte_offset(mm, address & huge_page_mask(h)); 3556 if (likely(ptep && pte_same(huge_ptep_get(ptep), pte))) { 3557 ClearPagePrivate(new_page); 3558 3559 /* Break COW */ 3560 huge_ptep_clear_flush(vma, address, ptep); 3561 mmu_notifier_invalidate_range(mm, mmun_start, mmun_end); 3562 set_huge_pte_at(mm, address, ptep, 3563 make_huge_pte(vma, new_page, 1)); 3564 page_remove_rmap(old_page, true); 3565 hugepage_add_new_anon_rmap(new_page, vma, address); 3566 /* Make the old page be freed below */ 3567 new_page = old_page; 3568 } 3569 spin_unlock(ptl); 3570 mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end); 3571 out_release_all: 3572 restore_reserve_on_error(h, vma, address, new_page); 3573 put_page(new_page); 3574 out_release_old: 3575 put_page(old_page); 3576 3577 spin_lock(ptl); /* Caller expects lock to be held */ 3578 return ret; 3579 } 3580 3581 /* Return the pagecache page at a given address within a VMA */ 3582 static struct page *hugetlbfs_pagecache_page(struct hstate *h, 3583 struct vm_area_struct *vma, unsigned long address) 3584 { 3585 struct address_space *mapping; 3586 pgoff_t idx; 3587 3588 mapping = vma->vm_file->f_mapping; 3589 idx = vma_hugecache_offset(h, vma, address); 3590 3591 return find_lock_page(mapping, idx); 3592 } 3593 3594 /* 3595 * Return whether there is a pagecache page to back given address within VMA. 3596 * Caller follow_hugetlb_page() holds page_table_lock so we cannot lock_page. 3597 */ 3598 static bool hugetlbfs_pagecache_present(struct hstate *h, 3599 struct vm_area_struct *vma, unsigned long address) 3600 { 3601 struct address_space *mapping; 3602 pgoff_t idx; 3603 struct page *page; 3604 3605 mapping = vma->vm_file->f_mapping; 3606 idx = vma_hugecache_offset(h, vma, address); 3607 3608 page = find_get_page(mapping, idx); 3609 if (page) 3610 put_page(page); 3611 return page != NULL; 3612 } 3613 3614 int huge_add_to_page_cache(struct page *page, struct address_space *mapping, 3615 pgoff_t idx) 3616 { 3617 struct inode *inode = mapping->host; 3618 struct hstate *h = hstate_inode(inode); 3619 int err = add_to_page_cache(page, mapping, idx, GFP_KERNEL); 3620 3621 if (err) 3622 return err; 3623 ClearPagePrivate(page); 3624 3625 spin_lock(&inode->i_lock); 3626 inode->i_blocks += blocks_per_huge_page(h); 3627 spin_unlock(&inode->i_lock); 3628 return 0; 3629 } 3630 3631 static int hugetlb_no_page(struct mm_struct *mm, struct vm_area_struct *vma, 3632 struct address_space *mapping, pgoff_t idx, 3633 unsigned long address, pte_t *ptep, unsigned int flags) 3634 { 3635 struct hstate *h = hstate_vma(vma); 3636 int ret = VM_FAULT_SIGBUS; 3637 int anon_rmap = 0; 3638 unsigned long size; 3639 struct page *page; 3640 pte_t new_pte; 3641 spinlock_t *ptl; 3642 3643 /* 3644 * Currently, we are forced to kill the process in the event the 3645 * original mapper has unmapped pages from the child due to a failed 3646 * COW. Warn that such a situation has occurred as it may not be obvious 3647 */ 3648 if (is_vma_resv_set(vma, HPAGE_RESV_UNMAPPED)) { 3649 pr_warn_ratelimited("PID %d killed due to inadequate hugepage pool\n", 3650 current->pid); 3651 return ret; 3652 } 3653 3654 /* 3655 * Use page lock to guard against racing truncation 3656 * before we get page_table_lock. 3657 */ 3658 retry: 3659 page = find_lock_page(mapping, idx); 3660 if (!page) { 3661 size = i_size_read(mapping->host) >> huge_page_shift(h); 3662 if (idx >= size) 3663 goto out; 3664 page = alloc_huge_page(vma, address, 0); 3665 if (IS_ERR(page)) { 3666 ret = PTR_ERR(page); 3667 if (ret == -ENOMEM) 3668 ret = VM_FAULT_OOM; 3669 else 3670 ret = VM_FAULT_SIGBUS; 3671 goto out; 3672 } 3673 clear_huge_page(page, address, pages_per_huge_page(h)); 3674 __SetPageUptodate(page); 3675 set_page_huge_active(page); 3676 3677 if (vma->vm_flags & VM_MAYSHARE) { 3678 int err = huge_add_to_page_cache(page, mapping, idx); 3679 if (err) { 3680 put_page(page); 3681 if (err == -EEXIST) 3682 goto retry; 3683 goto out; 3684 } 3685 } else { 3686 lock_page(page); 3687 if (unlikely(anon_vma_prepare(vma))) { 3688 ret = VM_FAULT_OOM; 3689 goto backout_unlocked; 3690 } 3691 anon_rmap = 1; 3692 } 3693 } else { 3694 /* 3695 * If memory error occurs between mmap() and fault, some process 3696 * don't have hwpoisoned swap entry for errored virtual address. 3697 * So we need to block hugepage fault by PG_hwpoison bit check. 3698 */ 3699 if (unlikely(PageHWPoison(page))) { 3700 ret = VM_FAULT_HWPOISON | 3701 VM_FAULT_SET_HINDEX(hstate_index(h)); 3702 goto backout_unlocked; 3703 } 3704 } 3705 3706 /* 3707 * If we are going to COW a private mapping later, we examine the 3708 * pending reservations for this page now. This will ensure that 3709 * any allocations necessary to record that reservation occur outside 3710 * the spinlock. 3711 */ 3712 if ((flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED)) { 3713 if (vma_needs_reservation(h, vma, address) < 0) { 3714 ret = VM_FAULT_OOM; 3715 goto backout_unlocked; 3716 } 3717 /* Just decrements count, does not deallocate */ 3718 vma_end_reservation(h, vma, address); 3719 } 3720 3721 ptl = huge_pte_lock(h, mm, ptep); 3722 size = i_size_read(mapping->host) >> huge_page_shift(h); 3723 if (idx >= size) 3724 goto backout; 3725 3726 ret = 0; 3727 if (!huge_pte_none(huge_ptep_get(ptep))) 3728 goto backout; 3729 3730 if (anon_rmap) { 3731 ClearPagePrivate(page); 3732 hugepage_add_new_anon_rmap(page, vma, address); 3733 } else 3734 page_dup_rmap(page, true); 3735 new_pte = make_huge_pte(vma, page, ((vma->vm_flags & VM_WRITE) 3736 && (vma->vm_flags & VM_SHARED))); 3737 set_huge_pte_at(mm, address, ptep, new_pte); 3738 3739 hugetlb_count_add(pages_per_huge_page(h), mm); 3740 if ((flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED)) { 3741 /* Optimization, do the COW without a second fault */ 3742 ret = hugetlb_cow(mm, vma, address, ptep, page, ptl); 3743 } 3744 3745 spin_unlock(ptl); 3746 unlock_page(page); 3747 out: 3748 return ret; 3749 3750 backout: 3751 spin_unlock(ptl); 3752 backout_unlocked: 3753 unlock_page(page); 3754 restore_reserve_on_error(h, vma, address, page); 3755 put_page(page); 3756 goto out; 3757 } 3758 3759 #ifdef CONFIG_SMP 3760 u32 hugetlb_fault_mutex_hash(struct hstate *h, struct mm_struct *mm, 3761 struct vm_area_struct *vma, 3762 struct address_space *mapping, 3763 pgoff_t idx, unsigned long address) 3764 { 3765 unsigned long key[2]; 3766 u32 hash; 3767 3768 if (vma->vm_flags & VM_SHARED) { 3769 key[0] = (unsigned long) mapping; 3770 key[1] = idx; 3771 } else { 3772 key[0] = (unsigned long) mm; 3773 key[1] = address >> huge_page_shift(h); 3774 } 3775 3776 hash = jhash2((u32 *)&key, sizeof(key)/sizeof(u32), 0); 3777 3778 return hash & (num_fault_mutexes - 1); 3779 } 3780 #else 3781 /* 3782 * For uniprocesor systems we always use a single mutex, so just 3783 * return 0 and avoid the hashing overhead. 3784 */ 3785 u32 hugetlb_fault_mutex_hash(struct hstate *h, struct mm_struct *mm, 3786 struct vm_area_struct *vma, 3787 struct address_space *mapping, 3788 pgoff_t idx, unsigned long address) 3789 { 3790 return 0; 3791 } 3792 #endif 3793 3794 int hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma, 3795 unsigned long address, unsigned int flags) 3796 { 3797 pte_t *ptep, entry; 3798 spinlock_t *ptl; 3799 int ret; 3800 u32 hash; 3801 pgoff_t idx; 3802 struct page *page = NULL; 3803 struct page *pagecache_page = NULL; 3804 struct hstate *h = hstate_vma(vma); 3805 struct address_space *mapping; 3806 int need_wait_lock = 0; 3807 3808 address &= huge_page_mask(h); 3809 3810 ptep = huge_pte_offset(mm, address); 3811 if (ptep) { 3812 entry = huge_ptep_get(ptep); 3813 if (unlikely(is_hugetlb_entry_migration(entry))) { 3814 migration_entry_wait_huge(vma, mm, ptep); 3815 return 0; 3816 } else if (unlikely(is_hugetlb_entry_hwpoisoned(entry))) 3817 return VM_FAULT_HWPOISON_LARGE | 3818 VM_FAULT_SET_HINDEX(hstate_index(h)); 3819 } else { 3820 ptep = huge_pte_alloc(mm, address, huge_page_size(h)); 3821 if (!ptep) 3822 return VM_FAULT_OOM; 3823 } 3824 3825 mapping = vma->vm_file->f_mapping; 3826 idx = vma_hugecache_offset(h, vma, address); 3827 3828 /* 3829 * Serialize hugepage allocation and instantiation, so that we don't 3830 * get spurious allocation failures if two CPUs race to instantiate 3831 * the same page in the page cache. 3832 */ 3833 hash = hugetlb_fault_mutex_hash(h, mm, vma, mapping, idx, address); 3834 mutex_lock(&hugetlb_fault_mutex_table[hash]); 3835 3836 entry = huge_ptep_get(ptep); 3837 if (huge_pte_none(entry)) { 3838 ret = hugetlb_no_page(mm, vma, mapping, idx, address, ptep, flags); 3839 goto out_mutex; 3840 } 3841 3842 ret = 0; 3843 3844 /* 3845 * entry could be a migration/hwpoison entry at this point, so this 3846 * check prevents the kernel from going below assuming that we have 3847 * a active hugepage in pagecache. This goto expects the 2nd page fault, 3848 * and is_hugetlb_entry_(migration|hwpoisoned) check will properly 3849 * handle it. 3850 */ 3851 if (!pte_present(entry)) 3852 goto out_mutex; 3853 3854 /* 3855 * If we are going to COW the mapping later, we examine the pending 3856 * reservations for this page now. This will ensure that any 3857 * allocations necessary to record that reservation occur outside the 3858 * spinlock. For private mappings, we also lookup the pagecache 3859 * page now as it is used to determine if a reservation has been 3860 * consumed. 3861 */ 3862 if ((flags & FAULT_FLAG_WRITE) && !huge_pte_write(entry)) { 3863 if (vma_needs_reservation(h, vma, address) < 0) { 3864 ret = VM_FAULT_OOM; 3865 goto out_mutex; 3866 } 3867 /* Just decrements count, does not deallocate */ 3868 vma_end_reservation(h, vma, address); 3869 3870 if (!(vma->vm_flags & VM_MAYSHARE)) 3871 pagecache_page = hugetlbfs_pagecache_page(h, 3872 vma, address); 3873 } 3874 3875 ptl = huge_pte_lock(h, mm, ptep); 3876 3877 /* Check for a racing update before calling hugetlb_cow */ 3878 if (unlikely(!pte_same(entry, huge_ptep_get(ptep)))) 3879 goto out_ptl; 3880 3881 /* 3882 * hugetlb_cow() requires page locks of pte_page(entry) and 3883 * pagecache_page, so here we need take the former one 3884 * when page != pagecache_page or !pagecache_page. 3885 */ 3886 page = pte_page(entry); 3887 if (page != pagecache_page) 3888 if (!trylock_page(page)) { 3889 need_wait_lock = 1; 3890 goto out_ptl; 3891 } 3892 3893 get_page(page); 3894 3895 if (flags & FAULT_FLAG_WRITE) { 3896 if (!huge_pte_write(entry)) { 3897 ret = hugetlb_cow(mm, vma, address, ptep, 3898 pagecache_page, ptl); 3899 goto out_put_page; 3900 } 3901 entry = huge_pte_mkdirty(entry); 3902 } 3903 entry = pte_mkyoung(entry); 3904 if (huge_ptep_set_access_flags(vma, address, ptep, entry, 3905 flags & FAULT_FLAG_WRITE)) 3906 update_mmu_cache(vma, address, ptep); 3907 out_put_page: 3908 if (page != pagecache_page) 3909 unlock_page(page); 3910 put_page(page); 3911 out_ptl: 3912 spin_unlock(ptl); 3913 3914 if (pagecache_page) { 3915 unlock_page(pagecache_page); 3916 put_page(pagecache_page); 3917 } 3918 out_mutex: 3919 mutex_unlock(&hugetlb_fault_mutex_table[hash]); 3920 /* 3921 * Generally it's safe to hold refcount during waiting page lock. But 3922 * here we just wait to defer the next page fault to avoid busy loop and 3923 * the page is not used after unlocked before returning from the current 3924 * page fault. So we are safe from accessing freed page, even if we wait 3925 * here without taking refcount. 3926 */ 3927 if (need_wait_lock) 3928 wait_on_page_locked(page); 3929 return ret; 3930 } 3931 3932 long follow_hugetlb_page(struct mm_struct *mm, struct vm_area_struct *vma, 3933 struct page **pages, struct vm_area_struct **vmas, 3934 unsigned long *position, unsigned long *nr_pages, 3935 long i, unsigned int flags) 3936 { 3937 unsigned long pfn_offset; 3938 unsigned long vaddr = *position; 3939 unsigned long remainder = *nr_pages; 3940 struct hstate *h = hstate_vma(vma); 3941 3942 while (vaddr < vma->vm_end && remainder) { 3943 pte_t *pte; 3944 spinlock_t *ptl = NULL; 3945 int absent; 3946 struct page *page; 3947 3948 /* 3949 * If we have a pending SIGKILL, don't keep faulting pages and 3950 * potentially allocating memory. 3951 */ 3952 if (unlikely(fatal_signal_pending(current))) { 3953 remainder = 0; 3954 break; 3955 } 3956 3957 /* 3958 * Some archs (sparc64, sh*) have multiple pte_ts to 3959 * each hugepage. We have to make sure we get the 3960 * first, for the page indexing below to work. 3961 * 3962 * Note that page table lock is not held when pte is null. 3963 */ 3964 pte = huge_pte_offset(mm, vaddr & huge_page_mask(h)); 3965 if (pte) 3966 ptl = huge_pte_lock(h, mm, pte); 3967 absent = !pte || huge_pte_none(huge_ptep_get(pte)); 3968 3969 /* 3970 * When coredumping, it suits get_dump_page if we just return 3971 * an error where there's an empty slot with no huge pagecache 3972 * to back it. This way, we avoid allocating a hugepage, and 3973 * the sparse dumpfile avoids allocating disk blocks, but its 3974 * huge holes still show up with zeroes where they need to be. 3975 */ 3976 if (absent && (flags & FOLL_DUMP) && 3977 !hugetlbfs_pagecache_present(h, vma, vaddr)) { 3978 if (pte) 3979 spin_unlock(ptl); 3980 remainder = 0; 3981 break; 3982 } 3983 3984 /* 3985 * We need call hugetlb_fault for both hugepages under migration 3986 * (in which case hugetlb_fault waits for the migration,) and 3987 * hwpoisoned hugepages (in which case we need to prevent the 3988 * caller from accessing to them.) In order to do this, we use 3989 * here is_swap_pte instead of is_hugetlb_entry_migration and 3990 * is_hugetlb_entry_hwpoisoned. This is because it simply covers 3991 * both cases, and because we can't follow correct pages 3992 * directly from any kind of swap entries. 3993 */ 3994 if (absent || is_swap_pte(huge_ptep_get(pte)) || 3995 ((flags & FOLL_WRITE) && 3996 !huge_pte_write(huge_ptep_get(pte)))) { 3997 int ret; 3998 3999 if (pte) 4000 spin_unlock(ptl); 4001 ret = hugetlb_fault(mm, vma, vaddr, 4002 (flags & FOLL_WRITE) ? FAULT_FLAG_WRITE : 0); 4003 if (!(ret & VM_FAULT_ERROR)) 4004 continue; 4005 4006 remainder = 0; 4007 break; 4008 } 4009 4010 pfn_offset = (vaddr & ~huge_page_mask(h)) >> PAGE_SHIFT; 4011 page = pte_page(huge_ptep_get(pte)); 4012 same_page: 4013 if (pages) { 4014 pages[i] = mem_map_offset(page, pfn_offset); 4015 get_page(pages[i]); 4016 } 4017 4018 if (vmas) 4019 vmas[i] = vma; 4020 4021 vaddr += PAGE_SIZE; 4022 ++pfn_offset; 4023 --remainder; 4024 ++i; 4025 if (vaddr < vma->vm_end && remainder && 4026 pfn_offset < pages_per_huge_page(h)) { 4027 /* 4028 * We use pfn_offset to avoid touching the pageframes 4029 * of this compound page. 4030 */ 4031 goto same_page; 4032 } 4033 spin_unlock(ptl); 4034 } 4035 *nr_pages = remainder; 4036 *position = vaddr; 4037 4038 return i ? i : -EFAULT; 4039 } 4040 4041 #ifndef __HAVE_ARCH_FLUSH_HUGETLB_TLB_RANGE 4042 /* 4043 * ARCHes with special requirements for evicting HUGETLB backing TLB entries can 4044 * implement this. 4045 */ 4046 #define flush_hugetlb_tlb_range(vma, addr, end) flush_tlb_range(vma, addr, end) 4047 #endif 4048 4049 unsigned long hugetlb_change_protection(struct vm_area_struct *vma, 4050 unsigned long address, unsigned long end, pgprot_t newprot) 4051 { 4052 struct mm_struct *mm = vma->vm_mm; 4053 unsigned long start = address; 4054 pte_t *ptep; 4055 pte_t pte; 4056 struct hstate *h = hstate_vma(vma); 4057 unsigned long pages = 0; 4058 4059 BUG_ON(address >= end); 4060 flush_cache_range(vma, address, end); 4061 4062 mmu_notifier_invalidate_range_start(mm, start, end); 4063 i_mmap_lock_write(vma->vm_file->f_mapping); 4064 for (; address < end; address += huge_page_size(h)) { 4065 spinlock_t *ptl; 4066 ptep = huge_pte_offset(mm, address); 4067 if (!ptep) 4068 continue; 4069 ptl = huge_pte_lock(h, mm, ptep); 4070 if (huge_pmd_unshare(mm, &address, ptep)) { 4071 pages++; 4072 spin_unlock(ptl); 4073 continue; 4074 } 4075 pte = huge_ptep_get(ptep); 4076 if (unlikely(is_hugetlb_entry_hwpoisoned(pte))) { 4077 spin_unlock(ptl); 4078 continue; 4079 } 4080 if (unlikely(is_hugetlb_entry_migration(pte))) { 4081 swp_entry_t entry = pte_to_swp_entry(pte); 4082 4083 if (is_write_migration_entry(entry)) { 4084 pte_t newpte; 4085 4086 make_migration_entry_read(&entry); 4087 newpte = swp_entry_to_pte(entry); 4088 set_huge_pte_at(mm, address, ptep, newpte); 4089 pages++; 4090 } 4091 spin_unlock(ptl); 4092 continue; 4093 } 4094 if (!huge_pte_none(pte)) { 4095 pte = huge_ptep_get_and_clear(mm, address, ptep); 4096 pte = pte_mkhuge(huge_pte_modify(pte, newprot)); 4097 pte = arch_make_huge_pte(pte, vma, NULL, 0); 4098 set_huge_pte_at(mm, address, ptep, pte); 4099 pages++; 4100 } 4101 spin_unlock(ptl); 4102 } 4103 /* 4104 * Must flush TLB before releasing i_mmap_rwsem: x86's huge_pmd_unshare 4105 * may have cleared our pud entry and done put_page on the page table: 4106 * once we release i_mmap_rwsem, another task can do the final put_page 4107 * and that page table be reused and filled with junk. 4108 */ 4109 flush_hugetlb_tlb_range(vma, start, end); 4110 mmu_notifier_invalidate_range(mm, start, end); 4111 i_mmap_unlock_write(vma->vm_file->f_mapping); 4112 mmu_notifier_invalidate_range_end(mm, start, end); 4113 4114 return pages << h->order; 4115 } 4116 4117 int hugetlb_reserve_pages(struct inode *inode, 4118 long from, long to, 4119 struct vm_area_struct *vma, 4120 vm_flags_t vm_flags) 4121 { 4122 long ret, chg; 4123 struct hstate *h = hstate_inode(inode); 4124 struct hugepage_subpool *spool = subpool_inode(inode); 4125 struct resv_map *resv_map; 4126 long gbl_reserve; 4127 4128 /* 4129 * Only apply hugepage reservation if asked. At fault time, an 4130 * attempt will be made for VM_NORESERVE to allocate a page 4131 * without using reserves 4132 */ 4133 if (vm_flags & VM_NORESERVE) 4134 return 0; 4135 4136 /* 4137 * Shared mappings base their reservation on the number of pages that 4138 * are already allocated on behalf of the file. Private mappings need 4139 * to reserve the full area even if read-only as mprotect() may be 4140 * called to make the mapping read-write. Assume !vma is a shm mapping 4141 */ 4142 if (!vma || vma->vm_flags & VM_MAYSHARE) { 4143 resv_map = inode_resv_map(inode); 4144 4145 chg = region_chg(resv_map, from, to); 4146 4147 } else { 4148 resv_map = resv_map_alloc(); 4149 if (!resv_map) 4150 return -ENOMEM; 4151 4152 chg = to - from; 4153 4154 set_vma_resv_map(vma, resv_map); 4155 set_vma_resv_flags(vma, HPAGE_RESV_OWNER); 4156 } 4157 4158 if (chg < 0) { 4159 ret = chg; 4160 goto out_err; 4161 } 4162 4163 /* 4164 * There must be enough pages in the subpool for the mapping. If 4165 * the subpool has a minimum size, there may be some global 4166 * reservations already in place (gbl_reserve). 4167 */ 4168 gbl_reserve = hugepage_subpool_get_pages(spool, chg); 4169 if (gbl_reserve < 0) { 4170 ret = -ENOSPC; 4171 goto out_err; 4172 } 4173 4174 /* 4175 * Check enough hugepages are available for the reservation. 4176 * Hand the pages back to the subpool if there are not 4177 */ 4178 ret = hugetlb_acct_memory(h, gbl_reserve); 4179 if (ret < 0) { 4180 /* put back original number of pages, chg */ 4181 (void)hugepage_subpool_put_pages(spool, chg); 4182 goto out_err; 4183 } 4184 4185 /* 4186 * Account for the reservations made. Shared mappings record regions 4187 * that have reservations as they are shared by multiple VMAs. 4188 * When the last VMA disappears, the region map says how much 4189 * the reservation was and the page cache tells how much of 4190 * the reservation was consumed. Private mappings are per-VMA and 4191 * only the consumed reservations are tracked. When the VMA 4192 * disappears, the original reservation is the VMA size and the 4193 * consumed reservations are stored in the map. Hence, nothing 4194 * else has to be done for private mappings here 4195 */ 4196 if (!vma || vma->vm_flags & VM_MAYSHARE) { 4197 long add = region_add(resv_map, from, to); 4198 4199 if (unlikely(chg > add)) { 4200 /* 4201 * pages in this range were added to the reserve 4202 * map between region_chg and region_add. This 4203 * indicates a race with alloc_huge_page. Adjust 4204 * the subpool and reserve counts modified above 4205 * based on the difference. 4206 */ 4207 long rsv_adjust; 4208 4209 rsv_adjust = hugepage_subpool_put_pages(spool, 4210 chg - add); 4211 hugetlb_acct_memory(h, -rsv_adjust); 4212 } 4213 } 4214 return 0; 4215 out_err: 4216 if (!vma || vma->vm_flags & VM_MAYSHARE) 4217 region_abort(resv_map, from, to); 4218 if (vma && is_vma_resv_set(vma, HPAGE_RESV_OWNER)) 4219 kref_put(&resv_map->refs, resv_map_release); 4220 return ret; 4221 } 4222 4223 long hugetlb_unreserve_pages(struct inode *inode, long start, long end, 4224 long freed) 4225 { 4226 struct hstate *h = hstate_inode(inode); 4227 struct resv_map *resv_map = inode_resv_map(inode); 4228 long chg = 0; 4229 struct hugepage_subpool *spool = subpool_inode(inode); 4230 long gbl_reserve; 4231 4232 if (resv_map) { 4233 chg = region_del(resv_map, start, end); 4234 /* 4235 * region_del() can fail in the rare case where a region 4236 * must be split and another region descriptor can not be 4237 * allocated. If end == LONG_MAX, it will not fail. 4238 */ 4239 if (chg < 0) 4240 return chg; 4241 } 4242 4243 spin_lock(&inode->i_lock); 4244 inode->i_blocks -= (blocks_per_huge_page(h) * freed); 4245 spin_unlock(&inode->i_lock); 4246 4247 /* 4248 * If the subpool has a minimum size, the number of global 4249 * reservations to be released may be adjusted. 4250 */ 4251 gbl_reserve = hugepage_subpool_put_pages(spool, (chg - freed)); 4252 hugetlb_acct_memory(h, -gbl_reserve); 4253 4254 return 0; 4255 } 4256 4257 #ifdef CONFIG_ARCH_WANT_HUGE_PMD_SHARE 4258 static unsigned long page_table_shareable(struct vm_area_struct *svma, 4259 struct vm_area_struct *vma, 4260 unsigned long addr, pgoff_t idx) 4261 { 4262 unsigned long saddr = ((idx - svma->vm_pgoff) << PAGE_SHIFT) + 4263 svma->vm_start; 4264 unsigned long sbase = saddr & PUD_MASK; 4265 unsigned long s_end = sbase + PUD_SIZE; 4266 4267 /* Allow segments to share if only one is marked locked */ 4268 unsigned long vm_flags = vma->vm_flags & VM_LOCKED_CLEAR_MASK; 4269 unsigned long svm_flags = svma->vm_flags & VM_LOCKED_CLEAR_MASK; 4270 4271 /* 4272 * match the virtual addresses, permission and the alignment of the 4273 * page table page. 4274 */ 4275 if (pmd_index(addr) != pmd_index(saddr) || 4276 vm_flags != svm_flags || 4277 sbase < svma->vm_start || svma->vm_end < s_end) 4278 return 0; 4279 4280 return saddr; 4281 } 4282 4283 static bool vma_shareable(struct vm_area_struct *vma, unsigned long addr) 4284 { 4285 unsigned long base = addr & PUD_MASK; 4286 unsigned long end = base + PUD_SIZE; 4287 4288 /* 4289 * check on proper vm_flags and page table alignment 4290 */ 4291 if (vma->vm_flags & VM_MAYSHARE && 4292 vma->vm_start <= base && end <= vma->vm_end) 4293 return true; 4294 return false; 4295 } 4296 4297 /* 4298 * Search for a shareable pmd page for hugetlb. In any case calls pmd_alloc() 4299 * and returns the corresponding pte. While this is not necessary for the 4300 * !shared pmd case because we can allocate the pmd later as well, it makes the 4301 * code much cleaner. pmd allocation is essential for the shared case because 4302 * pud has to be populated inside the same i_mmap_rwsem section - otherwise 4303 * racing tasks could either miss the sharing (see huge_pte_offset) or select a 4304 * bad pmd for sharing. 4305 */ 4306 pte_t *huge_pmd_share(struct mm_struct *mm, unsigned long addr, pud_t *pud) 4307 { 4308 struct vm_area_struct *vma = find_vma(mm, addr); 4309 struct address_space *mapping = vma->vm_file->f_mapping; 4310 pgoff_t idx = ((addr - vma->vm_start) >> PAGE_SHIFT) + 4311 vma->vm_pgoff; 4312 struct vm_area_struct *svma; 4313 unsigned long saddr; 4314 pte_t *spte = NULL; 4315 pte_t *pte; 4316 spinlock_t *ptl; 4317 4318 if (!vma_shareable(vma, addr)) 4319 return (pte_t *)pmd_alloc(mm, pud, addr); 4320 4321 i_mmap_lock_write(mapping); 4322 vma_interval_tree_foreach(svma, &mapping->i_mmap, idx, idx) { 4323 if (svma == vma) 4324 continue; 4325 4326 saddr = page_table_shareable(svma, vma, addr, idx); 4327 if (saddr) { 4328 spte = huge_pte_offset(svma->vm_mm, saddr); 4329 if (spte) { 4330 get_page(virt_to_page(spte)); 4331 break; 4332 } 4333 } 4334 } 4335 4336 if (!spte) 4337 goto out; 4338 4339 ptl = huge_pte_lock(hstate_vma(vma), mm, spte); 4340 if (pud_none(*pud)) { 4341 pud_populate(mm, pud, 4342 (pmd_t *)((unsigned long)spte & PAGE_MASK)); 4343 mm_inc_nr_pmds(mm); 4344 } else { 4345 put_page(virt_to_page(spte)); 4346 } 4347 spin_unlock(ptl); 4348 out: 4349 pte = (pte_t *)pmd_alloc(mm, pud, addr); 4350 i_mmap_unlock_write(mapping); 4351 return pte; 4352 } 4353 4354 /* 4355 * unmap huge page backed by shared pte. 4356 * 4357 * Hugetlb pte page is ref counted at the time of mapping. If pte is shared 4358 * indicated by page_count > 1, unmap is achieved by clearing pud and 4359 * decrementing the ref count. If count == 1, the pte page is not shared. 4360 * 4361 * called with page table lock held. 4362 * 4363 * returns: 1 successfully unmapped a shared pte page 4364 * 0 the underlying pte page is not shared, or it is the last user 4365 */ 4366 int huge_pmd_unshare(struct mm_struct *mm, unsigned long *addr, pte_t *ptep) 4367 { 4368 pgd_t *pgd = pgd_offset(mm, *addr); 4369 pud_t *pud = pud_offset(pgd, *addr); 4370 4371 BUG_ON(page_count(virt_to_page(ptep)) == 0); 4372 if (page_count(virt_to_page(ptep)) == 1) 4373 return 0; 4374 4375 pud_clear(pud); 4376 put_page(virt_to_page(ptep)); 4377 mm_dec_nr_pmds(mm); 4378 *addr = ALIGN(*addr, HPAGE_SIZE * PTRS_PER_PTE) - HPAGE_SIZE; 4379 return 1; 4380 } 4381 #define want_pmd_share() (1) 4382 #else /* !CONFIG_ARCH_WANT_HUGE_PMD_SHARE */ 4383 pte_t *huge_pmd_share(struct mm_struct *mm, unsigned long addr, pud_t *pud) 4384 { 4385 return NULL; 4386 } 4387 4388 int huge_pmd_unshare(struct mm_struct *mm, unsigned long *addr, pte_t *ptep) 4389 { 4390 return 0; 4391 } 4392 #define want_pmd_share() (0) 4393 #endif /* CONFIG_ARCH_WANT_HUGE_PMD_SHARE */ 4394 4395 #ifdef CONFIG_ARCH_WANT_GENERAL_HUGETLB 4396 pte_t *huge_pte_alloc(struct mm_struct *mm, 4397 unsigned long addr, unsigned long sz) 4398 { 4399 pgd_t *pgd; 4400 pud_t *pud; 4401 pte_t *pte = NULL; 4402 4403 pgd = pgd_offset(mm, addr); 4404 pud = pud_alloc(mm, pgd, addr); 4405 if (pud) { 4406 if (sz == PUD_SIZE) { 4407 pte = (pte_t *)pud; 4408 } else { 4409 BUG_ON(sz != PMD_SIZE); 4410 if (want_pmd_share() && pud_none(*pud)) 4411 pte = huge_pmd_share(mm, addr, pud); 4412 else 4413 pte = (pte_t *)pmd_alloc(mm, pud, addr); 4414 } 4415 } 4416 BUG_ON(pte && pte_present(*pte) && !pte_huge(*pte)); 4417 4418 return pte; 4419 } 4420 4421 pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr) 4422 { 4423 pgd_t *pgd; 4424 pud_t *pud; 4425 pmd_t *pmd = NULL; 4426 4427 pgd = pgd_offset(mm, addr); 4428 if (pgd_present(*pgd)) { 4429 pud = pud_offset(pgd, addr); 4430 if (pud_present(*pud)) { 4431 if (pud_huge(*pud)) 4432 return (pte_t *)pud; 4433 pmd = pmd_offset(pud, addr); 4434 } 4435 } 4436 return (pte_t *) pmd; 4437 } 4438 4439 #endif /* CONFIG_ARCH_WANT_GENERAL_HUGETLB */ 4440 4441 /* 4442 * These functions are overwritable if your architecture needs its own 4443 * behavior. 4444 */ 4445 struct page * __weak 4446 follow_huge_addr(struct mm_struct *mm, unsigned long address, 4447 int write) 4448 { 4449 return ERR_PTR(-EINVAL); 4450 } 4451 4452 struct page * __weak 4453 follow_huge_pmd(struct mm_struct *mm, unsigned long address, 4454 pmd_t *pmd, int flags) 4455 { 4456 struct page *page = NULL; 4457 spinlock_t *ptl; 4458 retry: 4459 ptl = pmd_lockptr(mm, pmd); 4460 spin_lock(ptl); 4461 /* 4462 * make sure that the address range covered by this pmd is not 4463 * unmapped from other threads. 4464 */ 4465 if (!pmd_huge(*pmd)) 4466 goto out; 4467 if (pmd_present(*pmd)) { 4468 page = pmd_page(*pmd) + ((address & ~PMD_MASK) >> PAGE_SHIFT); 4469 if (flags & FOLL_GET) 4470 get_page(page); 4471 } else { 4472 if (is_hugetlb_entry_migration(huge_ptep_get((pte_t *)pmd))) { 4473 spin_unlock(ptl); 4474 __migration_entry_wait(mm, (pte_t *)pmd, ptl); 4475 goto retry; 4476 } 4477 /* 4478 * hwpoisoned entry is treated as no_page_table in 4479 * follow_page_mask(). 4480 */ 4481 } 4482 out: 4483 spin_unlock(ptl); 4484 return page; 4485 } 4486 4487 struct page * __weak 4488 follow_huge_pud(struct mm_struct *mm, unsigned long address, 4489 pud_t *pud, int flags) 4490 { 4491 if (flags & FOLL_GET) 4492 return NULL; 4493 4494 return pte_page(*(pte_t *)pud) + ((address & ~PUD_MASK) >> PAGE_SHIFT); 4495 } 4496 4497 #ifdef CONFIG_MEMORY_FAILURE 4498 4499 /* 4500 * This function is called from memory failure code. 4501 */ 4502 int dequeue_hwpoisoned_huge_page(struct page *hpage) 4503 { 4504 struct hstate *h = page_hstate(hpage); 4505 int nid = page_to_nid(hpage); 4506 int ret = -EBUSY; 4507 4508 spin_lock(&hugetlb_lock); 4509 /* 4510 * Just checking !page_huge_active is not enough, because that could be 4511 * an isolated/hwpoisoned hugepage (which have >0 refcount). 4512 */ 4513 if (!page_huge_active(hpage) && !page_count(hpage)) { 4514 /* 4515 * Hwpoisoned hugepage isn't linked to activelist or freelist, 4516 * but dangling hpage->lru can trigger list-debug warnings 4517 * (this happens when we call unpoison_memory() on it), 4518 * so let it point to itself with list_del_init(). 4519 */ 4520 list_del_init(&hpage->lru); 4521 set_page_refcounted(hpage); 4522 h->free_huge_pages--; 4523 h->free_huge_pages_node[nid]--; 4524 ret = 0; 4525 } 4526 spin_unlock(&hugetlb_lock); 4527 return ret; 4528 } 4529 #endif 4530 4531 bool isolate_huge_page(struct page *page, struct list_head *list) 4532 { 4533 bool ret = true; 4534 4535 VM_BUG_ON_PAGE(!PageHead(page), page); 4536 spin_lock(&hugetlb_lock); 4537 if (!page_huge_active(page) || !get_page_unless_zero(page)) { 4538 ret = false; 4539 goto unlock; 4540 } 4541 clear_page_huge_active(page); 4542 list_move_tail(&page->lru, list); 4543 unlock: 4544 spin_unlock(&hugetlb_lock); 4545 return ret; 4546 } 4547 4548 void putback_active_hugepage(struct page *page) 4549 { 4550 VM_BUG_ON_PAGE(!PageHead(page), page); 4551 spin_lock(&hugetlb_lock); 4552 set_page_huge_active(page); 4553 list_move_tail(&page->lru, &(page_hstate(page))->hugepage_activelist); 4554 spin_unlock(&hugetlb_lock); 4555 put_page(page); 4556 } 4557