1 /* 2 * PPC Huge TLB Page Support for Kernel. 3 * 4 * Copyright (C) 2003 David Gibson, IBM Corporation. 5 * Copyright (C) 2011 Becky Bruce, Freescale Semiconductor 6 * 7 * Based on the IA-32 version: 8 * Copyright (C) 2002, Rohit Seth <rohit.seth@intel.com> 9 */ 10 11 #include <linux/mm.h> 12 #include <linux/io.h> 13 #include <linux/slab.h> 14 #include <linux/hugetlb.h> 15 #include <linux/export.h> 16 #include <linux/of_fdt.h> 17 #include <linux/memblock.h> 18 #include <linux/moduleparam.h> 19 #include <linux/swap.h> 20 #include <linux/swapops.h> 21 #include <linux/kmemleak.h> 22 #include <asm/pgalloc.h> 23 #include <asm/tlb.h> 24 #include <asm/setup.h> 25 #include <asm/hugetlb.h> 26 #include <asm/pte-walk.h> 27 28 bool hugetlb_disabled = false; 29 30 #define hugepd_none(hpd) (hpd_val(hpd) == 0) 31 32 #define PTE_T_ORDER (__builtin_ffs(sizeof(pte_basic_t)) - \ 33 __builtin_ffs(sizeof(void *))) 34 35 pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr, unsigned long sz) 36 { 37 /* 38 * Only called for hugetlbfs pages, hence can ignore THP and the 39 * irq disabled walk. 40 */ 41 return __find_linux_pte(mm->pgd, addr, NULL, NULL); 42 } 43 44 static int __hugepte_alloc(struct mm_struct *mm, hugepd_t *hpdp, 45 unsigned long address, unsigned int pdshift, 46 unsigned int pshift, spinlock_t *ptl) 47 { 48 struct kmem_cache *cachep; 49 pte_t *new; 50 int i; 51 int num_hugepd; 52 53 if (pshift >= pdshift) { 54 cachep = PGT_CACHE(PTE_T_ORDER); 55 num_hugepd = 1 << (pshift - pdshift); 56 } else { 57 cachep = PGT_CACHE(pdshift - pshift); 58 num_hugepd = 1; 59 } 60 61 if (!cachep) { 62 WARN_ONCE(1, "No page table cache created for hugetlb tables"); 63 return -ENOMEM; 64 } 65 66 new = kmem_cache_alloc(cachep, pgtable_gfp_flags(mm, GFP_KERNEL)); 67 68 BUG_ON(pshift > HUGEPD_SHIFT_MASK); 69 BUG_ON((unsigned long)new & HUGEPD_SHIFT_MASK); 70 71 if (!new) 72 return -ENOMEM; 73 74 /* 75 * Make sure other cpus find the hugepd set only after a 76 * properly initialized page table is visible to them. 77 * For more details look for comment in __pte_alloc(). 78 */ 79 smp_wmb(); 80 81 spin_lock(ptl); 82 /* 83 * We have multiple higher-level entries that point to the same 84 * actual pte location. Fill in each as we go and backtrack on error. 85 * We need all of these so the DTLB pgtable walk code can find the 86 * right higher-level entry without knowing if it's a hugepage or not. 87 */ 88 for (i = 0; i < num_hugepd; i++, hpdp++) { 89 if (unlikely(!hugepd_none(*hpdp))) 90 break; 91 hugepd_populate(hpdp, new, pshift); 92 } 93 /* If we bailed from the for loop early, an error occurred, clean up */ 94 if (i < num_hugepd) { 95 for (i = i - 1 ; i >= 0; i--, hpdp--) 96 *hpdp = __hugepd(0); 97 kmem_cache_free(cachep, new); 98 } else { 99 kmemleak_ignore(new); 100 } 101 spin_unlock(ptl); 102 return 0; 103 } 104 105 /* 106 * At this point we do the placement change only for BOOK3S 64. This would 107 * possibly work on other subarchs. 108 */ 109 pte_t *huge_pte_alloc(struct mm_struct *mm, unsigned long addr, unsigned long sz) 110 { 111 pgd_t *pg; 112 p4d_t *p4; 113 pud_t *pu; 114 pmd_t *pm; 115 hugepd_t *hpdp = NULL; 116 unsigned pshift = __ffs(sz); 117 unsigned pdshift = PGDIR_SHIFT; 118 spinlock_t *ptl; 119 120 addr &= ~(sz-1); 121 pg = pgd_offset(mm, addr); 122 p4 = p4d_offset(pg, addr); 123 124 #ifdef CONFIG_PPC_BOOK3S_64 125 if (pshift == PGDIR_SHIFT) 126 /* 16GB huge page */ 127 return (pte_t *) p4; 128 else if (pshift > PUD_SHIFT) { 129 /* 130 * We need to use hugepd table 131 */ 132 ptl = &mm->page_table_lock; 133 hpdp = (hugepd_t *)p4; 134 } else { 135 pdshift = PUD_SHIFT; 136 pu = pud_alloc(mm, p4, addr); 137 if (!pu) 138 return NULL; 139 if (pshift == PUD_SHIFT) 140 return (pte_t *)pu; 141 else if (pshift > PMD_SHIFT) { 142 ptl = pud_lockptr(mm, pu); 143 hpdp = (hugepd_t *)pu; 144 } else { 145 pdshift = PMD_SHIFT; 146 pm = pmd_alloc(mm, pu, addr); 147 if (!pm) 148 return NULL; 149 if (pshift == PMD_SHIFT) 150 /* 16MB hugepage */ 151 return (pte_t *)pm; 152 else { 153 ptl = pmd_lockptr(mm, pm); 154 hpdp = (hugepd_t *)pm; 155 } 156 } 157 } 158 #else 159 if (pshift >= PGDIR_SHIFT) { 160 ptl = &mm->page_table_lock; 161 hpdp = (hugepd_t *)p4; 162 } else { 163 pdshift = PUD_SHIFT; 164 pu = pud_alloc(mm, p4, addr); 165 if (!pu) 166 return NULL; 167 if (pshift >= PUD_SHIFT) { 168 ptl = pud_lockptr(mm, pu); 169 hpdp = (hugepd_t *)pu; 170 } else { 171 pdshift = PMD_SHIFT; 172 pm = pmd_alloc(mm, pu, addr); 173 if (!pm) 174 return NULL; 175 ptl = pmd_lockptr(mm, pm); 176 hpdp = (hugepd_t *)pm; 177 } 178 } 179 #endif 180 if (!hpdp) 181 return NULL; 182 183 if (IS_ENABLED(CONFIG_PPC_8xx) && pshift < PMD_SHIFT) 184 return pte_alloc_map(mm, (pmd_t *)hpdp, addr); 185 186 BUG_ON(!hugepd_none(*hpdp) && !hugepd_ok(*hpdp)); 187 188 if (hugepd_none(*hpdp) && __hugepte_alloc(mm, hpdp, addr, 189 pdshift, pshift, ptl)) 190 return NULL; 191 192 return hugepte_offset(*hpdp, addr, pdshift); 193 } 194 195 #ifdef CONFIG_PPC_BOOK3S_64 196 /* 197 * Tracks gpages after the device tree is scanned and before the 198 * huge_boot_pages list is ready on pseries. 199 */ 200 #define MAX_NUMBER_GPAGES 1024 201 __initdata static u64 gpage_freearray[MAX_NUMBER_GPAGES]; 202 __initdata static unsigned nr_gpages; 203 204 /* 205 * Build list of addresses of gigantic pages. This function is used in early 206 * boot before the buddy allocator is setup. 207 */ 208 void __init pseries_add_gpage(u64 addr, u64 page_size, unsigned long number_of_pages) 209 { 210 if (!addr) 211 return; 212 while (number_of_pages > 0) { 213 gpage_freearray[nr_gpages] = addr; 214 nr_gpages++; 215 number_of_pages--; 216 addr += page_size; 217 } 218 } 219 220 int __init pseries_alloc_bootmem_huge_page(struct hstate *hstate) 221 { 222 struct huge_bootmem_page *m; 223 if (nr_gpages == 0) 224 return 0; 225 m = phys_to_virt(gpage_freearray[--nr_gpages]); 226 gpage_freearray[nr_gpages] = 0; 227 list_add(&m->list, &huge_boot_pages); 228 m->hstate = hstate; 229 return 1; 230 } 231 #endif 232 233 234 int __init alloc_bootmem_huge_page(struct hstate *h) 235 { 236 237 #ifdef CONFIG_PPC_BOOK3S_64 238 if (firmware_has_feature(FW_FEATURE_LPAR) && !radix_enabled()) 239 return pseries_alloc_bootmem_huge_page(h); 240 #endif 241 return __alloc_bootmem_huge_page(h); 242 } 243 244 #ifndef CONFIG_PPC_BOOK3S_64 245 #define HUGEPD_FREELIST_SIZE \ 246 ((PAGE_SIZE - sizeof(struct hugepd_freelist)) / sizeof(pte_t)) 247 248 struct hugepd_freelist { 249 struct rcu_head rcu; 250 unsigned int index; 251 void *ptes[]; 252 }; 253 254 static DEFINE_PER_CPU(struct hugepd_freelist *, hugepd_freelist_cur); 255 256 static void hugepd_free_rcu_callback(struct rcu_head *head) 257 { 258 struct hugepd_freelist *batch = 259 container_of(head, struct hugepd_freelist, rcu); 260 unsigned int i; 261 262 for (i = 0; i < batch->index; i++) 263 kmem_cache_free(PGT_CACHE(PTE_T_ORDER), batch->ptes[i]); 264 265 free_page((unsigned long)batch); 266 } 267 268 static void hugepd_free(struct mmu_gather *tlb, void *hugepte) 269 { 270 struct hugepd_freelist **batchp; 271 272 batchp = &get_cpu_var(hugepd_freelist_cur); 273 274 if (atomic_read(&tlb->mm->mm_users) < 2 || 275 mm_is_thread_local(tlb->mm)) { 276 kmem_cache_free(PGT_CACHE(PTE_T_ORDER), hugepte); 277 put_cpu_var(hugepd_freelist_cur); 278 return; 279 } 280 281 if (*batchp == NULL) { 282 *batchp = (struct hugepd_freelist *)__get_free_page(GFP_ATOMIC); 283 (*batchp)->index = 0; 284 } 285 286 (*batchp)->ptes[(*batchp)->index++] = hugepte; 287 if ((*batchp)->index == HUGEPD_FREELIST_SIZE) { 288 call_rcu(&(*batchp)->rcu, hugepd_free_rcu_callback); 289 *batchp = NULL; 290 } 291 put_cpu_var(hugepd_freelist_cur); 292 } 293 #else 294 static inline void hugepd_free(struct mmu_gather *tlb, void *hugepte) {} 295 #endif 296 297 static void free_hugepd_range(struct mmu_gather *tlb, hugepd_t *hpdp, int pdshift, 298 unsigned long start, unsigned long end, 299 unsigned long floor, unsigned long ceiling) 300 { 301 pte_t *hugepte = hugepd_page(*hpdp); 302 int i; 303 304 unsigned long pdmask = ~((1UL << pdshift) - 1); 305 unsigned int num_hugepd = 1; 306 unsigned int shift = hugepd_shift(*hpdp); 307 308 /* Note: On fsl the hpdp may be the first of several */ 309 if (shift > pdshift) 310 num_hugepd = 1 << (shift - pdshift); 311 312 start &= pdmask; 313 if (start < floor) 314 return; 315 if (ceiling) { 316 ceiling &= pdmask; 317 if (! ceiling) 318 return; 319 } 320 if (end - 1 > ceiling - 1) 321 return; 322 323 for (i = 0; i < num_hugepd; i++, hpdp++) 324 *hpdp = __hugepd(0); 325 326 if (shift >= pdshift) 327 hugepd_free(tlb, hugepte); 328 else 329 pgtable_free_tlb(tlb, hugepte, 330 get_hugepd_cache_index(pdshift - shift)); 331 } 332 333 static void hugetlb_free_pte_range(struct mmu_gather *tlb, pmd_t *pmd, 334 unsigned long addr, unsigned long end, 335 unsigned long floor, unsigned long ceiling) 336 { 337 unsigned long start = addr; 338 pgtable_t token = pmd_pgtable(*pmd); 339 340 start &= PMD_MASK; 341 if (start < floor) 342 return; 343 if (ceiling) { 344 ceiling &= PMD_MASK; 345 if (!ceiling) 346 return; 347 } 348 if (end - 1 > ceiling - 1) 349 return; 350 351 pmd_clear(pmd); 352 pte_free_tlb(tlb, token, addr); 353 mm_dec_nr_ptes(tlb->mm); 354 } 355 356 static void hugetlb_free_pmd_range(struct mmu_gather *tlb, pud_t *pud, 357 unsigned long addr, unsigned long end, 358 unsigned long floor, unsigned long ceiling) 359 { 360 pmd_t *pmd; 361 unsigned long next; 362 unsigned long start; 363 364 start = addr; 365 do { 366 unsigned long more; 367 368 pmd = pmd_offset(pud, addr); 369 next = pmd_addr_end(addr, end); 370 if (!is_hugepd(__hugepd(pmd_val(*pmd)))) { 371 if (pmd_none_or_clear_bad(pmd)) 372 continue; 373 374 /* 375 * if it is not hugepd pointer, we should already find 376 * it cleared. 377 */ 378 WARN_ON(!IS_ENABLED(CONFIG_PPC_8xx)); 379 380 hugetlb_free_pte_range(tlb, pmd, addr, end, floor, ceiling); 381 382 continue; 383 } 384 /* 385 * Increment next by the size of the huge mapping since 386 * there may be more than one entry at this level for a 387 * single hugepage, but all of them point to 388 * the same kmem cache that holds the hugepte. 389 */ 390 more = addr + (1 << hugepd_shift(*(hugepd_t *)pmd)); 391 if (more > next) 392 next = more; 393 394 free_hugepd_range(tlb, (hugepd_t *)pmd, PMD_SHIFT, 395 addr, next, floor, ceiling); 396 } while (addr = next, addr != end); 397 398 start &= PUD_MASK; 399 if (start < floor) 400 return; 401 if (ceiling) { 402 ceiling &= PUD_MASK; 403 if (!ceiling) 404 return; 405 } 406 if (end - 1 > ceiling - 1) 407 return; 408 409 pmd = pmd_offset(pud, start); 410 pud_clear(pud); 411 pmd_free_tlb(tlb, pmd, start); 412 mm_dec_nr_pmds(tlb->mm); 413 } 414 415 static void hugetlb_free_pud_range(struct mmu_gather *tlb, p4d_t *p4d, 416 unsigned long addr, unsigned long end, 417 unsigned long floor, unsigned long ceiling) 418 { 419 pud_t *pud; 420 unsigned long next; 421 unsigned long start; 422 423 start = addr; 424 do { 425 pud = pud_offset(p4d, addr); 426 next = pud_addr_end(addr, end); 427 if (!is_hugepd(__hugepd(pud_val(*pud)))) { 428 if (pud_none_or_clear_bad(pud)) 429 continue; 430 hugetlb_free_pmd_range(tlb, pud, addr, next, floor, 431 ceiling); 432 } else { 433 unsigned long more; 434 /* 435 * Increment next by the size of the huge mapping since 436 * there may be more than one entry at this level for a 437 * single hugepage, but all of them point to 438 * the same kmem cache that holds the hugepte. 439 */ 440 more = addr + (1 << hugepd_shift(*(hugepd_t *)pud)); 441 if (more > next) 442 next = more; 443 444 free_hugepd_range(tlb, (hugepd_t *)pud, PUD_SHIFT, 445 addr, next, floor, ceiling); 446 } 447 } while (addr = next, addr != end); 448 449 start &= PGDIR_MASK; 450 if (start < floor) 451 return; 452 if (ceiling) { 453 ceiling &= PGDIR_MASK; 454 if (!ceiling) 455 return; 456 } 457 if (end - 1 > ceiling - 1) 458 return; 459 460 pud = pud_offset(p4d, start); 461 p4d_clear(p4d); 462 pud_free_tlb(tlb, pud, start); 463 mm_dec_nr_puds(tlb->mm); 464 } 465 466 /* 467 * This function frees user-level page tables of a process. 468 */ 469 void hugetlb_free_pgd_range(struct mmu_gather *tlb, 470 unsigned long addr, unsigned long end, 471 unsigned long floor, unsigned long ceiling) 472 { 473 pgd_t *pgd; 474 p4d_t *p4d; 475 unsigned long next; 476 477 /* 478 * Because there are a number of different possible pagetable 479 * layouts for hugepage ranges, we limit knowledge of how 480 * things should be laid out to the allocation path 481 * (huge_pte_alloc(), above). Everything else works out the 482 * structure as it goes from information in the hugepd 483 * pointers. That means that we can't here use the 484 * optimization used in the normal page free_pgd_range(), of 485 * checking whether we're actually covering a large enough 486 * range to have to do anything at the top level of the walk 487 * instead of at the bottom. 488 * 489 * To make sense of this, you should probably go read the big 490 * block comment at the top of the normal free_pgd_range(), 491 * too. 492 */ 493 494 do { 495 next = pgd_addr_end(addr, end); 496 pgd = pgd_offset(tlb->mm, addr); 497 p4d = p4d_offset(pgd, addr); 498 if (!is_hugepd(__hugepd(pgd_val(*pgd)))) { 499 if (p4d_none_or_clear_bad(p4d)) 500 continue; 501 hugetlb_free_pud_range(tlb, p4d, addr, next, floor, ceiling); 502 } else { 503 unsigned long more; 504 /* 505 * Increment next by the size of the huge mapping since 506 * there may be more than one entry at the pgd level 507 * for a single hugepage, but all of them point to the 508 * same kmem cache that holds the hugepte. 509 */ 510 more = addr + (1 << hugepd_shift(*(hugepd_t *)pgd)); 511 if (more > next) 512 next = more; 513 514 free_hugepd_range(tlb, (hugepd_t *)p4d, PGDIR_SHIFT, 515 addr, next, floor, ceiling); 516 } 517 } while (addr = next, addr != end); 518 } 519 520 struct page *follow_huge_pd(struct vm_area_struct *vma, 521 unsigned long address, hugepd_t hpd, 522 int flags, int pdshift) 523 { 524 pte_t *ptep; 525 spinlock_t *ptl; 526 struct page *page = NULL; 527 unsigned long mask; 528 int shift = hugepd_shift(hpd); 529 struct mm_struct *mm = vma->vm_mm; 530 531 retry: 532 /* 533 * hugepage directory entries are protected by mm->page_table_lock 534 * Use this instead of huge_pte_lockptr 535 */ 536 ptl = &mm->page_table_lock; 537 spin_lock(ptl); 538 539 ptep = hugepte_offset(hpd, address, pdshift); 540 if (pte_present(*ptep)) { 541 mask = (1UL << shift) - 1; 542 page = pte_page(*ptep); 543 page += ((address & mask) >> PAGE_SHIFT); 544 if (flags & FOLL_GET) 545 get_page(page); 546 } else { 547 if (is_hugetlb_entry_migration(*ptep)) { 548 spin_unlock(ptl); 549 __migration_entry_wait(mm, ptep, ptl); 550 goto retry; 551 } 552 } 553 spin_unlock(ptl); 554 return page; 555 } 556 557 #ifdef CONFIG_PPC_MM_SLICES 558 unsigned long hugetlb_get_unmapped_area(struct file *file, unsigned long addr, 559 unsigned long len, unsigned long pgoff, 560 unsigned long flags) 561 { 562 struct hstate *hstate = hstate_file(file); 563 int mmu_psize = shift_to_mmu_psize(huge_page_shift(hstate)); 564 565 #ifdef CONFIG_PPC_RADIX_MMU 566 if (radix_enabled()) 567 return radix__hugetlb_get_unmapped_area(file, addr, len, 568 pgoff, flags); 569 #endif 570 return slice_get_unmapped_area(addr, len, flags, mmu_psize, 1); 571 } 572 #endif 573 574 unsigned long vma_mmu_pagesize(struct vm_area_struct *vma) 575 { 576 /* With radix we don't use slice, so derive it from vma*/ 577 if (IS_ENABLED(CONFIG_PPC_MM_SLICES) && !radix_enabled()) { 578 unsigned int psize = get_slice_psize(vma->vm_mm, vma->vm_start); 579 580 return 1UL << mmu_psize_to_shift(psize); 581 } 582 return vma_kernel_pagesize(vma); 583 } 584 585 bool __init arch_hugetlb_valid_size(unsigned long size) 586 { 587 int shift = __ffs(size); 588 int mmu_psize; 589 590 /* Check that it is a page size supported by the hardware and 591 * that it fits within pagetable and slice limits. */ 592 if (size <= PAGE_SIZE || !is_power_of_2(size)) 593 return false; 594 595 mmu_psize = check_and_get_huge_psize(shift); 596 if (mmu_psize < 0) 597 return false; 598 599 BUG_ON(mmu_psize_defs[mmu_psize].shift != shift); 600 601 return true; 602 } 603 604 static int __init add_huge_page_size(unsigned long long size) 605 { 606 int shift = __ffs(size); 607 608 if (!arch_hugetlb_valid_size((unsigned long)size)) 609 return -EINVAL; 610 611 hugetlb_add_hstate(shift - PAGE_SHIFT); 612 return 0; 613 } 614 615 static int __init hugetlbpage_init(void) 616 { 617 bool configured = false; 618 int psize; 619 620 if (hugetlb_disabled) { 621 pr_info("HugeTLB support is disabled!\n"); 622 return 0; 623 } 624 625 if (IS_ENABLED(CONFIG_PPC_BOOK3S_64) && !radix_enabled() && 626 !mmu_has_feature(MMU_FTR_16M_PAGE)) 627 return -ENODEV; 628 629 for (psize = 0; psize < MMU_PAGE_COUNT; ++psize) { 630 unsigned shift; 631 unsigned pdshift; 632 633 if (!mmu_psize_defs[psize].shift) 634 continue; 635 636 shift = mmu_psize_to_shift(psize); 637 638 #ifdef CONFIG_PPC_BOOK3S_64 639 if (shift > PGDIR_SHIFT) 640 continue; 641 else if (shift > PUD_SHIFT) 642 pdshift = PGDIR_SHIFT; 643 else if (shift > PMD_SHIFT) 644 pdshift = PUD_SHIFT; 645 else 646 pdshift = PMD_SHIFT; 647 #else 648 if (shift < PUD_SHIFT) 649 pdshift = PMD_SHIFT; 650 else if (shift < PGDIR_SHIFT) 651 pdshift = PUD_SHIFT; 652 else 653 pdshift = PGDIR_SHIFT; 654 #endif 655 656 if (add_huge_page_size(1ULL << shift) < 0) 657 continue; 658 /* 659 * if we have pdshift and shift value same, we don't 660 * use pgt cache for hugepd. 661 */ 662 if (pdshift > shift) { 663 if (!IS_ENABLED(CONFIG_PPC_8xx)) 664 pgtable_cache_add(pdshift - shift); 665 } else if (IS_ENABLED(CONFIG_PPC_FSL_BOOK3E) || 666 IS_ENABLED(CONFIG_PPC_8xx)) { 667 pgtable_cache_add(PTE_T_ORDER); 668 } 669 670 configured = true; 671 } 672 673 if (configured) { 674 if (IS_ENABLED(CONFIG_HUGETLB_PAGE_SIZE_VARIABLE)) 675 hugetlbpage_init_default(); 676 } else 677 pr_info("Failed to initialize. Disabling HugeTLB"); 678 679 return 0; 680 } 681 682 arch_initcall(hugetlbpage_init); 683 684 void flush_dcache_icache_hugepage(struct page *page) 685 { 686 int i; 687 void *start; 688 689 BUG_ON(!PageCompound(page)); 690 691 for (i = 0; i < compound_nr(page); i++) { 692 if (!PageHighMem(page)) { 693 __flush_dcache_icache(page_address(page+i)); 694 } else { 695 start = kmap_atomic(page+i); 696 __flush_dcache_icache(start); 697 kunmap_atomic(start); 698 } 699 } 700 } 701 702 void __init gigantic_hugetlb_cma_reserve(void) 703 { 704 unsigned long order = 0; 705 706 if (radix_enabled()) 707 order = PUD_SHIFT - PAGE_SHIFT; 708 else if (!firmware_has_feature(FW_FEATURE_LPAR) && mmu_psize_defs[MMU_PAGE_16G].shift) 709 /* 710 * For pseries we do use ibm,expected#pages for reserving 16G pages. 711 */ 712 order = mmu_psize_to_shift(MMU_PAGE_16G) - PAGE_SHIFT; 713 714 if (order) { 715 VM_WARN_ON(order < MAX_ORDER); 716 hugetlb_cma_reserve(order); 717 } 718 } 719