1 /* 2 * This file contains ioremap and related functions for 64-bit machines. 3 * 4 * Derived from arch/ppc64/mm/init.c 5 * Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org) 6 * 7 * Modifications by Paul Mackerras (PowerMac) (paulus@samba.org) 8 * and Cort Dougan (PReP) (cort@cs.nmt.edu) 9 * Copyright (C) 1996 Paul Mackerras 10 * 11 * Derived from "arch/i386/mm/init.c" 12 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds 13 * 14 * Dave Engebretsen <engebret@us.ibm.com> 15 * Rework for PPC64 port. 16 * 17 * This program is free software; you can redistribute it and/or 18 * modify it under the terms of the GNU General Public License 19 * as published by the Free Software Foundation; either version 20 * 2 of the License, or (at your option) any later version. 21 * 22 */ 23 24 #include <linux/signal.h> 25 #include <linux/sched.h> 26 #include <linux/kernel.h> 27 #include <linux/errno.h> 28 #include <linux/string.h> 29 #include <linux/export.h> 30 #include <linux/types.h> 31 #include <linux/mman.h> 32 #include <linux/mm.h> 33 #include <linux/swap.h> 34 #include <linux/stddef.h> 35 #include <linux/vmalloc.h> 36 #include <linux/bootmem.h> 37 #include <linux/memblock.h> 38 #include <linux/slab.h> 39 40 #include <asm/pgalloc.h> 41 #include <asm/page.h> 42 #include <asm/prom.h> 43 #include <asm/io.h> 44 #include <asm/mmu_context.h> 45 #include <asm/pgtable.h> 46 #include <asm/mmu.h> 47 #include <asm/smp.h> 48 #include <asm/machdep.h> 49 #include <asm/tlb.h> 50 #include <asm/processor.h> 51 #include <asm/cputable.h> 52 #include <asm/sections.h> 53 #include <asm/firmware.h> 54 55 #include "mmu_decl.h" 56 57 /* Some sanity checking */ 58 #if TASK_SIZE_USER64 > PGTABLE_RANGE 59 #error TASK_SIZE_USER64 exceeds pagetable range 60 #endif 61 62 #ifdef CONFIG_PPC_STD_MMU_64 63 #if TASK_SIZE_USER64 > (1UL << (ESID_BITS + SID_SHIFT)) 64 #error TASK_SIZE_USER64 exceeds user VSID range 65 #endif 66 #endif 67 68 unsigned long ioremap_bot = IOREMAP_BASE; 69 70 #ifdef CONFIG_PPC_MMU_NOHASH 71 static void *early_alloc_pgtable(unsigned long size) 72 { 73 void *pt; 74 75 if (init_bootmem_done) 76 pt = __alloc_bootmem(size, size, __pa(MAX_DMA_ADDRESS)); 77 else 78 pt = __va(memblock_alloc_base(size, size, 79 __pa(MAX_DMA_ADDRESS))); 80 memset(pt, 0, size); 81 82 return pt; 83 } 84 #endif /* CONFIG_PPC_MMU_NOHASH */ 85 86 /* 87 * map_kernel_page currently only called by __ioremap 88 * map_kernel_page adds an entry to the ioremap page table 89 * and adds an entry to the HPT, possibly bolting it 90 */ 91 int map_kernel_page(unsigned long ea, unsigned long pa, int flags) 92 { 93 pgd_t *pgdp; 94 pud_t *pudp; 95 pmd_t *pmdp; 96 pte_t *ptep; 97 98 if (slab_is_available()) { 99 pgdp = pgd_offset_k(ea); 100 pudp = pud_alloc(&init_mm, pgdp, ea); 101 if (!pudp) 102 return -ENOMEM; 103 pmdp = pmd_alloc(&init_mm, pudp, ea); 104 if (!pmdp) 105 return -ENOMEM; 106 ptep = pte_alloc_kernel(pmdp, ea); 107 if (!ptep) 108 return -ENOMEM; 109 set_pte_at(&init_mm, ea, ptep, pfn_pte(pa >> PAGE_SHIFT, 110 __pgprot(flags))); 111 } else { 112 #ifdef CONFIG_PPC_MMU_NOHASH 113 /* Warning ! This will blow up if bootmem is not initialized 114 * which our ppc64 code is keen to do that, we'll need to 115 * fix it and/or be more careful 116 */ 117 pgdp = pgd_offset_k(ea); 118 #ifdef PUD_TABLE_SIZE 119 if (pgd_none(*pgdp)) { 120 pudp = early_alloc_pgtable(PUD_TABLE_SIZE); 121 BUG_ON(pudp == NULL); 122 pgd_populate(&init_mm, pgdp, pudp); 123 } 124 #endif /* PUD_TABLE_SIZE */ 125 pudp = pud_offset(pgdp, ea); 126 if (pud_none(*pudp)) { 127 pmdp = early_alloc_pgtable(PMD_TABLE_SIZE); 128 BUG_ON(pmdp == NULL); 129 pud_populate(&init_mm, pudp, pmdp); 130 } 131 pmdp = pmd_offset(pudp, ea); 132 if (!pmd_present(*pmdp)) { 133 ptep = early_alloc_pgtable(PAGE_SIZE); 134 BUG_ON(ptep == NULL); 135 pmd_populate_kernel(&init_mm, pmdp, ptep); 136 } 137 ptep = pte_offset_kernel(pmdp, ea); 138 set_pte_at(&init_mm, ea, ptep, pfn_pte(pa >> PAGE_SHIFT, 139 __pgprot(flags))); 140 #else /* CONFIG_PPC_MMU_NOHASH */ 141 /* 142 * If the mm subsystem is not fully up, we cannot create a 143 * linux page table entry for this mapping. Simply bolt an 144 * entry in the hardware page table. 145 * 146 */ 147 if (htab_bolt_mapping(ea, ea + PAGE_SIZE, pa, flags, 148 mmu_io_psize, mmu_kernel_ssize)) { 149 printk(KERN_ERR "Failed to do bolted mapping IO " 150 "memory at %016lx !\n", pa); 151 return -ENOMEM; 152 } 153 #endif /* !CONFIG_PPC_MMU_NOHASH */ 154 } 155 156 #ifdef CONFIG_PPC_BOOK3E_64 157 /* 158 * With hardware tablewalk, a sync is needed to ensure that 159 * subsequent accesses see the PTE we just wrote. Unlike userspace 160 * mappings, we can't tolerate spurious faults, so make sure 161 * the new PTE will be seen the first time. 162 */ 163 mb(); 164 #else 165 smp_wmb(); 166 #endif 167 return 0; 168 } 169 170 171 /** 172 * __ioremap_at - Low level function to establish the page tables 173 * for an IO mapping 174 */ 175 void __iomem * __ioremap_at(phys_addr_t pa, void *ea, unsigned long size, 176 unsigned long flags) 177 { 178 unsigned long i; 179 180 /* Make sure we have the base flags */ 181 if ((flags & _PAGE_PRESENT) == 0) 182 flags |= pgprot_val(PAGE_KERNEL); 183 184 /* Non-cacheable page cannot be coherent */ 185 if (flags & _PAGE_NO_CACHE) 186 flags &= ~_PAGE_COHERENT; 187 188 /* We don't support the 4K PFN hack with ioremap */ 189 if (flags & _PAGE_4K_PFN) 190 return NULL; 191 192 WARN_ON(pa & ~PAGE_MASK); 193 WARN_ON(((unsigned long)ea) & ~PAGE_MASK); 194 WARN_ON(size & ~PAGE_MASK); 195 196 for (i = 0; i < size; i += PAGE_SIZE) 197 if (map_kernel_page((unsigned long)ea+i, pa+i, flags)) 198 return NULL; 199 200 return (void __iomem *)ea; 201 } 202 203 /** 204 * __iounmap_from - Low level function to tear down the page tables 205 * for an IO mapping. This is used for mappings that 206 * are manipulated manually, like partial unmapping of 207 * PCI IOs or ISA space. 208 */ 209 void __iounmap_at(void *ea, unsigned long size) 210 { 211 WARN_ON(((unsigned long)ea) & ~PAGE_MASK); 212 WARN_ON(size & ~PAGE_MASK); 213 214 unmap_kernel_range((unsigned long)ea, size); 215 } 216 217 void __iomem * __ioremap_caller(phys_addr_t addr, unsigned long size, 218 unsigned long flags, void *caller) 219 { 220 phys_addr_t paligned; 221 void __iomem *ret; 222 223 /* 224 * Choose an address to map it to. 225 * Once the imalloc system is running, we use it. 226 * Before that, we map using addresses going 227 * up from ioremap_bot. imalloc will use 228 * the addresses from ioremap_bot through 229 * IMALLOC_END 230 * 231 */ 232 paligned = addr & PAGE_MASK; 233 size = PAGE_ALIGN(addr + size) - paligned; 234 235 if ((size == 0) || (paligned == 0)) 236 return NULL; 237 238 if (mem_init_done) { 239 struct vm_struct *area; 240 241 area = __get_vm_area_caller(size, VM_IOREMAP, 242 ioremap_bot, IOREMAP_END, 243 caller); 244 if (area == NULL) 245 return NULL; 246 247 area->phys_addr = paligned; 248 ret = __ioremap_at(paligned, area->addr, size, flags); 249 if (!ret) 250 vunmap(area->addr); 251 } else { 252 ret = __ioremap_at(paligned, (void *)ioremap_bot, size, flags); 253 if (ret) 254 ioremap_bot += size; 255 } 256 257 if (ret) 258 ret += addr & ~PAGE_MASK; 259 return ret; 260 } 261 262 void __iomem * __ioremap(phys_addr_t addr, unsigned long size, 263 unsigned long flags) 264 { 265 return __ioremap_caller(addr, size, flags, __builtin_return_address(0)); 266 } 267 268 void __iomem * ioremap(phys_addr_t addr, unsigned long size) 269 { 270 unsigned long flags = _PAGE_NO_CACHE | _PAGE_GUARDED; 271 void *caller = __builtin_return_address(0); 272 273 if (ppc_md.ioremap) 274 return ppc_md.ioremap(addr, size, flags, caller); 275 return __ioremap_caller(addr, size, flags, caller); 276 } 277 278 void __iomem * ioremap_wc(phys_addr_t addr, unsigned long size) 279 { 280 unsigned long flags = _PAGE_NO_CACHE; 281 void *caller = __builtin_return_address(0); 282 283 if (ppc_md.ioremap) 284 return ppc_md.ioremap(addr, size, flags, caller); 285 return __ioremap_caller(addr, size, flags, caller); 286 } 287 288 void __iomem * ioremap_prot(phys_addr_t addr, unsigned long size, 289 unsigned long flags) 290 { 291 void *caller = __builtin_return_address(0); 292 293 /* writeable implies dirty for kernel addresses */ 294 if (flags & _PAGE_RW) 295 flags |= _PAGE_DIRTY; 296 297 /* we don't want to let _PAGE_USER and _PAGE_EXEC leak out */ 298 flags &= ~(_PAGE_USER | _PAGE_EXEC); 299 300 #ifdef _PAGE_BAP_SR 301 /* _PAGE_USER contains _PAGE_BAP_SR on BookE using the new PTE format 302 * which means that we just cleared supervisor access... oops ;-) This 303 * restores it 304 */ 305 flags |= _PAGE_BAP_SR; 306 #endif 307 308 if (ppc_md.ioremap) 309 return ppc_md.ioremap(addr, size, flags, caller); 310 return __ioremap_caller(addr, size, flags, caller); 311 } 312 313 314 /* 315 * Unmap an IO region and remove it from imalloc'd list. 316 * Access to IO memory should be serialized by driver. 317 */ 318 void __iounmap(volatile void __iomem *token) 319 { 320 void *addr; 321 322 if (!mem_init_done) 323 return; 324 325 addr = (void *) ((unsigned long __force) 326 PCI_FIX_ADDR(token) & PAGE_MASK); 327 if ((unsigned long)addr < ioremap_bot) { 328 printk(KERN_WARNING "Attempt to iounmap early bolted mapping" 329 " at 0x%p\n", addr); 330 return; 331 } 332 vunmap(addr); 333 } 334 335 void iounmap(volatile void __iomem *token) 336 { 337 if (ppc_md.iounmap) 338 ppc_md.iounmap(token); 339 else 340 __iounmap(token); 341 } 342 343 EXPORT_SYMBOL(ioremap); 344 EXPORT_SYMBOL(ioremap_wc); 345 EXPORT_SYMBOL(ioremap_prot); 346 EXPORT_SYMBOL(__ioremap); 347 EXPORT_SYMBOL(__ioremap_at); 348 EXPORT_SYMBOL(iounmap); 349 EXPORT_SYMBOL(__iounmap); 350 EXPORT_SYMBOL(__iounmap_at); 351 352 /* 353 * For hugepage we have pfn in the pmd, we use PTE_RPN_SHIFT bits for flags 354 * For PTE page, we have a PTE_FRAG_SIZE (4K) aligned virtual address. 355 */ 356 struct page *pmd_page(pmd_t pmd) 357 { 358 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 359 if (pmd_trans_huge(pmd)) 360 return pfn_to_page(pmd_pfn(pmd)); 361 #endif 362 return virt_to_page(pmd_page_vaddr(pmd)); 363 } 364 365 #ifdef CONFIG_PPC_64K_PAGES 366 static pte_t *get_from_cache(struct mm_struct *mm) 367 { 368 void *pte_frag, *ret; 369 370 spin_lock(&mm->page_table_lock); 371 ret = mm->context.pte_frag; 372 if (ret) { 373 pte_frag = ret + PTE_FRAG_SIZE; 374 /* 375 * If we have taken up all the fragments mark PTE page NULL 376 */ 377 if (((unsigned long)pte_frag & ~PAGE_MASK) == 0) 378 pte_frag = NULL; 379 mm->context.pte_frag = pte_frag; 380 } 381 spin_unlock(&mm->page_table_lock); 382 return (pte_t *)ret; 383 } 384 385 static pte_t *__alloc_for_cache(struct mm_struct *mm, int kernel) 386 { 387 void *ret = NULL; 388 struct page *page = alloc_page(GFP_KERNEL | __GFP_NOTRACK | 389 __GFP_REPEAT | __GFP_ZERO); 390 if (!page) 391 return NULL; 392 if (!kernel && !pgtable_page_ctor(page)) { 393 __free_page(page); 394 return NULL; 395 } 396 397 ret = page_address(page); 398 spin_lock(&mm->page_table_lock); 399 /* 400 * If we find pgtable_page set, we return 401 * the allocated page with single fragement 402 * count. 403 */ 404 if (likely(!mm->context.pte_frag)) { 405 atomic_set(&page->_count, PTE_FRAG_NR); 406 mm->context.pte_frag = ret + PTE_FRAG_SIZE; 407 } 408 spin_unlock(&mm->page_table_lock); 409 410 return (pte_t *)ret; 411 } 412 413 pte_t *page_table_alloc(struct mm_struct *mm, unsigned long vmaddr, int kernel) 414 { 415 pte_t *pte; 416 417 pte = get_from_cache(mm); 418 if (pte) 419 return pte; 420 421 return __alloc_for_cache(mm, kernel); 422 } 423 424 void page_table_free(struct mm_struct *mm, unsigned long *table, int kernel) 425 { 426 struct page *page = virt_to_page(table); 427 if (put_page_testzero(page)) { 428 if (!kernel) 429 pgtable_page_dtor(page); 430 free_hot_cold_page(page, 0); 431 } 432 } 433 434 #ifdef CONFIG_SMP 435 static void page_table_free_rcu(void *table) 436 { 437 struct page *page = virt_to_page(table); 438 if (put_page_testzero(page)) { 439 pgtable_page_dtor(page); 440 free_hot_cold_page(page, 0); 441 } 442 } 443 444 void pgtable_free_tlb(struct mmu_gather *tlb, void *table, int shift) 445 { 446 unsigned long pgf = (unsigned long)table; 447 448 BUG_ON(shift > MAX_PGTABLE_INDEX_SIZE); 449 pgf |= shift; 450 tlb_remove_table(tlb, (void *)pgf); 451 } 452 453 void __tlb_remove_table(void *_table) 454 { 455 void *table = (void *)((unsigned long)_table & ~MAX_PGTABLE_INDEX_SIZE); 456 unsigned shift = (unsigned long)_table & MAX_PGTABLE_INDEX_SIZE; 457 458 if (!shift) 459 /* PTE page needs special handling */ 460 page_table_free_rcu(table); 461 else { 462 BUG_ON(shift > MAX_PGTABLE_INDEX_SIZE); 463 kmem_cache_free(PGT_CACHE(shift), table); 464 } 465 } 466 #else 467 void pgtable_free_tlb(struct mmu_gather *tlb, void *table, int shift) 468 { 469 if (!shift) { 470 /* PTE page needs special handling */ 471 struct page *page = virt_to_page(table); 472 if (put_page_testzero(page)) { 473 pgtable_page_dtor(page); 474 free_hot_cold_page(page, 0); 475 } 476 } else { 477 BUG_ON(shift > MAX_PGTABLE_INDEX_SIZE); 478 kmem_cache_free(PGT_CACHE(shift), table); 479 } 480 } 481 #endif 482 #endif /* CONFIG_PPC_64K_PAGES */ 483 484 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 485 486 /* 487 * This is called when relaxing access to a hugepage. It's also called in the page 488 * fault path when we don't hit any of the major fault cases, ie, a minor 489 * update of _PAGE_ACCESSED, _PAGE_DIRTY, etc... The generic code will have 490 * handled those two for us, we additionally deal with missing execute 491 * permission here on some processors 492 */ 493 int pmdp_set_access_flags(struct vm_area_struct *vma, unsigned long address, 494 pmd_t *pmdp, pmd_t entry, int dirty) 495 { 496 int changed; 497 #ifdef CONFIG_DEBUG_VM 498 WARN_ON(!pmd_trans_huge(*pmdp)); 499 assert_spin_locked(&vma->vm_mm->page_table_lock); 500 #endif 501 changed = !pmd_same(*(pmdp), entry); 502 if (changed) { 503 __ptep_set_access_flags(pmdp_ptep(pmdp), pmd_pte(entry)); 504 /* 505 * Since we are not supporting SW TLB systems, we don't 506 * have any thing similar to flush_tlb_page_nohash() 507 */ 508 } 509 return changed; 510 } 511 512 unsigned long pmd_hugepage_update(struct mm_struct *mm, unsigned long addr, 513 pmd_t *pmdp, unsigned long clr, 514 unsigned long set) 515 { 516 517 unsigned long old, tmp; 518 519 #ifdef CONFIG_DEBUG_VM 520 WARN_ON(!pmd_trans_huge(*pmdp)); 521 assert_spin_locked(&mm->page_table_lock); 522 #endif 523 524 #ifdef PTE_ATOMIC_UPDATES 525 __asm__ __volatile__( 526 "1: ldarx %0,0,%3\n\ 527 andi. %1,%0,%6\n\ 528 bne- 1b \n\ 529 andc %1,%0,%4 \n\ 530 or %1,%1,%7\n\ 531 stdcx. %1,0,%3 \n\ 532 bne- 1b" 533 : "=&r" (old), "=&r" (tmp), "=m" (*pmdp) 534 : "r" (pmdp), "r" (clr), "m" (*pmdp), "i" (_PAGE_BUSY), "r" (set) 535 : "cc" ); 536 #else 537 old = pmd_val(*pmdp); 538 *pmdp = __pmd((old & ~clr) | set); 539 #endif 540 if (old & _PAGE_HASHPTE) 541 hpte_do_hugepage_flush(mm, addr, pmdp); 542 return old; 543 } 544 545 pmd_t pmdp_clear_flush(struct vm_area_struct *vma, unsigned long address, 546 pmd_t *pmdp) 547 { 548 pmd_t pmd; 549 550 VM_BUG_ON(address & ~HPAGE_PMD_MASK); 551 if (pmd_trans_huge(*pmdp)) { 552 pmd = pmdp_get_and_clear(vma->vm_mm, address, pmdp); 553 } else { 554 /* 555 * khugepaged calls this for normal pmd 556 */ 557 pmd = *pmdp; 558 pmd_clear(pmdp); 559 /* 560 * Wait for all pending hash_page to finish. This is needed 561 * in case of subpage collapse. When we collapse normal pages 562 * to hugepage, we first clear the pmd, then invalidate all 563 * the PTE entries. The assumption here is that any low level 564 * page fault will see a none pmd and take the slow path that 565 * will wait on mmap_sem. But we could very well be in a 566 * hash_page with local ptep pointer value. Such a hash page 567 * can result in adding new HPTE entries for normal subpages. 568 * That means we could be modifying the page content as we 569 * copy them to a huge page. So wait for parallel hash_page 570 * to finish before invalidating HPTE entries. We can do this 571 * by sending an IPI to all the cpus and executing a dummy 572 * function there. 573 */ 574 kick_all_cpus_sync(); 575 /* 576 * Now invalidate the hpte entries in the range 577 * covered by pmd. This make sure we take a 578 * fault and will find the pmd as none, which will 579 * result in a major fault which takes mmap_sem and 580 * hence wait for collapse to complete. Without this 581 * the __collapse_huge_page_copy can result in copying 582 * the old content. 583 */ 584 flush_tlb_pmd_range(vma->vm_mm, &pmd, address); 585 } 586 return pmd; 587 } 588 589 int pmdp_test_and_clear_young(struct vm_area_struct *vma, 590 unsigned long address, pmd_t *pmdp) 591 { 592 return __pmdp_test_and_clear_young(vma->vm_mm, address, pmdp); 593 } 594 595 /* 596 * We currently remove entries from the hashtable regardless of whether 597 * the entry was young or dirty. The generic routines only flush if the 598 * entry was young or dirty which is not good enough. 599 * 600 * We should be more intelligent about this but for the moment we override 601 * these functions and force a tlb flush unconditionally 602 */ 603 int pmdp_clear_flush_young(struct vm_area_struct *vma, 604 unsigned long address, pmd_t *pmdp) 605 { 606 return __pmdp_test_and_clear_young(vma->vm_mm, address, pmdp); 607 } 608 609 /* 610 * We mark the pmd splitting and invalidate all the hpte 611 * entries for this hugepage. 612 */ 613 void pmdp_splitting_flush(struct vm_area_struct *vma, 614 unsigned long address, pmd_t *pmdp) 615 { 616 unsigned long old, tmp; 617 618 VM_BUG_ON(address & ~HPAGE_PMD_MASK); 619 620 #ifdef CONFIG_DEBUG_VM 621 WARN_ON(!pmd_trans_huge(*pmdp)); 622 assert_spin_locked(&vma->vm_mm->page_table_lock); 623 #endif 624 625 #ifdef PTE_ATOMIC_UPDATES 626 627 __asm__ __volatile__( 628 "1: ldarx %0,0,%3\n\ 629 andi. %1,%0,%6\n\ 630 bne- 1b \n\ 631 ori %1,%0,%4 \n\ 632 stdcx. %1,0,%3 \n\ 633 bne- 1b" 634 : "=&r" (old), "=&r" (tmp), "=m" (*pmdp) 635 : "r" (pmdp), "i" (_PAGE_SPLITTING), "m" (*pmdp), "i" (_PAGE_BUSY) 636 : "cc" ); 637 #else 638 old = pmd_val(*pmdp); 639 *pmdp = __pmd(old | _PAGE_SPLITTING); 640 #endif 641 /* 642 * If we didn't had the splitting flag set, go and flush the 643 * HPTE entries. 644 */ 645 if (!(old & _PAGE_SPLITTING)) { 646 /* We need to flush the hpte */ 647 if (old & _PAGE_HASHPTE) 648 hpte_do_hugepage_flush(vma->vm_mm, address, pmdp); 649 } 650 } 651 652 /* 653 * We want to put the pgtable in pmd and use pgtable for tracking 654 * the base page size hptes 655 */ 656 void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp, 657 pgtable_t pgtable) 658 { 659 pgtable_t *pgtable_slot; 660 assert_spin_locked(&mm->page_table_lock); 661 /* 662 * we store the pgtable in the second half of PMD 663 */ 664 pgtable_slot = (pgtable_t *)pmdp + PTRS_PER_PMD; 665 *pgtable_slot = pgtable; 666 /* 667 * expose the deposited pgtable to other cpus. 668 * before we set the hugepage PTE at pmd level 669 * hash fault code looks at the deposted pgtable 670 * to store hash index values. 671 */ 672 smp_wmb(); 673 } 674 675 pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp) 676 { 677 pgtable_t pgtable; 678 pgtable_t *pgtable_slot; 679 680 assert_spin_locked(&mm->page_table_lock); 681 pgtable_slot = (pgtable_t *)pmdp + PTRS_PER_PMD; 682 pgtable = *pgtable_slot; 683 /* 684 * Once we withdraw, mark the entry NULL. 685 */ 686 *pgtable_slot = NULL; 687 /* 688 * We store HPTE information in the deposited PTE fragment. 689 * zero out the content on withdraw. 690 */ 691 memset(pgtable, 0, PTE_FRAG_SIZE); 692 return pgtable; 693 } 694 695 /* 696 * set a new huge pmd. We should not be called for updating 697 * an existing pmd entry. That should go via pmd_hugepage_update. 698 */ 699 void set_pmd_at(struct mm_struct *mm, unsigned long addr, 700 pmd_t *pmdp, pmd_t pmd) 701 { 702 #ifdef CONFIG_DEBUG_VM 703 WARN_ON(pmd_val(*pmdp) & _PAGE_PRESENT); 704 assert_spin_locked(&mm->page_table_lock); 705 WARN_ON(!pmd_trans_huge(pmd)); 706 #endif 707 return set_pte_at(mm, addr, pmdp_ptep(pmdp), pmd_pte(pmd)); 708 } 709 710 void pmdp_invalidate(struct vm_area_struct *vma, unsigned long address, 711 pmd_t *pmdp) 712 { 713 pmd_hugepage_update(vma->vm_mm, address, pmdp, _PAGE_PRESENT, 0); 714 } 715 716 /* 717 * A linux hugepage PMD was changed and the corresponding hash table entries 718 * neesd to be flushed. 719 */ 720 void hpte_do_hugepage_flush(struct mm_struct *mm, unsigned long addr, 721 pmd_t *pmdp) 722 { 723 int ssize, i; 724 unsigned long s_addr; 725 int max_hpte_count; 726 unsigned int psize, valid; 727 unsigned char *hpte_slot_array; 728 unsigned long hidx, vpn, vsid, hash, shift, slot; 729 730 /* 731 * Flush all the hptes mapping this hugepage 732 */ 733 s_addr = addr & HPAGE_PMD_MASK; 734 hpte_slot_array = get_hpte_slot_array(pmdp); 735 /* 736 * IF we try to do a HUGE PTE update after a withdraw is done. 737 * we will find the below NULL. This happens when we do 738 * split_huge_page_pmd 739 */ 740 if (!hpte_slot_array) 741 return; 742 743 /* get the base page size */ 744 psize = get_slice_psize(mm, s_addr); 745 746 if (ppc_md.hugepage_invalidate) 747 return ppc_md.hugepage_invalidate(mm, hpte_slot_array, 748 s_addr, psize); 749 /* 750 * No bluk hpte removal support, invalidate each entry 751 */ 752 shift = mmu_psize_defs[psize].shift; 753 max_hpte_count = HPAGE_PMD_SIZE >> shift; 754 for (i = 0; i < max_hpte_count; i++) { 755 /* 756 * 8 bits per each hpte entries 757 * 000| [ secondary group (one bit) | hidx (3 bits) | valid bit] 758 */ 759 valid = hpte_valid(hpte_slot_array, i); 760 if (!valid) 761 continue; 762 hidx = hpte_hash_index(hpte_slot_array, i); 763 764 /* get the vpn */ 765 addr = s_addr + (i * (1ul << shift)); 766 if (!is_kernel_addr(addr)) { 767 ssize = user_segment_size(addr); 768 vsid = get_vsid(mm->context.id, addr, ssize); 769 WARN_ON(vsid == 0); 770 } else { 771 vsid = get_kernel_vsid(addr, mmu_kernel_ssize); 772 ssize = mmu_kernel_ssize; 773 } 774 775 vpn = hpt_vpn(addr, vsid, ssize); 776 hash = hpt_hash(vpn, shift, ssize); 777 if (hidx & _PTEIDX_SECONDARY) 778 hash = ~hash; 779 780 slot = (hash & htab_hash_mask) * HPTES_PER_GROUP; 781 slot += hidx & _PTEIDX_GROUP_IX; 782 ppc_md.hpte_invalidate(slot, vpn, psize, 783 MMU_PAGE_16M, ssize, 0); 784 } 785 } 786 787 static pmd_t pmd_set_protbits(pmd_t pmd, pgprot_t pgprot) 788 { 789 pmd_val(pmd) |= pgprot_val(pgprot); 790 return pmd; 791 } 792 793 pmd_t pfn_pmd(unsigned long pfn, pgprot_t pgprot) 794 { 795 pmd_t pmd; 796 /* 797 * For a valid pte, we would have _PAGE_PRESENT or _PAGE_FILE always 798 * set. We use this to check THP page at pmd level. 799 * leaf pte for huge page, bottom two bits != 00 800 */ 801 pmd_val(pmd) = pfn << PTE_RPN_SHIFT; 802 pmd_val(pmd) |= _PAGE_THP_HUGE; 803 pmd = pmd_set_protbits(pmd, pgprot); 804 return pmd; 805 } 806 807 pmd_t mk_pmd(struct page *page, pgprot_t pgprot) 808 { 809 return pfn_pmd(page_to_pfn(page), pgprot); 810 } 811 812 pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot) 813 { 814 815 pmd_val(pmd) &= _HPAGE_CHG_MASK; 816 pmd = pmd_set_protbits(pmd, newprot); 817 return pmd; 818 } 819 820 /* 821 * This is called at the end of handling a user page fault, when the 822 * fault has been handled by updating a HUGE PMD entry in the linux page tables. 823 * We use it to preload an HPTE into the hash table corresponding to 824 * the updated linux HUGE PMD entry. 825 */ 826 void update_mmu_cache_pmd(struct vm_area_struct *vma, unsigned long addr, 827 pmd_t *pmd) 828 { 829 return; 830 } 831 832 pmd_t pmdp_get_and_clear(struct mm_struct *mm, 833 unsigned long addr, pmd_t *pmdp) 834 { 835 pmd_t old_pmd; 836 pgtable_t pgtable; 837 unsigned long old; 838 pgtable_t *pgtable_slot; 839 840 old = pmd_hugepage_update(mm, addr, pmdp, ~0UL, 0); 841 old_pmd = __pmd(old); 842 /* 843 * We have pmd == none and we are holding page_table_lock. 844 * So we can safely go and clear the pgtable hash 845 * index info. 846 */ 847 pgtable_slot = (pgtable_t *)pmdp + PTRS_PER_PMD; 848 pgtable = *pgtable_slot; 849 /* 850 * Let's zero out old valid and hash index details 851 * hash fault look at them. 852 */ 853 memset(pgtable, 0, PTE_FRAG_SIZE); 854 return old_pmd; 855 } 856 857 int has_transparent_hugepage(void) 858 { 859 if (!mmu_has_feature(MMU_FTR_16M_PAGE)) 860 return 0; 861 /* 862 * We support THP only if PMD_SIZE is 16MB. 863 */ 864 if (mmu_psize_defs[MMU_PAGE_16M].shift != PMD_SHIFT) 865 return 0; 866 /* 867 * We need to make sure that we support 16MB hugepage in a segement 868 * with base page size 64K or 4K. We only enable THP with a PAGE_SIZE 869 * of 64K. 870 */ 871 /* 872 * If we have 64K HPTE, we will be using that by default 873 */ 874 if (mmu_psize_defs[MMU_PAGE_64K].shift && 875 (mmu_psize_defs[MMU_PAGE_64K].penc[MMU_PAGE_16M] == -1)) 876 return 0; 877 /* 878 * Ok we only have 4K HPTE 879 */ 880 if (mmu_psize_defs[MMU_PAGE_4K].penc[MMU_PAGE_16M] == -1) 881 return 0; 882 883 return 1; 884 } 885 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 886