7  * Based on the IA-32 version:
26 #include <asm/pte-walk.h>
33 #define PTE_T_ORDER	(__builtin_ffs(sizeof(pte_basic_t)) - \
42 	return __find_linux_pte(mm->pgd, addr, NULL, NULL);  in huge_pte_offset()
56 		num_hugepd = 1 << (pshift - pdshift);  in __hugepte_alloc()
58 		cachep = PGT_CACHE(pdshift - pshift);  in __hugepte_alloc()
64 		return -ENOMEM;  in __hugepte_alloc()
73 		return -ENOMEM;  in __hugepte_alloc()
84 	 * We have multiple higher-level entries that point to the same  in __hugepte_alloc()
87 	 * right higher-level entry without knowing if it's a hugepage or not.  in __hugepte_alloc()
96 		for (i = i - 1 ; i >= 0; i--, hpdp--)  in __hugepte_alloc()
107  * At this point we do the placement change only for BOOK3S 64. This would
122 	addr &= ~(sz-1);  in huge_pte_alloc()
134 		ptl = &mm->page_table_lock;  in huge_pte_alloc()
162 		ptl = &mm->page_table_lock;  in huge_pte_alloc()
217 		number_of_pages--;  in pseries_add_gpage()
227 	m = phys_to_virt(gpage_freearray[--nr_gpages]);  in pseries_alloc_bootmem_huge_page()
229 	list_add(&m->list, &huge_boot_pages);  in pseries_alloc_bootmem_huge_page()
230 	m->hstate = hstate;  in pseries_alloc_bootmem_huge_page()
253 	((PAGE_SIZE - sizeof(struct hugepd_freelist)) / sizeof(pte_t))
269 	for (i = 0; i < batch->index; i++)  in hugepd_free_rcu_callback()
270 		kmem_cache_free(PGT_CACHE(PTE_T_ORDER), batch->ptes[i]);  in hugepd_free_rcu_callback()
281 	if (atomic_read(&tlb->mm->mm_users) < 2 ||  in hugepd_free()
282 	    mm_is_thread_local(tlb->mm)) {  in hugepd_free()
290 		(*batchp)->index = 0;  in hugepd_free()
293 	(*batchp)->ptes[(*batchp)->index++] = hugepte;  in hugepd_free()
294 	if ((*batchp)->index == HUGEPD_FREELIST_SIZE) {  in hugepd_free()
295 		call_rcu(&(*batchp)->rcu, hugepd_free_rcu_callback);  in hugepd_free()
305 static bool range_is_outside_limits(unsigned long start, unsigned long end,  in range_is_outside_limits()  argument
316 	return end - 1 > ceiling - 1;  in range_is_outside_limits()
320 			      unsigned long start, unsigned long end,  in free_hugepd_range()  argument
326 	unsigned long pdmask = ~((1UL << pdshift) - 1);  in free_hugepd_range()
332 		num_hugepd = 1 << (shift - pdshift);  in free_hugepd_range()
334 	if (range_is_outside_limits(start, end, floor, ceiling, pdmask))  in free_hugepd_range()
344 				 get_hugepd_cache_index(pdshift - shift));  in free_hugepd_range()
348 				   unsigned long addr, unsigned long end,  in hugetlb_free_pte_range()  argument
353 	if (range_is_outside_limits(addr, end, floor, ceiling, PMD_MASK))  in hugetlb_free_pte_range()
358 	mm_dec_nr_ptes(tlb->mm);  in hugetlb_free_pte_range()
362 				   unsigned long addr, unsigned long end,  in hugetlb_free_pmd_range()  argument
374 		next = pmd_addr_end(addr, end);  in hugetlb_free_pmd_range()
385 			hugetlb_free_pte_range(tlb, pmd, addr, end, floor, ceiling);  in hugetlb_free_pmd_range()
392 		 * single hugepage, but all of them point to  in hugetlb_free_pmd_range()
401 	} while (addr = next, addr != end);  in hugetlb_free_pmd_range()
403 	if (range_is_outside_limits(start, end, floor, ceiling, PUD_MASK))  in hugetlb_free_pmd_range()
409 	mm_dec_nr_pmds(tlb->mm);  in hugetlb_free_pmd_range()
413 				   unsigned long addr, unsigned long end,  in hugetlb_free_pud_range()  argument
423 		next = pud_addr_end(addr, end);  in hugetlb_free_pud_range()
434 			 * single hugepage, but all of them point to  in hugetlb_free_pud_range()
444 	} while (addr = next, addr != end);  in hugetlb_free_pud_range()
446 	if (range_is_outside_limits(start, end, floor, ceiling, PGDIR_MASK))  in hugetlb_free_pud_range()
452 	mm_dec_nr_puds(tlb->mm);  in hugetlb_free_pud_range()
456  * This function frees user-level page tables of a process.
459 			    unsigned long addr, unsigned long end,  in hugetlb_free_pgd_range()  argument
484 		next = pgd_addr_end(addr, end);  in hugetlb_free_pgd_range()
485 		pgd = pgd_offset(tlb->mm, addr);  in hugetlb_free_pgd_range()
496 			 * for a single hugepage, but all of them point to the  in hugetlb_free_pgd_range()
506 	} while (addr = next, addr != end);  in hugetlb_free_pgd_range()
533 		return -EINVAL;  in add_huge_page_size()
535 	hugetlb_add_hstate(shift - PAGE_SHIFT);  in add_huge_page_size()
551 		return -ENODEV;  in hugetlbpage_init()
588 				pgtable_cache_add(pdshift - shift);  in hugetlbpage_init()
610 		order = PUD_SHIFT - PAGE_SHIFT;  in gigantic_hugetlb_cma_reserve()
615 		order = mmu_psize_to_shift(MMU_PAGE_16G) - PAGE_SHIFT;  in gigantic_hugetlb_cma_reserve()