xref: /openbmc/linux/arch/powerpc/mm/hugetlbpage.c (revision d2999e1b)
1 /*
2  * PPC Huge TLB Page Support for Kernel.
3  *
4  * Copyright (C) 2003 David Gibson, IBM Corporation.
5  * Copyright (C) 2011 Becky Bruce, Freescale Semiconductor
6  *
7  * Based on the IA-32 version:
8  * Copyright (C) 2002, Rohit Seth <rohit.seth@intel.com>
9  */
10 
11 #include <linux/mm.h>
12 #include <linux/io.h>
13 #include <linux/slab.h>
14 #include <linux/hugetlb.h>
15 #include <linux/export.h>
16 #include <linux/of_fdt.h>
17 #include <linux/memblock.h>
18 #include <linux/bootmem.h>
19 #include <linux/moduleparam.h>
20 #include <asm/pgtable.h>
21 #include <asm/pgalloc.h>
22 #include <asm/tlb.h>
23 #include <asm/setup.h>
24 #include <asm/hugetlb.h>
25 
26 #ifdef CONFIG_HUGETLB_PAGE
27 
28 #define PAGE_SHIFT_64K	16
29 #define PAGE_SHIFT_16M	24
30 #define PAGE_SHIFT_16G	34
31 
32 unsigned int HPAGE_SHIFT;
33 
34 /*
35  * Tracks gpages after the device tree is scanned and before the
36  * huge_boot_pages list is ready.  On non-Freescale implementations, this is
37  * just used to track 16G pages and so is a single array.  FSL-based
38  * implementations may have more than one gpage size, so we need multiple
39  * arrays
40  */
41 #ifdef CONFIG_PPC_FSL_BOOK3E
42 #define MAX_NUMBER_GPAGES	128
43 struct psize_gpages {
44 	u64 gpage_list[MAX_NUMBER_GPAGES];
45 	unsigned int nr_gpages;
46 };
47 static struct psize_gpages gpage_freearray[MMU_PAGE_COUNT];
48 #else
49 #define MAX_NUMBER_GPAGES	1024
50 static u64 gpage_freearray[MAX_NUMBER_GPAGES];
51 static unsigned nr_gpages;
52 #endif
53 
54 #define hugepd_none(hpd)	((hpd).pd == 0)
55 
56 #ifdef CONFIG_PPC_BOOK3S_64
57 /*
58  * At this point we do the placement change only for BOOK3S 64. This would
59  * possibly work on other subarchs.
60  */
61 
62 /*
63  * We have PGD_INDEX_SIZ = 12 and PTE_INDEX_SIZE = 8, so that we can have
64  * 16GB hugepage pte in PGD and 16MB hugepage pte at PMD;
65  */
66 int pmd_huge(pmd_t pmd)
67 {
68 	/*
69 	 * leaf pte for huge page, bottom two bits != 00
70 	 */
71 	return ((pmd_val(pmd) & 0x3) != 0x0);
72 }
73 
74 int pud_huge(pud_t pud)
75 {
76 	/*
77 	 * leaf pte for huge page, bottom two bits != 00
78 	 */
79 	return ((pud_val(pud) & 0x3) != 0x0);
80 }
81 
82 int pgd_huge(pgd_t pgd)
83 {
84 	/*
85 	 * leaf pte for huge page, bottom two bits != 00
86 	 */
87 	return ((pgd_val(pgd) & 0x3) != 0x0);
88 }
89 #else
90 int pmd_huge(pmd_t pmd)
91 {
92 	return 0;
93 }
94 
95 int pud_huge(pud_t pud)
96 {
97 	return 0;
98 }
99 
100 int pgd_huge(pgd_t pgd)
101 {
102 	return 0;
103 }
104 #endif
105 
106 pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr)
107 {
108 	/* Only called for hugetlbfs pages, hence can ignore THP */
109 	return find_linux_pte_or_hugepte(mm->pgd, addr, NULL);
110 }
111 
112 static int __hugepte_alloc(struct mm_struct *mm, hugepd_t *hpdp,
113 			   unsigned long address, unsigned pdshift, unsigned pshift)
114 {
115 	struct kmem_cache *cachep;
116 	pte_t *new;
117 
118 #ifdef CONFIG_PPC_FSL_BOOK3E
119 	int i;
120 	int num_hugepd = 1 << (pshift - pdshift);
121 	cachep = hugepte_cache;
122 #else
123 	cachep = PGT_CACHE(pdshift - pshift);
124 #endif
125 
126 	new = kmem_cache_zalloc(cachep, GFP_KERNEL|__GFP_REPEAT);
127 
128 	BUG_ON(pshift > HUGEPD_SHIFT_MASK);
129 	BUG_ON((unsigned long)new & HUGEPD_SHIFT_MASK);
130 
131 	if (! new)
132 		return -ENOMEM;
133 
134 	spin_lock(&mm->page_table_lock);
135 #ifdef CONFIG_PPC_FSL_BOOK3E
136 	/*
137 	 * We have multiple higher-level entries that point to the same
138 	 * actual pte location.  Fill in each as we go and backtrack on error.
139 	 * We need all of these so the DTLB pgtable walk code can find the
140 	 * right higher-level entry without knowing if it's a hugepage or not.
141 	 */
142 	for (i = 0; i < num_hugepd; i++, hpdp++) {
143 		if (unlikely(!hugepd_none(*hpdp)))
144 			break;
145 		else
146 			/* We use the old format for PPC_FSL_BOOK3E */
147 			hpdp->pd = ((unsigned long)new & ~PD_HUGE) | pshift;
148 	}
149 	/* If we bailed from the for loop early, an error occurred, clean up */
150 	if (i < num_hugepd) {
151 		for (i = i - 1 ; i >= 0; i--, hpdp--)
152 			hpdp->pd = 0;
153 		kmem_cache_free(cachep, new);
154 	}
155 #else
156 	if (!hugepd_none(*hpdp))
157 		kmem_cache_free(cachep, new);
158 	else {
159 #ifdef CONFIG_PPC_BOOK3S_64
160 		hpdp->pd = (unsigned long)new |
161 			    (shift_to_mmu_psize(pshift) << 2);
162 #else
163 		hpdp->pd = ((unsigned long)new & ~PD_HUGE) | pshift;
164 #endif
165 	}
166 #endif
167 	spin_unlock(&mm->page_table_lock);
168 	return 0;
169 }
170 
171 /*
172  * These macros define how to determine which level of the page table holds
173  * the hpdp.
174  */
175 #ifdef CONFIG_PPC_FSL_BOOK3E
176 #define HUGEPD_PGD_SHIFT PGDIR_SHIFT
177 #define HUGEPD_PUD_SHIFT PUD_SHIFT
178 #else
179 #define HUGEPD_PGD_SHIFT PUD_SHIFT
180 #define HUGEPD_PUD_SHIFT PMD_SHIFT
181 #endif
182 
183 #ifdef CONFIG_PPC_BOOK3S_64
184 /*
185  * At this point we do the placement change only for BOOK3S 64. This would
186  * possibly work on other subarchs.
187  */
188 pte_t *huge_pte_alloc(struct mm_struct *mm, unsigned long addr, unsigned long sz)
189 {
190 	pgd_t *pg;
191 	pud_t *pu;
192 	pmd_t *pm;
193 	hugepd_t *hpdp = NULL;
194 	unsigned pshift = __ffs(sz);
195 	unsigned pdshift = PGDIR_SHIFT;
196 
197 	addr &= ~(sz-1);
198 	pg = pgd_offset(mm, addr);
199 
200 	if (pshift == PGDIR_SHIFT)
201 		/* 16GB huge page */
202 		return (pte_t *) pg;
203 	else if (pshift > PUD_SHIFT)
204 		/*
205 		 * We need to use hugepd table
206 		 */
207 		hpdp = (hugepd_t *)pg;
208 	else {
209 		pdshift = PUD_SHIFT;
210 		pu = pud_alloc(mm, pg, addr);
211 		if (pshift == PUD_SHIFT)
212 			return (pte_t *)pu;
213 		else if (pshift > PMD_SHIFT)
214 			hpdp = (hugepd_t *)pu;
215 		else {
216 			pdshift = PMD_SHIFT;
217 			pm = pmd_alloc(mm, pu, addr);
218 			if (pshift == PMD_SHIFT)
219 				/* 16MB hugepage */
220 				return (pte_t *)pm;
221 			else
222 				hpdp = (hugepd_t *)pm;
223 		}
224 	}
225 	if (!hpdp)
226 		return NULL;
227 
228 	BUG_ON(!hugepd_none(*hpdp) && !hugepd_ok(*hpdp));
229 
230 	if (hugepd_none(*hpdp) && __hugepte_alloc(mm, hpdp, addr, pdshift, pshift))
231 		return NULL;
232 
233 	return hugepte_offset(hpdp, addr, pdshift);
234 }
235 
236 #else
237 
238 pte_t *huge_pte_alloc(struct mm_struct *mm, unsigned long addr, unsigned long sz)
239 {
240 	pgd_t *pg;
241 	pud_t *pu;
242 	pmd_t *pm;
243 	hugepd_t *hpdp = NULL;
244 	unsigned pshift = __ffs(sz);
245 	unsigned pdshift = PGDIR_SHIFT;
246 
247 	addr &= ~(sz-1);
248 
249 	pg = pgd_offset(mm, addr);
250 
251 	if (pshift >= HUGEPD_PGD_SHIFT) {
252 		hpdp = (hugepd_t *)pg;
253 	} else {
254 		pdshift = PUD_SHIFT;
255 		pu = pud_alloc(mm, pg, addr);
256 		if (pshift >= HUGEPD_PUD_SHIFT) {
257 			hpdp = (hugepd_t *)pu;
258 		} else {
259 			pdshift = PMD_SHIFT;
260 			pm = pmd_alloc(mm, pu, addr);
261 			hpdp = (hugepd_t *)pm;
262 		}
263 	}
264 
265 	if (!hpdp)
266 		return NULL;
267 
268 	BUG_ON(!hugepd_none(*hpdp) && !hugepd_ok(*hpdp));
269 
270 	if (hugepd_none(*hpdp) && __hugepte_alloc(mm, hpdp, addr, pdshift, pshift))
271 		return NULL;
272 
273 	return hugepte_offset(hpdp, addr, pdshift);
274 }
275 #endif
276 
277 #ifdef CONFIG_PPC_FSL_BOOK3E
278 /* Build list of addresses of gigantic pages.  This function is used in early
279  * boot before the buddy or bootmem allocator is setup.
280  */
281 void add_gpage(u64 addr, u64 page_size, unsigned long number_of_pages)
282 {
283 	unsigned int idx = shift_to_mmu_psize(__ffs(page_size));
284 	int i;
285 
286 	if (addr == 0)
287 		return;
288 
289 	gpage_freearray[idx].nr_gpages = number_of_pages;
290 
291 	for (i = 0; i < number_of_pages; i++) {
292 		gpage_freearray[idx].gpage_list[i] = addr;
293 		addr += page_size;
294 	}
295 }
296 
297 /*
298  * Moves the gigantic page addresses from the temporary list to the
299  * huge_boot_pages list.
300  */
301 int alloc_bootmem_huge_page(struct hstate *hstate)
302 {
303 	struct huge_bootmem_page *m;
304 	int idx = shift_to_mmu_psize(huge_page_shift(hstate));
305 	int nr_gpages = gpage_freearray[idx].nr_gpages;
306 
307 	if (nr_gpages == 0)
308 		return 0;
309 
310 #ifdef CONFIG_HIGHMEM
311 	/*
312 	 * If gpages can be in highmem we can't use the trick of storing the
313 	 * data structure in the page; allocate space for this
314 	 */
315 	m = alloc_bootmem(sizeof(struct huge_bootmem_page));
316 	m->phys = gpage_freearray[idx].gpage_list[--nr_gpages];
317 #else
318 	m = phys_to_virt(gpage_freearray[idx].gpage_list[--nr_gpages]);
319 #endif
320 
321 	list_add(&m->list, &huge_boot_pages);
322 	gpage_freearray[idx].nr_gpages = nr_gpages;
323 	gpage_freearray[idx].gpage_list[nr_gpages] = 0;
324 	m->hstate = hstate;
325 
326 	return 1;
327 }
328 /*
329  * Scan the command line hugepagesz= options for gigantic pages; store those in
330  * a list that we use to allocate the memory once all options are parsed.
331  */
332 
333 unsigned long gpage_npages[MMU_PAGE_COUNT];
334 
335 static int __init do_gpage_early_setup(char *param, char *val,
336 				       const char *unused)
337 {
338 	static phys_addr_t size;
339 	unsigned long npages;
340 
341 	/*
342 	 * The hugepagesz and hugepages cmdline options are interleaved.  We
343 	 * use the size variable to keep track of whether or not this was done
344 	 * properly and skip over instances where it is incorrect.  Other
345 	 * command-line parsing code will issue warnings, so we don't need to.
346 	 *
347 	 */
348 	if ((strcmp(param, "default_hugepagesz") == 0) ||
349 	    (strcmp(param, "hugepagesz") == 0)) {
350 		size = memparse(val, NULL);
351 	} else if (strcmp(param, "hugepages") == 0) {
352 		if (size != 0) {
353 			if (sscanf(val, "%lu", &npages) <= 0)
354 				npages = 0;
355 			gpage_npages[shift_to_mmu_psize(__ffs(size))] = npages;
356 			size = 0;
357 		}
358 	}
359 	return 0;
360 }
361 
362 
363 /*
364  * This function allocates physical space for pages that are larger than the
365  * buddy allocator can handle.  We want to allocate these in highmem because
366  * the amount of lowmem is limited.  This means that this function MUST be
367  * called before lowmem_end_addr is set up in MMU_init() in order for the lmb
368  * allocate to grab highmem.
369  */
370 void __init reserve_hugetlb_gpages(void)
371 {
372 	static __initdata char cmdline[COMMAND_LINE_SIZE];
373 	phys_addr_t size, base;
374 	int i;
375 
376 	strlcpy(cmdline, boot_command_line, COMMAND_LINE_SIZE);
377 	parse_args("hugetlb gpages", cmdline, NULL, 0, 0, 0,
378 			&do_gpage_early_setup);
379 
380 	/*
381 	 * Walk gpage list in reverse, allocating larger page sizes first.
382 	 * Skip over unsupported sizes, or sizes that have 0 gpages allocated.
383 	 * When we reach the point in the list where pages are no longer
384 	 * considered gpages, we're done.
385 	 */
386 	for (i = MMU_PAGE_COUNT-1; i >= 0; i--) {
387 		if (mmu_psize_defs[i].shift == 0 || gpage_npages[i] == 0)
388 			continue;
389 		else if (mmu_psize_to_shift(i) < (MAX_ORDER + PAGE_SHIFT))
390 			break;
391 
392 		size = (phys_addr_t)(1ULL << mmu_psize_to_shift(i));
393 		base = memblock_alloc_base(size * gpage_npages[i], size,
394 					   MEMBLOCK_ALLOC_ANYWHERE);
395 		add_gpage(base, size, gpage_npages[i]);
396 	}
397 }
398 
399 #else /* !PPC_FSL_BOOK3E */
400 
401 /* Build list of addresses of gigantic pages.  This function is used in early
402  * boot before the buddy or bootmem allocator is setup.
403  */
404 void add_gpage(u64 addr, u64 page_size, unsigned long number_of_pages)
405 {
406 	if (!addr)
407 		return;
408 	while (number_of_pages > 0) {
409 		gpage_freearray[nr_gpages] = addr;
410 		nr_gpages++;
411 		number_of_pages--;
412 		addr += page_size;
413 	}
414 }
415 
416 /* Moves the gigantic page addresses from the temporary list to the
417  * huge_boot_pages list.
418  */
419 int alloc_bootmem_huge_page(struct hstate *hstate)
420 {
421 	struct huge_bootmem_page *m;
422 	if (nr_gpages == 0)
423 		return 0;
424 	m = phys_to_virt(gpage_freearray[--nr_gpages]);
425 	gpage_freearray[nr_gpages] = 0;
426 	list_add(&m->list, &huge_boot_pages);
427 	m->hstate = hstate;
428 	return 1;
429 }
430 #endif
431 
432 int huge_pmd_unshare(struct mm_struct *mm, unsigned long *addr, pte_t *ptep)
433 {
434 	return 0;
435 }
436 
437 #ifdef CONFIG_PPC_FSL_BOOK3E
438 #define HUGEPD_FREELIST_SIZE \
439 	((PAGE_SIZE - sizeof(struct hugepd_freelist)) / sizeof(pte_t))
440 
441 struct hugepd_freelist {
442 	struct rcu_head	rcu;
443 	unsigned int index;
444 	void *ptes[0];
445 };
446 
447 static DEFINE_PER_CPU(struct hugepd_freelist *, hugepd_freelist_cur);
448 
449 static void hugepd_free_rcu_callback(struct rcu_head *head)
450 {
451 	struct hugepd_freelist *batch =
452 		container_of(head, struct hugepd_freelist, rcu);
453 	unsigned int i;
454 
455 	for (i = 0; i < batch->index; i++)
456 		kmem_cache_free(hugepte_cache, batch->ptes[i]);
457 
458 	free_page((unsigned long)batch);
459 }
460 
461 static void hugepd_free(struct mmu_gather *tlb, void *hugepte)
462 {
463 	struct hugepd_freelist **batchp;
464 
465 	batchp = &get_cpu_var(hugepd_freelist_cur);
466 
467 	if (atomic_read(&tlb->mm->mm_users) < 2 ||
468 	    cpumask_equal(mm_cpumask(tlb->mm),
469 			  cpumask_of(smp_processor_id()))) {
470 		kmem_cache_free(hugepte_cache, hugepte);
471         put_cpu_var(hugepd_freelist_cur);
472 		return;
473 	}
474 
475 	if (*batchp == NULL) {
476 		*batchp = (struct hugepd_freelist *)__get_free_page(GFP_ATOMIC);
477 		(*batchp)->index = 0;
478 	}
479 
480 	(*batchp)->ptes[(*batchp)->index++] = hugepte;
481 	if ((*batchp)->index == HUGEPD_FREELIST_SIZE) {
482 		call_rcu_sched(&(*batchp)->rcu, hugepd_free_rcu_callback);
483 		*batchp = NULL;
484 	}
485 	put_cpu_var(hugepd_freelist_cur);
486 }
487 #endif
488 
489 static void free_hugepd_range(struct mmu_gather *tlb, hugepd_t *hpdp, int pdshift,
490 			      unsigned long start, unsigned long end,
491 			      unsigned long floor, unsigned long ceiling)
492 {
493 	pte_t *hugepte = hugepd_page(*hpdp);
494 	int i;
495 
496 	unsigned long pdmask = ~((1UL << pdshift) - 1);
497 	unsigned int num_hugepd = 1;
498 
499 #ifdef CONFIG_PPC_FSL_BOOK3E
500 	/* Note: On fsl the hpdp may be the first of several */
501 	num_hugepd = (1 << (hugepd_shift(*hpdp) - pdshift));
502 #else
503 	unsigned int shift = hugepd_shift(*hpdp);
504 #endif
505 
506 	start &= pdmask;
507 	if (start < floor)
508 		return;
509 	if (ceiling) {
510 		ceiling &= pdmask;
511 		if (! ceiling)
512 			return;
513 	}
514 	if (end - 1 > ceiling - 1)
515 		return;
516 
517 	for (i = 0; i < num_hugepd; i++, hpdp++)
518 		hpdp->pd = 0;
519 
520 	tlb->need_flush = 1;
521 
522 #ifdef CONFIG_PPC_FSL_BOOK3E
523 	hugepd_free(tlb, hugepte);
524 #else
525 	pgtable_free_tlb(tlb, hugepte, pdshift - shift);
526 #endif
527 }
528 
529 static void hugetlb_free_pmd_range(struct mmu_gather *tlb, pud_t *pud,
530 				   unsigned long addr, unsigned long end,
531 				   unsigned long floor, unsigned long ceiling)
532 {
533 	pmd_t *pmd;
534 	unsigned long next;
535 	unsigned long start;
536 
537 	start = addr;
538 	do {
539 		pmd = pmd_offset(pud, addr);
540 		next = pmd_addr_end(addr, end);
541 		if (!is_hugepd(pmd)) {
542 			/*
543 			 * if it is not hugepd pointer, we should already find
544 			 * it cleared.
545 			 */
546 			WARN_ON(!pmd_none_or_clear_bad(pmd));
547 			continue;
548 		}
549 #ifdef CONFIG_PPC_FSL_BOOK3E
550 		/*
551 		 * Increment next by the size of the huge mapping since
552 		 * there may be more than one entry at this level for a
553 		 * single hugepage, but all of them point to
554 		 * the same kmem cache that holds the hugepte.
555 		 */
556 		next = addr + (1 << hugepd_shift(*(hugepd_t *)pmd));
557 #endif
558 		free_hugepd_range(tlb, (hugepd_t *)pmd, PMD_SHIFT,
559 				  addr, next, floor, ceiling);
560 	} while (addr = next, addr != end);
561 
562 	start &= PUD_MASK;
563 	if (start < floor)
564 		return;
565 	if (ceiling) {
566 		ceiling &= PUD_MASK;
567 		if (!ceiling)
568 			return;
569 	}
570 	if (end - 1 > ceiling - 1)
571 		return;
572 
573 	pmd = pmd_offset(pud, start);
574 	pud_clear(pud);
575 	pmd_free_tlb(tlb, pmd, start);
576 }
577 
578 static void hugetlb_free_pud_range(struct mmu_gather *tlb, pgd_t *pgd,
579 				   unsigned long addr, unsigned long end,
580 				   unsigned long floor, unsigned long ceiling)
581 {
582 	pud_t *pud;
583 	unsigned long next;
584 	unsigned long start;
585 
586 	start = addr;
587 	do {
588 		pud = pud_offset(pgd, addr);
589 		next = pud_addr_end(addr, end);
590 		if (!is_hugepd(pud)) {
591 			if (pud_none_or_clear_bad(pud))
592 				continue;
593 			hugetlb_free_pmd_range(tlb, pud, addr, next, floor,
594 					       ceiling);
595 		} else {
596 #ifdef CONFIG_PPC_FSL_BOOK3E
597 			/*
598 			 * Increment next by the size of the huge mapping since
599 			 * there may be more than one entry at this level for a
600 			 * single hugepage, but all of them point to
601 			 * the same kmem cache that holds the hugepte.
602 			 */
603 			next = addr + (1 << hugepd_shift(*(hugepd_t *)pud));
604 #endif
605 			free_hugepd_range(tlb, (hugepd_t *)pud, PUD_SHIFT,
606 					  addr, next, floor, ceiling);
607 		}
608 	} while (addr = next, addr != end);
609 
610 	start &= PGDIR_MASK;
611 	if (start < floor)
612 		return;
613 	if (ceiling) {
614 		ceiling &= PGDIR_MASK;
615 		if (!ceiling)
616 			return;
617 	}
618 	if (end - 1 > ceiling - 1)
619 		return;
620 
621 	pud = pud_offset(pgd, start);
622 	pgd_clear(pgd);
623 	pud_free_tlb(tlb, pud, start);
624 }
625 
626 /*
627  * This function frees user-level page tables of a process.
628  */
629 void hugetlb_free_pgd_range(struct mmu_gather *tlb,
630 			    unsigned long addr, unsigned long end,
631 			    unsigned long floor, unsigned long ceiling)
632 {
633 	pgd_t *pgd;
634 	unsigned long next;
635 
636 	/*
637 	 * Because there are a number of different possible pagetable
638 	 * layouts for hugepage ranges, we limit knowledge of how
639 	 * things should be laid out to the allocation path
640 	 * (huge_pte_alloc(), above).  Everything else works out the
641 	 * structure as it goes from information in the hugepd
642 	 * pointers.  That means that we can't here use the
643 	 * optimization used in the normal page free_pgd_range(), of
644 	 * checking whether we're actually covering a large enough
645 	 * range to have to do anything at the top level of the walk
646 	 * instead of at the bottom.
647 	 *
648 	 * To make sense of this, you should probably go read the big
649 	 * block comment at the top of the normal free_pgd_range(),
650 	 * too.
651 	 */
652 
653 	do {
654 		next = pgd_addr_end(addr, end);
655 		pgd = pgd_offset(tlb->mm, addr);
656 		if (!is_hugepd(pgd)) {
657 			if (pgd_none_or_clear_bad(pgd))
658 				continue;
659 			hugetlb_free_pud_range(tlb, pgd, addr, next, floor, ceiling);
660 		} else {
661 #ifdef CONFIG_PPC_FSL_BOOK3E
662 			/*
663 			 * Increment next by the size of the huge mapping since
664 			 * there may be more than one entry at the pgd level
665 			 * for a single hugepage, but all of them point to the
666 			 * same kmem cache that holds the hugepte.
667 			 */
668 			next = addr + (1 << hugepd_shift(*(hugepd_t *)pgd));
669 #endif
670 			free_hugepd_range(tlb, (hugepd_t *)pgd, PGDIR_SHIFT,
671 					  addr, next, floor, ceiling);
672 		}
673 	} while (addr = next, addr != end);
674 }
675 
676 struct page *
677 follow_huge_addr(struct mm_struct *mm, unsigned long address, int write)
678 {
679 	pte_t *ptep;
680 	struct page *page;
681 	unsigned shift;
682 	unsigned long mask;
683 	/*
684 	 * Transparent hugepages are handled by generic code. We can skip them
685 	 * here.
686 	 */
687 	ptep = find_linux_pte_or_hugepte(mm->pgd, address, &shift);
688 
689 	/* Verify it is a huge page else bail. */
690 	if (!ptep || !shift || pmd_trans_huge(*(pmd_t *)ptep))
691 		return ERR_PTR(-EINVAL);
692 
693 	mask = (1UL << shift) - 1;
694 	page = pte_page(*ptep);
695 	if (page)
696 		page += (address & mask) / PAGE_SIZE;
697 
698 	return page;
699 }
700 
701 struct page *
702 follow_huge_pmd(struct mm_struct *mm, unsigned long address,
703 		pmd_t *pmd, int write)
704 {
705 	BUG();
706 	return NULL;
707 }
708 
709 static unsigned long hugepte_addr_end(unsigned long addr, unsigned long end,
710 				      unsigned long sz)
711 {
712 	unsigned long __boundary = (addr + sz) & ~(sz-1);
713 	return (__boundary - 1 < end - 1) ? __boundary : end;
714 }
715 
716 int gup_hugepd(hugepd_t *hugepd, unsigned pdshift,
717 	       unsigned long addr, unsigned long end,
718 	       int write, struct page **pages, int *nr)
719 {
720 	pte_t *ptep;
721 	unsigned long sz = 1UL << hugepd_shift(*hugepd);
722 	unsigned long next;
723 
724 	ptep = hugepte_offset(hugepd, addr, pdshift);
725 	do {
726 		next = hugepte_addr_end(addr, end, sz);
727 		if (!gup_hugepte(ptep, sz, addr, end, write, pages, nr))
728 			return 0;
729 	} while (ptep++, addr = next, addr != end);
730 
731 	return 1;
732 }
733 
734 #ifdef CONFIG_PPC_MM_SLICES
735 unsigned long hugetlb_get_unmapped_area(struct file *file, unsigned long addr,
736 					unsigned long len, unsigned long pgoff,
737 					unsigned long flags)
738 {
739 	struct hstate *hstate = hstate_file(file);
740 	int mmu_psize = shift_to_mmu_psize(huge_page_shift(hstate));
741 
742 	return slice_get_unmapped_area(addr, len, flags, mmu_psize, 1);
743 }
744 #endif
745 
746 unsigned long vma_mmu_pagesize(struct vm_area_struct *vma)
747 {
748 #ifdef CONFIG_PPC_MM_SLICES
749 	unsigned int psize = get_slice_psize(vma->vm_mm, vma->vm_start);
750 
751 	return 1UL << mmu_psize_to_shift(psize);
752 #else
753 	if (!is_vm_hugetlb_page(vma))
754 		return PAGE_SIZE;
755 
756 	return huge_page_size(hstate_vma(vma));
757 #endif
758 }
759 
760 static inline bool is_power_of_4(unsigned long x)
761 {
762 	if (is_power_of_2(x))
763 		return (__ilog2(x) % 2) ? false : true;
764 	return false;
765 }
766 
767 static int __init add_huge_page_size(unsigned long long size)
768 {
769 	int shift = __ffs(size);
770 	int mmu_psize;
771 
772 	/* Check that it is a page size supported by the hardware and
773 	 * that it fits within pagetable and slice limits. */
774 #ifdef CONFIG_PPC_FSL_BOOK3E
775 	if ((size < PAGE_SIZE) || !is_power_of_4(size))
776 		return -EINVAL;
777 #else
778 	if (!is_power_of_2(size)
779 	    || (shift > SLICE_HIGH_SHIFT) || (shift <= PAGE_SHIFT))
780 		return -EINVAL;
781 #endif
782 
783 	if ((mmu_psize = shift_to_mmu_psize(shift)) < 0)
784 		return -EINVAL;
785 
786 #ifdef CONFIG_SPU_FS_64K_LS
787 	/* Disable support for 64K huge pages when 64K SPU local store
788 	 * support is enabled as the current implementation conflicts.
789 	 */
790 	if (shift == PAGE_SHIFT_64K)
791 		return -EINVAL;
792 #endif /* CONFIG_SPU_FS_64K_LS */
793 
794 	BUG_ON(mmu_psize_defs[mmu_psize].shift != shift);
795 
796 	/* Return if huge page size has already been setup */
797 	if (size_to_hstate(size))
798 		return 0;
799 
800 	hugetlb_add_hstate(shift - PAGE_SHIFT);
801 
802 	return 0;
803 }
804 
805 static int __init hugepage_setup_sz(char *str)
806 {
807 	unsigned long long size;
808 
809 	size = memparse(str, &str);
810 
811 	if (add_huge_page_size(size) != 0)
812 		printk(KERN_WARNING "Invalid huge page size specified(%llu)\n", size);
813 
814 	return 1;
815 }
816 __setup("hugepagesz=", hugepage_setup_sz);
817 
818 #ifdef CONFIG_PPC_FSL_BOOK3E
819 struct kmem_cache *hugepte_cache;
820 static int __init hugetlbpage_init(void)
821 {
822 	int psize;
823 
824 	for (psize = 0; psize < MMU_PAGE_COUNT; ++psize) {
825 		unsigned shift;
826 
827 		if (!mmu_psize_defs[psize].shift)
828 			continue;
829 
830 		shift = mmu_psize_to_shift(psize);
831 
832 		/* Don't treat normal page sizes as huge... */
833 		if (shift != PAGE_SHIFT)
834 			if (add_huge_page_size(1ULL << shift) < 0)
835 				continue;
836 	}
837 
838 	/*
839 	 * Create a kmem cache for hugeptes.  The bottom bits in the pte have
840 	 * size information encoded in them, so align them to allow this
841 	 */
842 	hugepte_cache =  kmem_cache_create("hugepte-cache", sizeof(pte_t),
843 					   HUGEPD_SHIFT_MASK + 1, 0, NULL);
844 	if (hugepte_cache == NULL)
845 		panic("%s: Unable to create kmem cache for hugeptes\n",
846 		      __func__);
847 
848 	/* Default hpage size = 4M */
849 	if (mmu_psize_defs[MMU_PAGE_4M].shift)
850 		HPAGE_SHIFT = mmu_psize_defs[MMU_PAGE_4M].shift;
851 	else
852 		panic("%s: Unable to set default huge page size\n", __func__);
853 
854 
855 	return 0;
856 }
857 #else
858 static int __init hugetlbpage_init(void)
859 {
860 	int psize;
861 
862 	if (!mmu_has_feature(MMU_FTR_16M_PAGE))
863 		return -ENODEV;
864 
865 	for (psize = 0; psize < MMU_PAGE_COUNT; ++psize) {
866 		unsigned shift;
867 		unsigned pdshift;
868 
869 		if (!mmu_psize_defs[psize].shift)
870 			continue;
871 
872 		shift = mmu_psize_to_shift(psize);
873 
874 		if (add_huge_page_size(1ULL << shift) < 0)
875 			continue;
876 
877 		if (shift < PMD_SHIFT)
878 			pdshift = PMD_SHIFT;
879 		else if (shift < PUD_SHIFT)
880 			pdshift = PUD_SHIFT;
881 		else
882 			pdshift = PGDIR_SHIFT;
883 		/*
884 		 * if we have pdshift and shift value same, we don't
885 		 * use pgt cache for hugepd.
886 		 */
887 		if (pdshift != shift) {
888 			pgtable_cache_add(pdshift - shift, NULL);
889 			if (!PGT_CACHE(pdshift - shift))
890 				panic("hugetlbpage_init(): could not create "
891 				      "pgtable cache for %d bit pagesize\n", shift);
892 		}
893 	}
894 
895 	/* Set default large page size. Currently, we pick 16M or 1M
896 	 * depending on what is available
897 	 */
898 	if (mmu_psize_defs[MMU_PAGE_16M].shift)
899 		HPAGE_SHIFT = mmu_psize_defs[MMU_PAGE_16M].shift;
900 	else if (mmu_psize_defs[MMU_PAGE_1M].shift)
901 		HPAGE_SHIFT = mmu_psize_defs[MMU_PAGE_1M].shift;
902 
903 	return 0;
904 }
905 #endif
906 module_init(hugetlbpage_init);
907 
908 void flush_dcache_icache_hugepage(struct page *page)
909 {
910 	int i;
911 	void *start;
912 
913 	BUG_ON(!PageCompound(page));
914 
915 	for (i = 0; i < (1UL << compound_order(page)); i++) {
916 		if (!PageHighMem(page)) {
917 			__flush_dcache_icache(page_address(page+i));
918 		} else {
919 			start = kmap_atomic(page+i);
920 			__flush_dcache_icache(start);
921 			kunmap_atomic(start);
922 		}
923 	}
924 }
925 
926 #endif /* CONFIG_HUGETLB_PAGE */
927 
928 /*
929  * We have 4 cases for pgds and pmds:
930  * (1) invalid (all zeroes)
931  * (2) pointer to next table, as normal; bottom 6 bits == 0
932  * (3) leaf pte for huge page, bottom two bits != 00
933  * (4) hugepd pointer, bottom two bits == 00, next 4 bits indicate size of table
934  *
935  * So long as we atomically load page table pointers we are safe against teardown,
936  * we can follow the address down to the the page and take a ref on it.
937  */
938 
939 pte_t *find_linux_pte_or_hugepte(pgd_t *pgdir, unsigned long ea, unsigned *shift)
940 {
941 	pgd_t pgd, *pgdp;
942 	pud_t pud, *pudp;
943 	pmd_t pmd, *pmdp;
944 	pte_t *ret_pte;
945 	hugepd_t *hpdp = NULL;
946 	unsigned pdshift = PGDIR_SHIFT;
947 
948 	if (shift)
949 		*shift = 0;
950 
951 	pgdp = pgdir + pgd_index(ea);
952 	pgd  = ACCESS_ONCE(*pgdp);
953 	/*
954 	 * Always operate on the local stack value. This make sure the
955 	 * value don't get updated by a parallel THP split/collapse,
956 	 * page fault or a page unmap. The return pte_t * is still not
957 	 * stable. So should be checked there for above conditions.
958 	 */
959 	if (pgd_none(pgd))
960 		return NULL;
961 	else if (pgd_huge(pgd)) {
962 		ret_pte = (pte_t *) pgdp;
963 		goto out;
964 	} else if (is_hugepd(&pgd))
965 		hpdp = (hugepd_t *)&pgd;
966 	else {
967 		/*
968 		 * Even if we end up with an unmap, the pgtable will not
969 		 * be freed, because we do an rcu free and here we are
970 		 * irq disabled
971 		 */
972 		pdshift = PUD_SHIFT;
973 		pudp = pud_offset(&pgd, ea);
974 		pud  = ACCESS_ONCE(*pudp);
975 
976 		if (pud_none(pud))
977 			return NULL;
978 		else if (pud_huge(pud)) {
979 			ret_pte = (pte_t *) pudp;
980 			goto out;
981 		} else if (is_hugepd(&pud))
982 			hpdp = (hugepd_t *)&pud;
983 		else {
984 			pdshift = PMD_SHIFT;
985 			pmdp = pmd_offset(&pud, ea);
986 			pmd  = ACCESS_ONCE(*pmdp);
987 			/*
988 			 * A hugepage collapse is captured by pmd_none, because
989 			 * it mark the pmd none and do a hpte invalidate.
990 			 *
991 			 * A hugepage split is captured by pmd_trans_splitting
992 			 * because we mark the pmd trans splitting and do a
993 			 * hpte invalidate
994 			 *
995 			 */
996 			if (pmd_none(pmd) || pmd_trans_splitting(pmd))
997 				return NULL;
998 
999 			if (pmd_huge(pmd) || pmd_large(pmd)) {
1000 				ret_pte = (pte_t *) pmdp;
1001 				goto out;
1002 			} else if (is_hugepd(&pmd))
1003 				hpdp = (hugepd_t *)&pmd;
1004 			else
1005 				return pte_offset_kernel(&pmd, ea);
1006 		}
1007 	}
1008 	if (!hpdp)
1009 		return NULL;
1010 
1011 	ret_pte = hugepte_offset(hpdp, ea, pdshift);
1012 	pdshift = hugepd_shift(*hpdp);
1013 out:
1014 	if (shift)
1015 		*shift = pdshift;
1016 	return ret_pte;
1017 }
1018 EXPORT_SYMBOL_GPL(find_linux_pte_or_hugepte);
1019 
1020 int gup_hugepte(pte_t *ptep, unsigned long sz, unsigned long addr,
1021 		unsigned long end, int write, struct page **pages, int *nr)
1022 {
1023 	unsigned long mask;
1024 	unsigned long pte_end;
1025 	struct page *head, *page, *tail;
1026 	pte_t pte;
1027 	int refs;
1028 
1029 	pte_end = (addr + sz) & ~(sz-1);
1030 	if (pte_end < end)
1031 		end = pte_end;
1032 
1033 	pte = ACCESS_ONCE(*ptep);
1034 	mask = _PAGE_PRESENT | _PAGE_USER;
1035 	if (write)
1036 		mask |= _PAGE_RW;
1037 
1038 	if ((pte_val(pte) & mask) != mask)
1039 		return 0;
1040 
1041 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
1042 	/*
1043 	 * check for splitting here
1044 	 */
1045 	if (pmd_trans_splitting(pte_pmd(pte)))
1046 		return 0;
1047 #endif
1048 
1049 	/* hugepages are never "special" */
1050 	VM_BUG_ON(!pfn_valid(pte_pfn(pte)));
1051 
1052 	refs = 0;
1053 	head = pte_page(pte);
1054 
1055 	page = head + ((addr & (sz-1)) >> PAGE_SHIFT);
1056 	tail = page;
1057 	do {
1058 		VM_BUG_ON(compound_head(page) != head);
1059 		pages[*nr] = page;
1060 		(*nr)++;
1061 		page++;
1062 		refs++;
1063 	} while (addr += PAGE_SIZE, addr != end);
1064 
1065 	if (!page_cache_add_speculative(head, refs)) {
1066 		*nr -= refs;
1067 		return 0;
1068 	}
1069 
1070 	if (unlikely(pte_val(pte) != pte_val(*ptep))) {
1071 		/* Could be optimized better */
1072 		*nr -= refs;
1073 		while (refs--)
1074 			put_page(head);
1075 		return 0;
1076 	}
1077 
1078 	/*
1079 	 * Any tail page need their mapcount reference taken before we
1080 	 * return.
1081 	 */
1082 	while (refs--) {
1083 		if (PageTail(tail))
1084 			get_huge_page_tail(tail);
1085 		tail++;
1086 	}
1087 
1088 	return 1;
1089 }
1090