1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * PPC64 Huge TLB Page Support for hash based MMUs (POWER4 and later)
4  *
5  * Copyright (C) 2003 David Gibson, IBM Corporation.
6  *
7  * Based on the IA-32 version:
8  * Copyright (C) 2002, Rohit Seth <rohit.seth@intel.com>
9  */
10 
11 #include <linux/mm.h>
12 #include <linux/hugetlb.h>
13 #include <asm/cacheflush.h>
14 #include <asm/machdep.h>
15 
16 unsigned int hpage_shift;
17 EXPORT_SYMBOL(hpage_shift);
18 
19 #ifdef CONFIG_PPC_64S_HASH_MMU
20 int __hash_page_huge(unsigned long ea, unsigned long access, unsigned long vsid,
21 		     pte_t *ptep, unsigned long trap, unsigned long flags,
22 		     int ssize, unsigned int shift, unsigned int mmu_psize)
23 {
24 	real_pte_t rpte;
25 	unsigned long vpn;
26 	unsigned long old_pte, new_pte;
27 	unsigned long rflags, pa;
28 	long slot, offset;
29 
30 	BUG_ON(shift != mmu_psize_defs[mmu_psize].shift);
31 
32 	/* Search the Linux page table for a match with va */
33 	vpn = hpt_vpn(ea, vsid, ssize);
34 
35 	/*
36 	 * At this point, we have a pte (old_pte) which can be used to build
37 	 * or update an HPTE. There are 2 cases:
38 	 *
39 	 * 1. There is a valid (present) pte with no associated HPTE (this is
40 	 *	the most common case)
41 	 * 2. There is a valid (present) pte with an associated HPTE. The
42 	 *	current values of the pp bits in the HPTE prevent access
43 	 *	because we are doing software DIRTY bit management and the
44 	 *	page is currently not DIRTY.
45 	 */
46 
47 
48 	do {
49 		old_pte = pte_val(*ptep);
50 		/* If PTE busy, retry the access */
51 		if (unlikely(old_pte & H_PAGE_BUSY))
52 			return 0;
53 		/* If PTE permissions don't match, take page fault */
54 		if (unlikely(!check_pte_access(access, old_pte)))
55 			return 1;
56 
57 		/*
58 		 * Try to lock the PTE, add ACCESSED and DIRTY if it was
59 		 * a write access
60 		 */
61 		new_pte = old_pte | H_PAGE_BUSY | _PAGE_ACCESSED;
62 		if (access & _PAGE_WRITE)
63 			new_pte |= _PAGE_DIRTY;
64 	} while(!pte_xchg(ptep, __pte(old_pte), __pte(new_pte)));
65 
66 	/* Make sure this is a hugetlb entry */
67 	if (old_pte & (H_PAGE_THP_HUGE | _PAGE_DEVMAP))
68 		return 0;
69 
70 	rflags = htab_convert_pte_flags(new_pte, flags);
71 	if (unlikely(mmu_psize == MMU_PAGE_16G))
72 		offset = PTRS_PER_PUD;
73 	else
74 		offset = PTRS_PER_PMD;
75 	rpte = __real_pte(__pte(old_pte), ptep, offset);
76 
77 	if (!cpu_has_feature(CPU_FTR_COHERENT_ICACHE))
78 		/*
79 		 * No CPU has hugepages but lacks no execute, so we
80 		 * don't need to worry about that case
81 		 */
82 		rflags = hash_page_do_lazy_icache(rflags, __pte(old_pte), trap);
83 
84 	/* Check if pte already has an hpte (case 2) */
85 	if (unlikely(old_pte & H_PAGE_HASHPTE)) {
86 		/* There MIGHT be an HPTE for this pte */
87 		unsigned long gslot;
88 
89 		gslot = pte_get_hash_gslot(vpn, shift, ssize, rpte, 0);
90 		if (mmu_hash_ops.hpte_updatepp(gslot, rflags, vpn, mmu_psize,
91 					       mmu_psize, ssize, flags) == -1)
92 			old_pte &= ~_PAGE_HPTEFLAGS;
93 	}
94 
95 	if (likely(!(old_pte & H_PAGE_HASHPTE))) {
96 		unsigned long hash = hpt_hash(vpn, shift, ssize);
97 
98 		pa = pte_pfn(__pte(old_pte)) << PAGE_SHIFT;
99 
100 		/* clear HPTE slot informations in new PTE */
101 		new_pte = (new_pte & ~_PAGE_HPTEFLAGS) | H_PAGE_HASHPTE;
102 
103 		slot = hpte_insert_repeating(hash, vpn, pa, rflags, 0,
104 					     mmu_psize, ssize);
105 
106 		/*
107 		 * Hypervisor failure. Restore old pte and return -1
108 		 * similar to __hash_page_*
109 		 */
110 		if (unlikely(slot == -2)) {
111 			*ptep = __pte(old_pte);
112 			hash_failure_debug(ea, access, vsid, trap, ssize,
113 					   mmu_psize, mmu_psize, old_pte);
114 			return -1;
115 		}
116 
117 		new_pte |= pte_set_hidx(ptep, rpte, 0, slot, offset);
118 	}
119 
120 	/*
121 	 * No need to use ldarx/stdcx here
122 	 */
123 	*ptep = __pte(new_pte & ~H_PAGE_BUSY);
124 	return 0;
125 }
126 #endif
127 
128 pte_t huge_ptep_modify_prot_start(struct vm_area_struct *vma,
129 				  unsigned long addr, pte_t *ptep)
130 {
131 	unsigned long pte_val;
132 	/*
133 	 * Clear the _PAGE_PRESENT so that no hardware parallel update is
134 	 * possible. Also keep the pte_present true so that we don't take
135 	 * wrong fault.
136 	 */
137 	pte_val = pte_update(vma->vm_mm, addr, ptep,
138 			     _PAGE_PRESENT, _PAGE_INVALID, 1);
139 
140 	return __pte(pte_val);
141 }
142 
143 void huge_ptep_modify_prot_commit(struct vm_area_struct *vma, unsigned long addr,
144 				  pte_t *ptep, pte_t old_pte, pte_t pte)
145 {
146 	unsigned long psize;
147 
148 	if (radix_enabled())
149 		return radix__huge_ptep_modify_prot_commit(vma, addr, ptep,
150 							   old_pte, pte);
151 
152 	psize = huge_page_size(hstate_vma(vma));
153 	set_huge_pte_at(vma->vm_mm, addr, ptep, pte, psize);
154 }
155 
156 void __init hugetlbpage_init_defaultsize(void)
157 {
158 	/* Set default large page size. Currently, we pick 16M or 1M
159 	 * depending on what is available
160 	 */
161 	if (mmu_psize_defs[MMU_PAGE_16M].shift)
162 		hpage_shift = mmu_psize_defs[MMU_PAGE_16M].shift;
163 	else if (mmu_psize_defs[MMU_PAGE_1M].shift)
164 		hpage_shift = mmu_psize_defs[MMU_PAGE_1M].shift;
165 	else if (mmu_psize_defs[MMU_PAGE_2M].shift)
166 		hpage_shift = mmu_psize_defs[MMU_PAGE_2M].shift;
167 }
168