1 /* 2 * IBM System z Huge TLB Page Support for Kernel. 3 * 4 * Copyright IBM Corp. 2007 5 * Author(s): Gerald Schaefer <gerald.schaefer@de.ibm.com> 6 */ 7 8 #include <linux/mm.h> 9 #include <linux/hugetlb.h> 10 11 static inline pmd_t __pte_to_pmd(pte_t pte) 12 { 13 pmd_t pmd; 14 15 /* 16 * Convert encoding pte bits pmd bits 17 * lIR.uswrdy.p dy..R...I...wr 18 * empty 010.000000.0 -> 00..0...1...00 19 * prot-none, clean, old 111.000000.1 -> 00..1...1...00 20 * prot-none, clean, young 111.000001.1 -> 01..1...1...00 21 * prot-none, dirty, old 111.000010.1 -> 10..1...1...00 22 * prot-none, dirty, young 111.000011.1 -> 11..1...1...00 23 * read-only, clean, old 111.000100.1 -> 00..1...1...01 24 * read-only, clean, young 101.000101.1 -> 01..1...0...01 25 * read-only, dirty, old 111.000110.1 -> 10..1...1...01 26 * read-only, dirty, young 101.000111.1 -> 11..1...0...01 27 * read-write, clean, old 111.001100.1 -> 00..1...1...11 28 * read-write, clean, young 101.001101.1 -> 01..1...0...11 29 * read-write, dirty, old 110.001110.1 -> 10..0...1...11 30 * read-write, dirty, young 100.001111.1 -> 11..0...0...11 31 * HW-bits: R read-only, I invalid 32 * SW-bits: p present, y young, d dirty, r read, w write, s special, 33 * u unused, l large 34 */ 35 if (pte_present(pte)) { 36 pmd_val(pmd) = pte_val(pte) & PAGE_MASK; 37 pmd_val(pmd) |= (pte_val(pte) & _PAGE_READ) >> 4; 38 pmd_val(pmd) |= (pte_val(pte) & _PAGE_WRITE) >> 4; 39 pmd_val(pmd) |= (pte_val(pte) & _PAGE_INVALID) >> 5; 40 pmd_val(pmd) |= (pte_val(pte) & _PAGE_PROTECT); 41 pmd_val(pmd) |= (pte_val(pte) & _PAGE_DIRTY) << 10; 42 pmd_val(pmd) |= (pte_val(pte) & _PAGE_YOUNG) << 10; 43 } else 44 pmd_val(pmd) = _SEGMENT_ENTRY_INVALID; 45 return pmd; 46 } 47 48 static inline pte_t __pmd_to_pte(pmd_t pmd) 49 { 50 pte_t pte; 51 52 /* 53 * Convert encoding pmd bits pte bits 54 * dy..R...I...wr lIR.uswrdy.p 55 * empty 00..0...1...00 -> 010.000000.0 56 * prot-none, clean, old 00..1...1...00 -> 111.000000.1 57 * prot-none, clean, young 01..1...1...00 -> 111.000001.1 58 * prot-none, dirty, old 10..1...1...00 -> 111.000010.1 59 * prot-none, dirty, young 11..1...1...00 -> 111.000011.1 60 * read-only, clean, old 00..1...1...01 -> 111.000100.1 61 * read-only, clean, young 01..1...0...01 -> 101.000101.1 62 * read-only, dirty, old 10..1...1...01 -> 111.000110.1 63 * read-only, dirty, young 11..1...0...01 -> 101.000111.1 64 * read-write, clean, old 00..1...1...11 -> 111.001100.1 65 * read-write, clean, young 01..1...0...11 -> 101.001101.1 66 * read-write, dirty, old 10..0...1...11 -> 110.001110.1 67 * read-write, dirty, young 11..0...0...11 -> 100.001111.1 68 * HW-bits: R read-only, I invalid 69 * SW-bits: p present, y young, d dirty, r read, w write, s special, 70 * u unused, l large 71 */ 72 if (pmd_present(pmd)) { 73 pte_val(pte) = pmd_val(pmd) & _SEGMENT_ENTRY_ORIGIN_LARGE; 74 pte_val(pte) |= _PAGE_LARGE | _PAGE_PRESENT; 75 pte_val(pte) |= (pmd_val(pmd) & _SEGMENT_ENTRY_READ) << 4; 76 pte_val(pte) |= (pmd_val(pmd) & _SEGMENT_ENTRY_WRITE) << 4; 77 pte_val(pte) |= (pmd_val(pmd) & _SEGMENT_ENTRY_INVALID) << 5; 78 pte_val(pte) |= (pmd_val(pmd) & _SEGMENT_ENTRY_PROTECT); 79 pte_val(pte) |= (pmd_val(pmd) & _SEGMENT_ENTRY_DIRTY) >> 10; 80 pte_val(pte) |= (pmd_val(pmd) & _SEGMENT_ENTRY_YOUNG) >> 10; 81 } else 82 pte_val(pte) = _PAGE_INVALID; 83 return pte; 84 } 85 86 void set_huge_pte_at(struct mm_struct *mm, unsigned long addr, 87 pte_t *ptep, pte_t pte) 88 { 89 pmd_t pmd; 90 91 pmd = __pte_to_pmd(pte); 92 if (!MACHINE_HAS_HPAGE) { 93 /* Emulated huge ptes loose the dirty and young bit */ 94 pmd_val(pmd) &= ~_SEGMENT_ENTRY_ORIGIN; 95 pmd_val(pmd) |= pte_page(pte)[1].index; 96 } else 97 pmd_val(pmd) |= _SEGMENT_ENTRY_LARGE; 98 *(pmd_t *) ptep = pmd; 99 } 100 101 pte_t huge_ptep_get(pte_t *ptep) 102 { 103 unsigned long origin; 104 pmd_t pmd; 105 106 pmd = *(pmd_t *) ptep; 107 if (!MACHINE_HAS_HPAGE && pmd_present(pmd)) { 108 origin = pmd_val(pmd) & _SEGMENT_ENTRY_ORIGIN; 109 pmd_val(pmd) &= ~_SEGMENT_ENTRY_ORIGIN; 110 pmd_val(pmd) |= *(unsigned long *) origin; 111 /* Emulated huge ptes are young and dirty by definition */ 112 pmd_val(pmd) |= _SEGMENT_ENTRY_YOUNG | _SEGMENT_ENTRY_DIRTY; 113 } 114 return __pmd_to_pte(pmd); 115 } 116 117 pte_t huge_ptep_get_and_clear(struct mm_struct *mm, 118 unsigned long addr, pte_t *ptep) 119 { 120 pmd_t *pmdp = (pmd_t *) ptep; 121 pte_t pte = huge_ptep_get(ptep); 122 123 pmdp_flush_direct(mm, addr, pmdp); 124 pmd_val(*pmdp) = _SEGMENT_ENTRY_EMPTY; 125 return pte; 126 } 127 128 int arch_prepare_hugepage(struct page *page) 129 { 130 unsigned long addr = page_to_phys(page); 131 pte_t pte; 132 pte_t *ptep; 133 int i; 134 135 if (MACHINE_HAS_HPAGE) 136 return 0; 137 138 ptep = (pte_t *) pte_alloc_one(&init_mm, addr); 139 if (!ptep) 140 return -ENOMEM; 141 142 pte_val(pte) = addr; 143 for (i = 0; i < PTRS_PER_PTE; i++) { 144 set_pte_at(&init_mm, addr + i * PAGE_SIZE, ptep + i, pte); 145 pte_val(pte) += PAGE_SIZE; 146 } 147 page[1].index = (unsigned long) ptep; 148 return 0; 149 } 150 151 void arch_release_hugepage(struct page *page) 152 { 153 pte_t *ptep; 154 155 if (MACHINE_HAS_HPAGE) 156 return; 157 158 ptep = (pte_t *) page[1].index; 159 if (!ptep) 160 return; 161 clear_table((unsigned long *) ptep, _PAGE_INVALID, 162 PTRS_PER_PTE * sizeof(pte_t)); 163 page_table_free(&init_mm, (unsigned long *) ptep); 164 page[1].index = 0; 165 } 166 167 pte_t *huge_pte_alloc(struct mm_struct *mm, 168 unsigned long addr, unsigned long sz) 169 { 170 pgd_t *pgdp; 171 pud_t *pudp; 172 pmd_t *pmdp = NULL; 173 174 pgdp = pgd_offset(mm, addr); 175 pudp = pud_alloc(mm, pgdp, addr); 176 if (pudp) 177 pmdp = pmd_alloc(mm, pudp, addr); 178 return (pte_t *) pmdp; 179 } 180 181 pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr) 182 { 183 pgd_t *pgdp; 184 pud_t *pudp; 185 pmd_t *pmdp = NULL; 186 187 pgdp = pgd_offset(mm, addr); 188 if (pgd_present(*pgdp)) { 189 pudp = pud_offset(pgdp, addr); 190 if (pud_present(*pudp)) 191 pmdp = pmd_offset(pudp, addr); 192 } 193 return (pte_t *) pmdp; 194 } 195 196 int huge_pmd_unshare(struct mm_struct *mm, unsigned long *addr, pte_t *ptep) 197 { 198 return 0; 199 } 200 201 int pmd_huge(pmd_t pmd) 202 { 203 if (!MACHINE_HAS_HPAGE) 204 return 0; 205 206 return !!(pmd_val(pmd) & _SEGMENT_ENTRY_LARGE); 207 } 208 209 int pud_huge(pud_t pud) 210 { 211 return 0; 212 } 213