1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * linux/arch/arm/mm/pgd.c 4 * 5 * Copyright (C) 1998-2005 Russell King 6 */ 7 #include <linux/mm.h> 8 #include <linux/gfp.h> 9 #include <linux/highmem.h> 10 #include <linux/slab.h> 11 12 #include <asm/cp15.h> 13 #include <asm/pgalloc.h> 14 #include <asm/page.h> 15 #include <asm/tlbflush.h> 16 17 #include "mm.h" 18 19 #ifdef CONFIG_ARM_LPAE 20 #define __pgd_alloc() kmalloc_array(PTRS_PER_PGD, sizeof(pgd_t), GFP_KERNEL) 21 #define __pgd_free(pgd) kfree(pgd) 22 #else 23 #define __pgd_alloc() (pgd_t *)__get_free_pages(GFP_KERNEL, 2) 24 #define __pgd_free(pgd) free_pages((unsigned long)pgd, 2) 25 #endif 26 27 /* 28 * need to get a 16k page for level 1 29 */ 30 pgd_t *pgd_alloc(struct mm_struct *mm) 31 { 32 pgd_t *new_pgd, *init_pgd; 33 p4d_t *new_p4d, *init_p4d; 34 pud_t *new_pud, *init_pud; 35 pmd_t *new_pmd, *init_pmd; 36 pte_t *new_pte, *init_pte; 37 38 new_pgd = __pgd_alloc(); 39 if (!new_pgd) 40 goto no_pgd; 41 42 memset(new_pgd, 0, USER_PTRS_PER_PGD * sizeof(pgd_t)); 43 44 /* 45 * Copy over the kernel and IO PGD entries 46 */ 47 init_pgd = pgd_offset_k(0); 48 memcpy(new_pgd + USER_PTRS_PER_PGD, init_pgd + USER_PTRS_PER_PGD, 49 (PTRS_PER_PGD - USER_PTRS_PER_PGD) * sizeof(pgd_t)); 50 51 clean_dcache_area(new_pgd, PTRS_PER_PGD * sizeof(pgd_t)); 52 53 #ifdef CONFIG_ARM_LPAE 54 /* 55 * Allocate PMD table for modules and pkmap mappings. 56 */ 57 new_p4d = p4d_alloc(mm, new_pgd + pgd_index(MODULES_VADDR), 58 MODULES_VADDR); 59 if (!new_p4d) 60 goto no_p4d; 61 62 new_pud = pud_alloc(mm, new_p4d, MODULES_VADDR); 63 if (!new_pud) 64 goto no_pud; 65 66 new_pmd = pmd_alloc(mm, new_pud, 0); 67 if (!new_pmd) 68 goto no_pmd; 69 #endif 70 71 if (!vectors_high()) { 72 /* 73 * On ARM, first page must always be allocated since it 74 * contains the machine vectors. The vectors are always high 75 * with LPAE. 76 */ 77 new_p4d = p4d_alloc(mm, new_pgd, 0); 78 if (!new_p4d) 79 goto no_p4d; 80 81 new_pud = pud_alloc(mm, new_p4d, 0); 82 if (!new_pud) 83 goto no_pud; 84 85 new_pmd = pmd_alloc(mm, new_pud, 0); 86 if (!new_pmd) 87 goto no_pmd; 88 89 new_pte = pte_alloc_map(mm, new_pmd, 0); 90 if (!new_pte) 91 goto no_pte; 92 93 #ifndef CONFIG_ARM_LPAE 94 /* 95 * Modify the PTE pointer to have the correct domain. This 96 * needs to be the vectors domain to avoid the low vectors 97 * being unmapped. 98 */ 99 pmd_val(*new_pmd) &= ~PMD_DOMAIN_MASK; 100 pmd_val(*new_pmd) |= PMD_DOMAIN(DOMAIN_VECTORS); 101 #endif 102 103 init_p4d = p4d_offset(init_pgd, 0); 104 init_pud = pud_offset(init_p4d, 0); 105 init_pmd = pmd_offset(init_pud, 0); 106 init_pte = pte_offset_map(init_pmd, 0); 107 set_pte_ext(new_pte + 0, init_pte[0], 0); 108 set_pte_ext(new_pte + 1, init_pte[1], 0); 109 pte_unmap(init_pte); 110 pte_unmap(new_pte); 111 } 112 113 return new_pgd; 114 115 no_pte: 116 pmd_free(mm, new_pmd); 117 mm_dec_nr_pmds(mm); 118 no_pmd: 119 pud_free(mm, new_pud); 120 no_pud: 121 p4d_free(mm, new_p4d); 122 no_p4d: 123 __pgd_free(new_pgd); 124 no_pgd: 125 return NULL; 126 } 127 128 void pgd_free(struct mm_struct *mm, pgd_t *pgd_base) 129 { 130 pgd_t *pgd; 131 p4d_t *p4d; 132 pud_t *pud; 133 pmd_t *pmd; 134 pgtable_t pte; 135 136 if (!pgd_base) 137 return; 138 139 pgd = pgd_base + pgd_index(0); 140 if (pgd_none_or_clear_bad(pgd)) 141 goto no_pgd; 142 143 p4d = p4d_offset(pgd, 0); 144 if (p4d_none_or_clear_bad(p4d)) 145 goto no_p4d; 146 147 pud = pud_offset(p4d, 0); 148 if (pud_none_or_clear_bad(pud)) 149 goto no_pud; 150 151 pmd = pmd_offset(pud, 0); 152 if (pmd_none_or_clear_bad(pmd)) 153 goto no_pmd; 154 155 pte = pmd_pgtable(*pmd); 156 pmd_clear(pmd); 157 pte_free(mm, pte); 158 mm_dec_nr_ptes(mm); 159 no_pmd: 160 pud_clear(pud); 161 pmd_free(mm, pmd); 162 mm_dec_nr_pmds(mm); 163 no_pud: 164 p4d_clear(p4d); 165 pud_free(mm, pud); 166 no_p4d: 167 pgd_clear(pgd); 168 p4d_free(mm, p4d); 169 no_pgd: 170 #ifdef CONFIG_ARM_LPAE 171 /* 172 * Free modules/pkmap or identity pmd tables. 173 */ 174 for (pgd = pgd_base; pgd < pgd_base + PTRS_PER_PGD; pgd++) { 175 if (pgd_none_or_clear_bad(pgd)) 176 continue; 177 if (pgd_val(*pgd) & L_PGD_SWAPPER) 178 continue; 179 p4d = p4d_offset(pgd, 0); 180 if (p4d_none_or_clear_bad(p4d)) 181 continue; 182 pud = pud_offset(p4d, 0); 183 if (pud_none_or_clear_bad(pud)) 184 continue; 185 pmd = pmd_offset(pud, 0); 186 pud_clear(pud); 187 pmd_free(mm, pmd); 188 mm_dec_nr_pmds(mm); 189 p4d_clear(p4d); 190 pud_free(mm, pud); 191 mm_dec_nr_puds(mm); 192 pgd_clear(pgd); 193 p4d_free(mm, p4d); 194 } 195 #endif 196 __pgd_free(pgd_base); 197 } 198