1 // SPDX-License-Identifier: GPL-2.0 2 3 /* 4 * Transitional page tables for kexec and hibernate 5 * 6 * This file derived from: arch/arm64/kernel/hibernate.c 7 * 8 * Copyright (c) 2021, Microsoft Corporation. 9 * Pasha Tatashin <pasha.tatashin@soleen.com> 10 * 11 */ 12 13 /* 14 * Transitional tables are used during system transferring from one world to 15 * another: such as during hibernate restore, and kexec reboots. During these 16 * phases one cannot rely on page table not being overwritten. This is because 17 * hibernate and kexec can overwrite the current page tables during transition. 18 */ 19 20 #include <asm/trans_pgd.h> 21 #include <asm/pgalloc.h> 22 #include <asm/pgtable.h> 23 #include <linux/suspend.h> 24 #include <linux/bug.h> 25 #include <linux/mm.h> 26 #include <linux/mmzone.h> 27 28 static void *trans_alloc(struct trans_pgd_info *info) 29 { 30 return info->trans_alloc_page(info->trans_alloc_arg); 31 } 32 33 static void _copy_pte(pte_t *dst_ptep, pte_t *src_ptep, unsigned long addr) 34 { 35 pte_t pte = READ_ONCE(*src_ptep); 36 37 if (pte_valid(pte)) { 38 /* 39 * Resume will overwrite areas that may be marked 40 * read only (code, rodata). Clear the RDONLY bit from 41 * the temporary mappings we use during restore. 42 */ 43 set_pte(dst_ptep, pte_mkwrite(pte)); 44 } else if (debug_pagealloc_enabled() && !pte_none(pte)) { 45 /* 46 * debug_pagealloc will removed the PTE_VALID bit if 47 * the page isn't in use by the resume kernel. It may have 48 * been in use by the original kernel, in which case we need 49 * to put it back in our copy to do the restore. 50 * 51 * Before marking this entry valid, check the pfn should 52 * be mapped. 53 */ 54 BUG_ON(!pfn_valid(pte_pfn(pte))); 55 56 set_pte(dst_ptep, pte_mkpresent(pte_mkwrite(pte))); 57 } 58 } 59 60 static int copy_pte(struct trans_pgd_info *info, pmd_t *dst_pmdp, 61 pmd_t *src_pmdp, unsigned long start, unsigned long end) 62 { 63 pte_t *src_ptep; 64 pte_t *dst_ptep; 65 unsigned long addr = start; 66 67 dst_ptep = trans_alloc(info); 68 if (!dst_ptep) 69 return -ENOMEM; 70 pmd_populate_kernel(NULL, dst_pmdp, dst_ptep); 71 dst_ptep = pte_offset_kernel(dst_pmdp, start); 72 73 src_ptep = pte_offset_kernel(src_pmdp, start); 74 do { 75 _copy_pte(dst_ptep, src_ptep, addr); 76 } while (dst_ptep++, src_ptep++, addr += PAGE_SIZE, addr != end); 77 78 return 0; 79 } 80 81 static int copy_pmd(struct trans_pgd_info *info, pud_t *dst_pudp, 82 pud_t *src_pudp, unsigned long start, unsigned long end) 83 { 84 pmd_t *src_pmdp; 85 pmd_t *dst_pmdp; 86 unsigned long next; 87 unsigned long addr = start; 88 89 if (pud_none(READ_ONCE(*dst_pudp))) { 90 dst_pmdp = trans_alloc(info); 91 if (!dst_pmdp) 92 return -ENOMEM; 93 pud_populate(NULL, dst_pudp, dst_pmdp); 94 } 95 dst_pmdp = pmd_offset(dst_pudp, start); 96 97 src_pmdp = pmd_offset(src_pudp, start); 98 do { 99 pmd_t pmd = READ_ONCE(*src_pmdp); 100 101 next = pmd_addr_end(addr, end); 102 if (pmd_none(pmd)) 103 continue; 104 if (pmd_table(pmd)) { 105 if (copy_pte(info, dst_pmdp, src_pmdp, addr, next)) 106 return -ENOMEM; 107 } else { 108 set_pmd(dst_pmdp, 109 __pmd(pmd_val(pmd) & ~PMD_SECT_RDONLY)); 110 } 111 } while (dst_pmdp++, src_pmdp++, addr = next, addr != end); 112 113 return 0; 114 } 115 116 static int copy_pud(struct trans_pgd_info *info, p4d_t *dst_p4dp, 117 p4d_t *src_p4dp, unsigned long start, 118 unsigned long end) 119 { 120 pud_t *dst_pudp; 121 pud_t *src_pudp; 122 unsigned long next; 123 unsigned long addr = start; 124 125 if (p4d_none(READ_ONCE(*dst_p4dp))) { 126 dst_pudp = trans_alloc(info); 127 if (!dst_pudp) 128 return -ENOMEM; 129 p4d_populate(NULL, dst_p4dp, dst_pudp); 130 } 131 dst_pudp = pud_offset(dst_p4dp, start); 132 133 src_pudp = pud_offset(src_p4dp, start); 134 do { 135 pud_t pud = READ_ONCE(*src_pudp); 136 137 next = pud_addr_end(addr, end); 138 if (pud_none(pud)) 139 continue; 140 if (pud_table(pud)) { 141 if (copy_pmd(info, dst_pudp, src_pudp, addr, next)) 142 return -ENOMEM; 143 } else { 144 set_pud(dst_pudp, 145 __pud(pud_val(pud) & ~PUD_SECT_RDONLY)); 146 } 147 } while (dst_pudp++, src_pudp++, addr = next, addr != end); 148 149 return 0; 150 } 151 152 static int copy_p4d(struct trans_pgd_info *info, pgd_t *dst_pgdp, 153 pgd_t *src_pgdp, unsigned long start, 154 unsigned long end) 155 { 156 p4d_t *dst_p4dp; 157 p4d_t *src_p4dp; 158 unsigned long next; 159 unsigned long addr = start; 160 161 dst_p4dp = p4d_offset(dst_pgdp, start); 162 src_p4dp = p4d_offset(src_pgdp, start); 163 do { 164 next = p4d_addr_end(addr, end); 165 if (p4d_none(READ_ONCE(*src_p4dp))) 166 continue; 167 if (copy_pud(info, dst_p4dp, src_p4dp, addr, next)) 168 return -ENOMEM; 169 } while (dst_p4dp++, src_p4dp++, addr = next, addr != end); 170 171 return 0; 172 } 173 174 static int copy_page_tables(struct trans_pgd_info *info, pgd_t *dst_pgdp, 175 unsigned long start, unsigned long end) 176 { 177 unsigned long next; 178 unsigned long addr = start; 179 pgd_t *src_pgdp = pgd_offset_k(start); 180 181 dst_pgdp = pgd_offset_pgd(dst_pgdp, start); 182 do { 183 next = pgd_addr_end(addr, end); 184 if (pgd_none(READ_ONCE(*src_pgdp))) 185 continue; 186 if (copy_p4d(info, dst_pgdp, src_pgdp, addr, next)) 187 return -ENOMEM; 188 } while (dst_pgdp++, src_pgdp++, addr = next, addr != end); 189 190 return 0; 191 } 192 193 /* 194 * Create trans_pgd and copy linear map. 195 * info: contains allocator and its argument 196 * dst_pgdp: new page table that is created, and to which map is copied. 197 * start: Start of the interval (inclusive). 198 * end: End of the interval (exclusive). 199 * 200 * Returns 0 on success, and -ENOMEM on failure. 201 */ 202 int trans_pgd_create_copy(struct trans_pgd_info *info, pgd_t **dst_pgdp, 203 unsigned long start, unsigned long end) 204 { 205 int rc; 206 pgd_t *trans_pgd = trans_alloc(info); 207 208 if (!trans_pgd) { 209 pr_err("Failed to allocate memory for temporary page tables.\n"); 210 return -ENOMEM; 211 } 212 213 rc = copy_page_tables(info, trans_pgd, start, end); 214 if (!rc) 215 *dst_pgdp = trans_pgd; 216 217 return rc; 218 } 219 220 /* 221 * The page we want to idmap may be outside the range covered by VA_BITS that 222 * can be built using the kernel's p?d_populate() helpers. As a one off, for a 223 * single page, we build these page tables bottom up and just assume that will 224 * need the maximum T0SZ. 225 * 226 * Returns 0 on success, and -ENOMEM on failure. 227 * On success trans_ttbr0 contains page table with idmapped page, t0sz is set to 228 * maximum T0SZ for this page. 229 */ 230 int trans_pgd_idmap_page(struct trans_pgd_info *info, phys_addr_t *trans_ttbr0, 231 unsigned long *t0sz, void *page) 232 { 233 phys_addr_t dst_addr = virt_to_phys(page); 234 unsigned long pfn = __phys_to_pfn(dst_addr); 235 int max_msb = (dst_addr & GENMASK(52, 48)) ? 51 : 47; 236 int bits_mapped = PAGE_SHIFT - 4; 237 unsigned long level_mask, prev_level_entry, *levels[4]; 238 int this_level, index, level_lsb, level_msb; 239 240 dst_addr &= PAGE_MASK; 241 prev_level_entry = pte_val(pfn_pte(pfn, PAGE_KERNEL_EXEC)); 242 243 for (this_level = 3; this_level >= 0; this_level--) { 244 levels[this_level] = trans_alloc(info); 245 if (!levels[this_level]) 246 return -ENOMEM; 247 248 level_lsb = ARM64_HW_PGTABLE_LEVEL_SHIFT(this_level); 249 level_msb = min(level_lsb + bits_mapped, max_msb); 250 level_mask = GENMASK_ULL(level_msb, level_lsb); 251 252 index = (dst_addr & level_mask) >> level_lsb; 253 *(levels[this_level] + index) = prev_level_entry; 254 255 pfn = virt_to_pfn(levels[this_level]); 256 prev_level_entry = pte_val(pfn_pte(pfn, 257 __pgprot(PMD_TYPE_TABLE))); 258 259 if (level_msb == max_msb) 260 break; 261 } 262 263 *trans_ttbr0 = phys_to_ttbr(__pfn_to_phys(pfn)); 264 *t0sz = TCR_T0SZ(max_msb + 1); 265 266 return 0; 267 } 268 269 /* 270 * Create a copy of the vector table so we can call HVC_SET_VECTORS or 271 * HVC_SOFT_RESTART from contexts where the table may be overwritten. 272 */ 273 int trans_pgd_copy_el2_vectors(struct trans_pgd_info *info, 274 phys_addr_t *el2_vectors) 275 { 276 void *hyp_stub = trans_alloc(info); 277 278 if (!hyp_stub) 279 return -ENOMEM; 280 *el2_vectors = virt_to_phys(hyp_stub); 281 memcpy(hyp_stub, &trans_pgd_stub_vectors, ARM64_VECTOR_TABLE_LEN); 282 caches_clean_inval_pou((unsigned long)hyp_stub, 283 (unsigned long)hyp_stub + 284 ARM64_VECTOR_TABLE_LEN); 285 dcache_clean_inval_poc((unsigned long)hyp_stub, 286 (unsigned long)hyp_stub + 287 ARM64_VECTOR_TABLE_LEN); 288 289 return 0; 290 } 291