1 // SPDX-License-Identifier: GPL-2.0 2 #include <linux/sched/task.h> 3 #include <linux/pgtable.h> 4 #include <linux/kasan.h> 5 #include <asm/pgalloc.h> 6 #include <asm/facility.h> 7 #include <asm/sections.h> 8 #include <asm/physmem_info.h> 9 #include <asm/maccess.h> 10 #include <asm/abs_lowcore.h> 11 #include "decompressor.h" 12 #include "boot.h" 13 14 unsigned long __bootdata_preserved(s390_invalid_asce); 15 16 #ifdef CONFIG_PROC_FS 17 atomic_long_t __bootdata_preserved(direct_pages_count[PG_DIRECT_MAP_MAX]); 18 #endif 19 20 #define init_mm (*(struct mm_struct *)vmlinux.init_mm_off) 21 #define swapper_pg_dir vmlinux.swapper_pg_dir_off 22 #define invalid_pg_dir vmlinux.invalid_pg_dir_off 23 24 enum populate_mode { 25 POPULATE_NONE, 26 POPULATE_DIRECT, 27 POPULATE_ABS_LOWCORE, 28 #ifdef CONFIG_KASAN 29 POPULATE_KASAN_MAP_SHADOW, 30 POPULATE_KASAN_ZERO_SHADOW, 31 POPULATE_KASAN_SHALLOW 32 #endif 33 }; 34 35 static void pgtable_populate(unsigned long addr, unsigned long end, enum populate_mode mode); 36 37 #ifdef CONFIG_KASAN 38 39 #define kasan_early_shadow_page vmlinux.kasan_early_shadow_page_off 40 #define kasan_early_shadow_pte ((pte_t *)vmlinux.kasan_early_shadow_pte_off) 41 #define kasan_early_shadow_pmd ((pmd_t *)vmlinux.kasan_early_shadow_pmd_off) 42 #define kasan_early_shadow_pud ((pud_t *)vmlinux.kasan_early_shadow_pud_off) 43 #define kasan_early_shadow_p4d ((p4d_t *)vmlinux.kasan_early_shadow_p4d_off) 44 #define __sha(x) ((unsigned long)kasan_mem_to_shadow((void *)x)) 45 46 static pte_t pte_z; 47 48 static void kasan_populate_shadow(void) 49 { 50 pmd_t pmd_z = __pmd(__pa(kasan_early_shadow_pte) | _SEGMENT_ENTRY); 51 pud_t pud_z = __pud(__pa(kasan_early_shadow_pmd) | _REGION3_ENTRY); 52 p4d_t p4d_z = __p4d(__pa(kasan_early_shadow_pud) | _REGION2_ENTRY); 53 unsigned long untracked_end; 54 unsigned long start, end; 55 int i; 56 57 pte_z = __pte(__pa(kasan_early_shadow_page) | pgprot_val(PAGE_KERNEL_RO)); 58 if (!machine.has_nx) 59 pte_z = clear_pte_bit(pte_z, __pgprot(_PAGE_NOEXEC)); 60 crst_table_init((unsigned long *)kasan_early_shadow_p4d, p4d_val(p4d_z)); 61 crst_table_init((unsigned long *)kasan_early_shadow_pud, pud_val(pud_z)); 62 crst_table_init((unsigned long *)kasan_early_shadow_pmd, pmd_val(pmd_z)); 63 memset64((u64 *)kasan_early_shadow_pte, pte_val(pte_z), PTRS_PER_PTE); 64 65 /* 66 * Current memory layout: 67 * +- 0 -------------+ +- shadow start -+ 68 * |1:1 ident mapping| /|1/8 of ident map| 69 * | | / | | 70 * +-end of ident map+ / +----------------+ 71 * | ... gap ... | / | kasan | 72 * | | / | zero page | 73 * +- vmalloc area -+ / | mapping | 74 * | vmalloc_size | / | (untracked) | 75 * +- modules vaddr -+ / +----------------+ 76 * | 2Gb |/ | unmapped | allocated per module 77 * +- shadow start -+ +----------------+ 78 * | 1/8 addr space | | zero pg mapping| (untracked) 79 * +- shadow end ----+---------+- shadow end ---+ 80 * 81 * Current memory layout (KASAN_VMALLOC): 82 * +- 0 -------------+ +- shadow start -+ 83 * |1:1 ident mapping| /|1/8 of ident map| 84 * | | / | | 85 * +-end of ident map+ / +----------------+ 86 * | ... gap ... | / | kasan zero page| (untracked) 87 * | | / | mapping | 88 * +- vmalloc area -+ / +----------------+ 89 * | vmalloc_size | / |shallow populate| 90 * +- modules vaddr -+ / +----------------+ 91 * | 2Gb |/ |shallow populate| 92 * +- shadow start -+ +----------------+ 93 * | 1/8 addr space | | zero pg mapping| (untracked) 94 * +- shadow end ----+---------+- shadow end ---+ 95 */ 96 97 for_each_physmem_usable_range(i, &start, &end) 98 pgtable_populate(__sha(start), __sha(end), POPULATE_KASAN_MAP_SHADOW); 99 if (IS_ENABLED(CONFIG_KASAN_VMALLOC)) { 100 untracked_end = VMALLOC_START; 101 /* shallowly populate kasan shadow for vmalloc and modules */ 102 pgtable_populate(__sha(VMALLOC_START), __sha(MODULES_END), POPULATE_KASAN_SHALLOW); 103 } else { 104 untracked_end = MODULES_VADDR; 105 } 106 /* populate kasan shadow for untracked memory */ 107 pgtable_populate(__sha(ident_map_size), __sha(untracked_end), POPULATE_KASAN_ZERO_SHADOW); 108 pgtable_populate(__sha(MODULES_END), __sha(_REGION1_SIZE), POPULATE_KASAN_ZERO_SHADOW); 109 } 110 111 static bool kasan_pgd_populate_zero_shadow(pgd_t *pgd, unsigned long addr, 112 unsigned long end, enum populate_mode mode) 113 { 114 if (mode == POPULATE_KASAN_ZERO_SHADOW && 115 IS_ALIGNED(addr, PGDIR_SIZE) && end - addr >= PGDIR_SIZE) { 116 pgd_populate(&init_mm, pgd, kasan_early_shadow_p4d); 117 return true; 118 } 119 return false; 120 } 121 122 static bool kasan_p4d_populate_zero_shadow(p4d_t *p4d, unsigned long addr, 123 unsigned long end, enum populate_mode mode) 124 { 125 if (mode == POPULATE_KASAN_ZERO_SHADOW && 126 IS_ALIGNED(addr, P4D_SIZE) && end - addr >= P4D_SIZE) { 127 p4d_populate(&init_mm, p4d, kasan_early_shadow_pud); 128 return true; 129 } 130 return false; 131 } 132 133 static bool kasan_pud_populate_zero_shadow(pud_t *pud, unsigned long addr, 134 unsigned long end, enum populate_mode mode) 135 { 136 if (mode == POPULATE_KASAN_ZERO_SHADOW && 137 IS_ALIGNED(addr, PUD_SIZE) && end - addr >= PUD_SIZE) { 138 pud_populate(&init_mm, pud, kasan_early_shadow_pmd); 139 return true; 140 } 141 return false; 142 } 143 144 static bool kasan_pmd_populate_zero_shadow(pmd_t *pmd, unsigned long addr, 145 unsigned long end, enum populate_mode mode) 146 { 147 if (mode == POPULATE_KASAN_ZERO_SHADOW && 148 IS_ALIGNED(addr, PMD_SIZE) && end - addr >= PMD_SIZE) { 149 pmd_populate(&init_mm, pmd, kasan_early_shadow_pte); 150 return true; 151 } 152 return false; 153 } 154 155 static bool kasan_pte_populate_zero_shadow(pte_t *pte, enum populate_mode mode) 156 { 157 pte_t entry; 158 159 if (mode == POPULATE_KASAN_ZERO_SHADOW) { 160 set_pte(pte, pte_z); 161 return true; 162 } 163 return false; 164 } 165 #else 166 167 static inline void kasan_populate_shadow(void) {} 168 169 static inline bool kasan_pgd_populate_zero_shadow(pgd_t *pgd, unsigned long addr, 170 unsigned long end, enum populate_mode mode) 171 { 172 return false; 173 } 174 175 static inline bool kasan_p4d_populate_zero_shadow(p4d_t *p4d, unsigned long addr, 176 unsigned long end, enum populate_mode mode) 177 { 178 return false; 179 } 180 181 static inline bool kasan_pud_populate_zero_shadow(pud_t *pud, unsigned long addr, 182 unsigned long end, enum populate_mode mode) 183 { 184 return false; 185 } 186 187 static inline bool kasan_pmd_populate_zero_shadow(pmd_t *pmd, unsigned long addr, 188 unsigned long end, enum populate_mode mode) 189 { 190 return false; 191 } 192 193 static bool kasan_pte_populate_zero_shadow(pte_t *pte, enum populate_mode mode) 194 { 195 return false; 196 } 197 198 #endif 199 200 /* 201 * Mimic virt_to_kpte() in lack of init_mm symbol. Skip pmd NULL check though. 202 */ 203 static inline pte_t *__virt_to_kpte(unsigned long va) 204 { 205 return pte_offset_kernel(pmd_offset(pud_offset(p4d_offset(pgd_offset_k(va), va), va), va), va); 206 } 207 208 static void *boot_crst_alloc(unsigned long val) 209 { 210 unsigned long size = PAGE_SIZE << CRST_ALLOC_ORDER; 211 unsigned long *table; 212 213 table = (unsigned long *)physmem_alloc_top_down(RR_VMEM, size, size); 214 crst_table_init(table, val); 215 return table; 216 } 217 218 static pte_t *boot_pte_alloc(void) 219 { 220 static void *pte_leftover; 221 pte_t *pte; 222 223 /* 224 * handling pte_leftovers this way helps to avoid memory fragmentation 225 * during POPULATE_KASAN_MAP_SHADOW when EDAT is off 226 */ 227 if (!pte_leftover) { 228 pte_leftover = (void *)physmem_alloc_top_down(RR_VMEM, PAGE_SIZE, PAGE_SIZE); 229 pte = pte_leftover + _PAGE_TABLE_SIZE; 230 } else { 231 pte = pte_leftover; 232 pte_leftover = NULL; 233 } 234 235 memset64((u64 *)pte, _PAGE_INVALID, PTRS_PER_PTE); 236 return pte; 237 } 238 239 static unsigned long _pa(unsigned long addr, unsigned long size, enum populate_mode mode) 240 { 241 switch (mode) { 242 case POPULATE_NONE: 243 return -1; 244 case POPULATE_DIRECT: 245 return addr; 246 case POPULATE_ABS_LOWCORE: 247 return __abs_lowcore_pa(addr); 248 #ifdef CONFIG_KASAN 249 case POPULATE_KASAN_MAP_SHADOW: 250 addr = physmem_alloc_top_down(RR_VMEM, size, size); 251 memset((void *)addr, 0, size); 252 return addr; 253 #endif 254 default: 255 return -1; 256 } 257 } 258 259 static bool can_large_pud(pud_t *pu_dir, unsigned long addr, unsigned long end) 260 { 261 return machine.has_edat2 && 262 IS_ALIGNED(addr, PUD_SIZE) && (end - addr) >= PUD_SIZE; 263 } 264 265 static bool can_large_pmd(pmd_t *pm_dir, unsigned long addr, unsigned long end) 266 { 267 return machine.has_edat1 && 268 IS_ALIGNED(addr, PMD_SIZE) && (end - addr) >= PMD_SIZE; 269 } 270 271 static void pgtable_pte_populate(pmd_t *pmd, unsigned long addr, unsigned long end, 272 enum populate_mode mode) 273 { 274 unsigned long pages = 0; 275 pte_t *pte, entry; 276 277 pte = pte_offset_kernel(pmd, addr); 278 for (; addr < end; addr += PAGE_SIZE, pte++) { 279 if (pte_none(*pte)) { 280 if (kasan_pte_populate_zero_shadow(pte, mode)) 281 continue; 282 entry = __pte(_pa(addr, PAGE_SIZE, mode)); 283 entry = set_pte_bit(entry, PAGE_KERNEL_EXEC); 284 set_pte(pte, entry); 285 pages++; 286 } 287 } 288 if (mode == POPULATE_DIRECT) 289 update_page_count(PG_DIRECT_MAP_4K, pages); 290 } 291 292 static void pgtable_pmd_populate(pud_t *pud, unsigned long addr, unsigned long end, 293 enum populate_mode mode) 294 { 295 unsigned long next, pages = 0; 296 pmd_t *pmd, entry; 297 pte_t *pte; 298 299 pmd = pmd_offset(pud, addr); 300 for (; addr < end; addr = next, pmd++) { 301 next = pmd_addr_end(addr, end); 302 if (pmd_none(*pmd)) { 303 if (kasan_pmd_populate_zero_shadow(pmd, addr, next, mode)) 304 continue; 305 if (can_large_pmd(pmd, addr, next)) { 306 entry = __pmd(_pa(addr, _SEGMENT_SIZE, mode)); 307 entry = set_pmd_bit(entry, SEGMENT_KERNEL_EXEC); 308 set_pmd(pmd, entry); 309 pages++; 310 continue; 311 } 312 pte = boot_pte_alloc(); 313 pmd_populate(&init_mm, pmd, pte); 314 } else if (pmd_large(*pmd)) { 315 continue; 316 } 317 pgtable_pte_populate(pmd, addr, next, mode); 318 } 319 if (mode == POPULATE_DIRECT) 320 update_page_count(PG_DIRECT_MAP_1M, pages); 321 } 322 323 static void pgtable_pud_populate(p4d_t *p4d, unsigned long addr, unsigned long end, 324 enum populate_mode mode) 325 { 326 unsigned long next, pages = 0; 327 pud_t *pud, entry; 328 pmd_t *pmd; 329 330 pud = pud_offset(p4d, addr); 331 for (; addr < end; addr = next, pud++) { 332 next = pud_addr_end(addr, end); 333 if (pud_none(*pud)) { 334 if (kasan_pud_populate_zero_shadow(pud, addr, next, mode)) 335 continue; 336 if (can_large_pud(pud, addr, next)) { 337 entry = __pud(_pa(addr, _REGION3_SIZE, mode)); 338 entry = set_pud_bit(entry, REGION3_KERNEL_EXEC); 339 set_pud(pud, entry); 340 pages++; 341 continue; 342 } 343 pmd = boot_crst_alloc(_SEGMENT_ENTRY_EMPTY); 344 pud_populate(&init_mm, pud, pmd); 345 } else if (pud_large(*pud)) { 346 continue; 347 } 348 pgtable_pmd_populate(pud, addr, next, mode); 349 } 350 if (mode == POPULATE_DIRECT) 351 update_page_count(PG_DIRECT_MAP_2G, pages); 352 } 353 354 static void pgtable_p4d_populate(pgd_t *pgd, unsigned long addr, unsigned long end, 355 enum populate_mode mode) 356 { 357 unsigned long next; 358 p4d_t *p4d; 359 pud_t *pud; 360 361 p4d = p4d_offset(pgd, addr); 362 for (; addr < end; addr = next, p4d++) { 363 next = p4d_addr_end(addr, end); 364 if (p4d_none(*p4d)) { 365 if (kasan_p4d_populate_zero_shadow(p4d, addr, next, mode)) 366 continue; 367 pud = boot_crst_alloc(_REGION3_ENTRY_EMPTY); 368 p4d_populate(&init_mm, p4d, pud); 369 } 370 pgtable_pud_populate(p4d, addr, next, mode); 371 } 372 } 373 374 static void pgtable_populate(unsigned long addr, unsigned long end, enum populate_mode mode) 375 { 376 unsigned long next; 377 pgd_t *pgd; 378 p4d_t *p4d; 379 380 pgd = pgd_offset(&init_mm, addr); 381 for (; addr < end; addr = next, pgd++) { 382 next = pgd_addr_end(addr, end); 383 if (pgd_none(*pgd)) { 384 if (kasan_pgd_populate_zero_shadow(pgd, addr, next, mode)) 385 continue; 386 p4d = boot_crst_alloc(_REGION2_ENTRY_EMPTY); 387 pgd_populate(&init_mm, pgd, p4d); 388 } 389 #ifdef CONFIG_KASAN 390 if (mode == POPULATE_KASAN_SHALLOW) 391 continue; 392 #endif 393 pgtable_p4d_populate(pgd, addr, next, mode); 394 } 395 } 396 397 void setup_vmem(unsigned long asce_limit) 398 { 399 unsigned long start, end; 400 unsigned long asce_type; 401 unsigned long asce_bits; 402 int i; 403 404 if (asce_limit == _REGION1_SIZE) { 405 asce_type = _REGION2_ENTRY_EMPTY; 406 asce_bits = _ASCE_TYPE_REGION2 | _ASCE_TABLE_LENGTH; 407 } else { 408 asce_type = _REGION3_ENTRY_EMPTY; 409 asce_bits = _ASCE_TYPE_REGION3 | _ASCE_TABLE_LENGTH; 410 } 411 s390_invalid_asce = invalid_pg_dir | _ASCE_TYPE_REGION3 | _ASCE_TABLE_LENGTH; 412 413 crst_table_init((unsigned long *)swapper_pg_dir, asce_type); 414 crst_table_init((unsigned long *)invalid_pg_dir, _REGION3_ENTRY_EMPTY); 415 416 /* 417 * To allow prefixing the lowcore must be mapped with 4KB pages. 418 * To prevent creation of a large page at address 0 first map 419 * the lowcore and create the identity mapping only afterwards. 420 */ 421 pgtable_populate(0, sizeof(struct lowcore), POPULATE_DIRECT); 422 for_each_physmem_usable_range(i, &start, &end) 423 pgtable_populate(start, end, POPULATE_DIRECT); 424 pgtable_populate(__abs_lowcore, __abs_lowcore + sizeof(struct lowcore), 425 POPULATE_ABS_LOWCORE); 426 pgtable_populate(__memcpy_real_area, __memcpy_real_area + PAGE_SIZE, 427 POPULATE_NONE); 428 memcpy_real_ptep = __virt_to_kpte(__memcpy_real_area); 429 430 kasan_populate_shadow(); 431 432 S390_lowcore.kernel_asce = swapper_pg_dir | asce_bits; 433 S390_lowcore.user_asce = s390_invalid_asce; 434 435 __ctl_load(S390_lowcore.kernel_asce, 1, 1); 436 __ctl_load(S390_lowcore.user_asce, 7, 7); 437 __ctl_load(S390_lowcore.kernel_asce, 13, 13); 438 439 init_mm.context.asce = S390_lowcore.kernel_asce; 440 } 441