1 // SPDX-License-Identifier: GPL-2.0 2 #include <linux/sched/task.h> 3 #include <linux/pgtable.h> 4 #include <linux/kasan.h> 5 #include <asm/pgalloc.h> 6 #include <asm/facility.h> 7 #include <asm/sections.h> 8 #include <asm/physmem_info.h> 9 #include <asm/maccess.h> 10 #include <asm/abs_lowcore.h> 11 #include "decompressor.h" 12 #include "boot.h" 13 14 unsigned long __bootdata_preserved(s390_invalid_asce); 15 16 #ifdef CONFIG_PROC_FS 17 atomic_long_t __bootdata_preserved(direct_pages_count[PG_DIRECT_MAP_MAX]); 18 #endif 19 20 #define init_mm (*(struct mm_struct *)vmlinux.init_mm_off) 21 #define swapper_pg_dir vmlinux.swapper_pg_dir_off 22 #define invalid_pg_dir vmlinux.invalid_pg_dir_off 23 24 enum populate_mode { 25 POPULATE_NONE, 26 POPULATE_DIRECT, 27 POPULATE_ABS_LOWCORE, 28 #ifdef CONFIG_KASAN 29 POPULATE_KASAN_MAP_SHADOW, 30 POPULATE_KASAN_ZERO_SHADOW, 31 POPULATE_KASAN_SHALLOW 32 #endif 33 }; 34 35 static void pgtable_populate(unsigned long addr, unsigned long end, enum populate_mode mode); 36 37 #ifdef CONFIG_KASAN 38 39 #define kasan_early_shadow_page vmlinux.kasan_early_shadow_page_off 40 #define kasan_early_shadow_pte ((pte_t *)vmlinux.kasan_early_shadow_pte_off) 41 #define kasan_early_shadow_pmd ((pmd_t *)vmlinux.kasan_early_shadow_pmd_off) 42 #define kasan_early_shadow_pud ((pud_t *)vmlinux.kasan_early_shadow_pud_off) 43 #define kasan_early_shadow_p4d ((p4d_t *)vmlinux.kasan_early_shadow_p4d_off) 44 #define __sha(x) ((unsigned long)kasan_mem_to_shadow((void *)x)) 45 46 static pte_t pte_z; 47 48 static inline void kasan_populate(unsigned long start, unsigned long end, enum populate_mode mode) 49 { 50 start = PAGE_ALIGN_DOWN(__sha(start)); 51 end = PAGE_ALIGN(__sha(end)); 52 pgtable_populate(start, end, mode); 53 } 54 55 static void kasan_populate_shadow(void) 56 { 57 pmd_t pmd_z = __pmd(__pa(kasan_early_shadow_pte) | _SEGMENT_ENTRY); 58 pud_t pud_z = __pud(__pa(kasan_early_shadow_pmd) | _REGION3_ENTRY); 59 p4d_t p4d_z = __p4d(__pa(kasan_early_shadow_pud) | _REGION2_ENTRY); 60 unsigned long memgap_start = 0; 61 unsigned long untracked_end; 62 unsigned long start, end; 63 int i; 64 65 pte_z = __pte(__pa(kasan_early_shadow_page) | pgprot_val(PAGE_KERNEL_RO)); 66 if (!machine.has_nx) 67 pte_z = clear_pte_bit(pte_z, __pgprot(_PAGE_NOEXEC)); 68 crst_table_init((unsigned long *)kasan_early_shadow_p4d, p4d_val(p4d_z)); 69 crst_table_init((unsigned long *)kasan_early_shadow_pud, pud_val(pud_z)); 70 crst_table_init((unsigned long *)kasan_early_shadow_pmd, pmd_val(pmd_z)); 71 memset64((u64 *)kasan_early_shadow_pte, pte_val(pte_z), PTRS_PER_PTE); 72 73 /* 74 * Current memory layout: 75 * +- 0 -------------+ +- shadow start -+ 76 * |1:1 ident mapping| /|1/8 of ident map| 77 * | | / | | 78 * +-end of ident map+ / +----------------+ 79 * | ... gap ... | / | kasan | 80 * | | / | zero page | 81 * +- vmalloc area -+ / | mapping | 82 * | vmalloc_size | / | (untracked) | 83 * +- modules vaddr -+ / +----------------+ 84 * | 2Gb |/ | unmapped | allocated per module 85 * +- shadow start -+ +----------------+ 86 * | 1/8 addr space | | zero pg mapping| (untracked) 87 * +- shadow end ----+---------+- shadow end ---+ 88 * 89 * Current memory layout (KASAN_VMALLOC): 90 * +- 0 -------------+ +- shadow start -+ 91 * |1:1 ident mapping| /|1/8 of ident map| 92 * | | / | | 93 * +-end of ident map+ / +----------------+ 94 * | ... gap ... | / | kasan zero page| (untracked) 95 * | | / | mapping | 96 * +- vmalloc area -+ / +----------------+ 97 * | vmalloc_size | / |shallow populate| 98 * +- modules vaddr -+ / +----------------+ 99 * | 2Gb |/ |shallow populate| 100 * +- shadow start -+ +----------------+ 101 * | 1/8 addr space | | zero pg mapping| (untracked) 102 * +- shadow end ----+---------+- shadow end ---+ 103 */ 104 105 for_each_physmem_usable_range(i, &start, &end) { 106 kasan_populate(start, end, POPULATE_KASAN_MAP_SHADOW); 107 if (memgap_start && physmem_info.info_source == MEM_DETECT_DIAG260) 108 kasan_populate(memgap_start, start, POPULATE_KASAN_ZERO_SHADOW); 109 memgap_start = end; 110 } 111 if (IS_ENABLED(CONFIG_KASAN_VMALLOC)) { 112 untracked_end = VMALLOC_START; 113 /* shallowly populate kasan shadow for vmalloc and modules */ 114 kasan_populate(VMALLOC_START, MODULES_END, POPULATE_KASAN_SHALLOW); 115 } else { 116 untracked_end = MODULES_VADDR; 117 } 118 /* populate kasan shadow for untracked memory */ 119 kasan_populate(ident_map_size, untracked_end, POPULATE_KASAN_ZERO_SHADOW); 120 kasan_populate(MODULES_END, _REGION1_SIZE, POPULATE_KASAN_ZERO_SHADOW); 121 } 122 123 static bool kasan_pgd_populate_zero_shadow(pgd_t *pgd, unsigned long addr, 124 unsigned long end, enum populate_mode mode) 125 { 126 if (mode == POPULATE_KASAN_ZERO_SHADOW && 127 IS_ALIGNED(addr, PGDIR_SIZE) && end - addr >= PGDIR_SIZE) { 128 pgd_populate(&init_mm, pgd, kasan_early_shadow_p4d); 129 return true; 130 } 131 return false; 132 } 133 134 static bool kasan_p4d_populate_zero_shadow(p4d_t *p4d, unsigned long addr, 135 unsigned long end, enum populate_mode mode) 136 { 137 if (mode == POPULATE_KASAN_ZERO_SHADOW && 138 IS_ALIGNED(addr, P4D_SIZE) && end - addr >= P4D_SIZE) { 139 p4d_populate(&init_mm, p4d, kasan_early_shadow_pud); 140 return true; 141 } 142 return false; 143 } 144 145 static bool kasan_pud_populate_zero_shadow(pud_t *pud, unsigned long addr, 146 unsigned long end, enum populate_mode mode) 147 { 148 if (mode == POPULATE_KASAN_ZERO_SHADOW && 149 IS_ALIGNED(addr, PUD_SIZE) && end - addr >= PUD_SIZE) { 150 pud_populate(&init_mm, pud, kasan_early_shadow_pmd); 151 return true; 152 } 153 return false; 154 } 155 156 static bool kasan_pmd_populate_zero_shadow(pmd_t *pmd, unsigned long addr, 157 unsigned long end, enum populate_mode mode) 158 { 159 if (mode == POPULATE_KASAN_ZERO_SHADOW && 160 IS_ALIGNED(addr, PMD_SIZE) && end - addr >= PMD_SIZE) { 161 pmd_populate(&init_mm, pmd, kasan_early_shadow_pte); 162 return true; 163 } 164 return false; 165 } 166 167 static bool kasan_pte_populate_zero_shadow(pte_t *pte, enum populate_mode mode) 168 { 169 pte_t entry; 170 171 if (mode == POPULATE_KASAN_ZERO_SHADOW) { 172 set_pte(pte, pte_z); 173 return true; 174 } 175 return false; 176 } 177 #else 178 179 static inline void kasan_populate_shadow(void) {} 180 181 static inline bool kasan_pgd_populate_zero_shadow(pgd_t *pgd, unsigned long addr, 182 unsigned long end, enum populate_mode mode) 183 { 184 return false; 185 } 186 187 static inline bool kasan_p4d_populate_zero_shadow(p4d_t *p4d, unsigned long addr, 188 unsigned long end, enum populate_mode mode) 189 { 190 return false; 191 } 192 193 static inline bool kasan_pud_populate_zero_shadow(pud_t *pud, unsigned long addr, 194 unsigned long end, enum populate_mode mode) 195 { 196 return false; 197 } 198 199 static inline bool kasan_pmd_populate_zero_shadow(pmd_t *pmd, unsigned long addr, 200 unsigned long end, enum populate_mode mode) 201 { 202 return false; 203 } 204 205 static bool kasan_pte_populate_zero_shadow(pte_t *pte, enum populate_mode mode) 206 { 207 return false; 208 } 209 210 #endif 211 212 /* 213 * Mimic virt_to_kpte() in lack of init_mm symbol. Skip pmd NULL check though. 214 */ 215 static inline pte_t *__virt_to_kpte(unsigned long va) 216 { 217 return pte_offset_kernel(pmd_offset(pud_offset(p4d_offset(pgd_offset_k(va), va), va), va), va); 218 } 219 220 static void *boot_crst_alloc(unsigned long val) 221 { 222 unsigned long size = PAGE_SIZE << CRST_ALLOC_ORDER; 223 unsigned long *table; 224 225 table = (unsigned long *)physmem_alloc_top_down(RR_VMEM, size, size); 226 crst_table_init(table, val); 227 return table; 228 } 229 230 static pte_t *boot_pte_alloc(void) 231 { 232 static void *pte_leftover; 233 pte_t *pte; 234 235 /* 236 * handling pte_leftovers this way helps to avoid memory fragmentation 237 * during POPULATE_KASAN_MAP_SHADOW when EDAT is off 238 */ 239 if (!pte_leftover) { 240 pte_leftover = (void *)physmem_alloc_top_down(RR_VMEM, PAGE_SIZE, PAGE_SIZE); 241 pte = pte_leftover + _PAGE_TABLE_SIZE; 242 } else { 243 pte = pte_leftover; 244 pte_leftover = NULL; 245 } 246 247 memset64((u64 *)pte, _PAGE_INVALID, PTRS_PER_PTE); 248 return pte; 249 } 250 251 static unsigned long _pa(unsigned long addr, unsigned long size, enum populate_mode mode) 252 { 253 switch (mode) { 254 case POPULATE_NONE: 255 return -1; 256 case POPULATE_DIRECT: 257 return addr; 258 case POPULATE_ABS_LOWCORE: 259 return __abs_lowcore_pa(addr); 260 #ifdef CONFIG_KASAN 261 case POPULATE_KASAN_MAP_SHADOW: 262 addr = physmem_alloc_top_down(RR_VMEM, size, size); 263 memset((void *)addr, 0, size); 264 return addr; 265 #endif 266 default: 267 return -1; 268 } 269 } 270 271 static bool can_large_pud(pud_t *pu_dir, unsigned long addr, unsigned long end) 272 { 273 return machine.has_edat2 && 274 IS_ALIGNED(addr, PUD_SIZE) && (end - addr) >= PUD_SIZE; 275 } 276 277 static bool can_large_pmd(pmd_t *pm_dir, unsigned long addr, unsigned long end) 278 { 279 return machine.has_edat1 && 280 IS_ALIGNED(addr, PMD_SIZE) && (end - addr) >= PMD_SIZE; 281 } 282 283 static void pgtable_pte_populate(pmd_t *pmd, unsigned long addr, unsigned long end, 284 enum populate_mode mode) 285 { 286 unsigned long pages = 0; 287 pte_t *pte, entry; 288 289 pte = pte_offset_kernel(pmd, addr); 290 for (; addr < end; addr += PAGE_SIZE, pte++) { 291 if (pte_none(*pte)) { 292 if (kasan_pte_populate_zero_shadow(pte, mode)) 293 continue; 294 entry = __pte(_pa(addr, PAGE_SIZE, mode)); 295 entry = set_pte_bit(entry, PAGE_KERNEL); 296 if (!machine.has_nx) 297 entry = clear_pte_bit(entry, __pgprot(_PAGE_NOEXEC)); 298 set_pte(pte, entry); 299 pages++; 300 } 301 } 302 if (mode == POPULATE_DIRECT) 303 update_page_count(PG_DIRECT_MAP_4K, pages); 304 } 305 306 static void pgtable_pmd_populate(pud_t *pud, unsigned long addr, unsigned long end, 307 enum populate_mode mode) 308 { 309 unsigned long next, pages = 0; 310 pmd_t *pmd, entry; 311 pte_t *pte; 312 313 pmd = pmd_offset(pud, addr); 314 for (; addr < end; addr = next, pmd++) { 315 next = pmd_addr_end(addr, end); 316 if (pmd_none(*pmd)) { 317 if (kasan_pmd_populate_zero_shadow(pmd, addr, next, mode)) 318 continue; 319 if (can_large_pmd(pmd, addr, next)) { 320 entry = __pmd(_pa(addr, _SEGMENT_SIZE, mode)); 321 entry = set_pmd_bit(entry, SEGMENT_KERNEL); 322 if (!machine.has_nx) 323 entry = clear_pmd_bit(entry, __pgprot(_SEGMENT_ENTRY_NOEXEC)); 324 set_pmd(pmd, entry); 325 pages++; 326 continue; 327 } 328 pte = boot_pte_alloc(); 329 pmd_populate(&init_mm, pmd, pte); 330 } else if (pmd_large(*pmd)) { 331 continue; 332 } 333 pgtable_pte_populate(pmd, addr, next, mode); 334 } 335 if (mode == POPULATE_DIRECT) 336 update_page_count(PG_DIRECT_MAP_1M, pages); 337 } 338 339 static void pgtable_pud_populate(p4d_t *p4d, unsigned long addr, unsigned long end, 340 enum populate_mode mode) 341 { 342 unsigned long next, pages = 0; 343 pud_t *pud, entry; 344 pmd_t *pmd; 345 346 pud = pud_offset(p4d, addr); 347 for (; addr < end; addr = next, pud++) { 348 next = pud_addr_end(addr, end); 349 if (pud_none(*pud)) { 350 if (kasan_pud_populate_zero_shadow(pud, addr, next, mode)) 351 continue; 352 if (can_large_pud(pud, addr, next)) { 353 entry = __pud(_pa(addr, _REGION3_SIZE, mode)); 354 entry = set_pud_bit(entry, REGION3_KERNEL); 355 if (!machine.has_nx) 356 entry = clear_pud_bit(entry, __pgprot(_REGION_ENTRY_NOEXEC)); 357 set_pud(pud, entry); 358 pages++; 359 continue; 360 } 361 pmd = boot_crst_alloc(_SEGMENT_ENTRY_EMPTY); 362 pud_populate(&init_mm, pud, pmd); 363 } else if (pud_leaf(*pud)) { 364 continue; 365 } 366 pgtable_pmd_populate(pud, addr, next, mode); 367 } 368 if (mode == POPULATE_DIRECT) 369 update_page_count(PG_DIRECT_MAP_2G, pages); 370 } 371 372 static void pgtable_p4d_populate(pgd_t *pgd, unsigned long addr, unsigned long end, 373 enum populate_mode mode) 374 { 375 unsigned long next; 376 p4d_t *p4d; 377 pud_t *pud; 378 379 p4d = p4d_offset(pgd, addr); 380 for (; addr < end; addr = next, p4d++) { 381 next = p4d_addr_end(addr, end); 382 if (p4d_none(*p4d)) { 383 if (kasan_p4d_populate_zero_shadow(p4d, addr, next, mode)) 384 continue; 385 pud = boot_crst_alloc(_REGION3_ENTRY_EMPTY); 386 p4d_populate(&init_mm, p4d, pud); 387 } 388 pgtable_pud_populate(p4d, addr, next, mode); 389 } 390 } 391 392 static void pgtable_populate(unsigned long addr, unsigned long end, enum populate_mode mode) 393 { 394 unsigned long next; 395 pgd_t *pgd; 396 p4d_t *p4d; 397 398 pgd = pgd_offset(&init_mm, addr); 399 for (; addr < end; addr = next, pgd++) { 400 next = pgd_addr_end(addr, end); 401 if (pgd_none(*pgd)) { 402 if (kasan_pgd_populate_zero_shadow(pgd, addr, next, mode)) 403 continue; 404 p4d = boot_crst_alloc(_REGION2_ENTRY_EMPTY); 405 pgd_populate(&init_mm, pgd, p4d); 406 } 407 #ifdef CONFIG_KASAN 408 if (mode == POPULATE_KASAN_SHALLOW) 409 continue; 410 #endif 411 pgtable_p4d_populate(pgd, addr, next, mode); 412 } 413 } 414 415 void setup_vmem(unsigned long asce_limit) 416 { 417 unsigned long start, end; 418 unsigned long asce_type; 419 unsigned long asce_bits; 420 int i; 421 422 if (asce_limit == _REGION1_SIZE) { 423 asce_type = _REGION2_ENTRY_EMPTY; 424 asce_bits = _ASCE_TYPE_REGION2 | _ASCE_TABLE_LENGTH; 425 } else { 426 asce_type = _REGION3_ENTRY_EMPTY; 427 asce_bits = _ASCE_TYPE_REGION3 | _ASCE_TABLE_LENGTH; 428 } 429 s390_invalid_asce = invalid_pg_dir | _ASCE_TYPE_REGION3 | _ASCE_TABLE_LENGTH; 430 431 crst_table_init((unsigned long *)swapper_pg_dir, asce_type); 432 crst_table_init((unsigned long *)invalid_pg_dir, _REGION3_ENTRY_EMPTY); 433 434 /* 435 * To allow prefixing the lowcore must be mapped with 4KB pages. 436 * To prevent creation of a large page at address 0 first map 437 * the lowcore and create the identity mapping only afterwards. 438 */ 439 pgtable_populate(0, sizeof(struct lowcore), POPULATE_DIRECT); 440 for_each_physmem_usable_range(i, &start, &end) 441 pgtable_populate(start, end, POPULATE_DIRECT); 442 pgtable_populate(__abs_lowcore, __abs_lowcore + sizeof(struct lowcore), 443 POPULATE_ABS_LOWCORE); 444 pgtable_populate(__memcpy_real_area, __memcpy_real_area + PAGE_SIZE, 445 POPULATE_NONE); 446 memcpy_real_ptep = __virt_to_kpte(__memcpy_real_area); 447 448 kasan_populate_shadow(); 449 450 S390_lowcore.kernel_asce = swapper_pg_dir | asce_bits; 451 S390_lowcore.user_asce = s390_invalid_asce; 452 453 __ctl_load(S390_lowcore.kernel_asce, 1, 1); 454 __ctl_load(S390_lowcore.user_asce, 7, 7); 455 __ctl_load(S390_lowcore.kernel_asce, 13, 13); 456 457 init_mm.context.asce = S390_lowcore.kernel_asce; 458 } 459