1bb1520d5SAlexander Gordeev // SPDX-License-Identifier: GPL-2.0 2bb1520d5SAlexander Gordeev #include <linux/sched/task.h> 3bb1520d5SAlexander Gordeev #include <linux/pgtable.h> 4bb1520d5SAlexander Gordeev #include <asm/pgalloc.h> 5bb1520d5SAlexander Gordeev #include <asm/facility.h> 6bb1520d5SAlexander Gordeev #include <asm/sections.h> 78c37cb7dSVasily Gorbik #include <asm/physmem_info.h> 88e9205d2SAlexander Gordeev #include <asm/maccess.h> 92154e0b3SAlexander Gordeev #include <asm/abs_lowcore.h> 10bb1520d5SAlexander Gordeev #include "decompressor.h" 11bb1520d5SAlexander Gordeev #include "boot.h" 12bb1520d5SAlexander Gordeev 13*f913a660SVasily Gorbik unsigned long __bootdata_preserved(s390_invalid_asce); 14*f913a660SVasily Gorbik 15bb1520d5SAlexander Gordeev #define init_mm (*(struct mm_struct *)vmlinux.init_mm_off) 16bb1520d5SAlexander Gordeev #define swapper_pg_dir vmlinux.swapper_pg_dir_off 17bb1520d5SAlexander Gordeev #define invalid_pg_dir vmlinux.invalid_pg_dir_off 18bb1520d5SAlexander Gordeev 198e9205d2SAlexander Gordeev /* 208e9205d2SAlexander Gordeev * Mimic virt_to_kpte() in lack of init_mm symbol. Skip pmd NULL check though. 218e9205d2SAlexander Gordeev */ 228e9205d2SAlexander Gordeev static inline pte_t *__virt_to_kpte(unsigned long va) 238e9205d2SAlexander Gordeev { 248e9205d2SAlexander Gordeev return pte_offset_kernel(pmd_offset(pud_offset(p4d_offset(pgd_offset_k(va), va), va), va), va); 258e9205d2SAlexander Gordeev } 268e9205d2SAlexander Gordeev 27e0e0a87bSAlexander Gordeev enum populate_mode { 288e9205d2SAlexander Gordeev POPULATE_NONE, 29e0e0a87bSAlexander Gordeev POPULATE_ONE2ONE, 302154e0b3SAlexander Gordeev POPULATE_ABS_LOWCORE, 31e0e0a87bSAlexander Gordeev }; 32e0e0a87bSAlexander Gordeev 33bb1520d5SAlexander Gordeev static void *boot_crst_alloc(unsigned long val) 34bb1520d5SAlexander Gordeev { 35*f913a660SVasily Gorbik unsigned long size = PAGE_SIZE << CRST_ALLOC_ORDER; 36bb1520d5SAlexander Gordeev unsigned long *table; 37bb1520d5SAlexander Gordeev 38*f913a660SVasily Gorbik table = (unsigned long *)physmem_alloc_top_down(RR_VMEM, size, size); 39bb1520d5SAlexander Gordeev crst_table_init(table, val); 40bb1520d5SAlexander Gordeev return table; 41bb1520d5SAlexander Gordeev } 42bb1520d5SAlexander Gordeev 43bb1520d5SAlexander Gordeev static pte_t *boot_pte_alloc(void) 44bb1520d5SAlexander Gordeev { 45bb1520d5SAlexander Gordeev pte_t *pte; 46bb1520d5SAlexander Gordeev 47*f913a660SVasily Gorbik pte = (pte_t *)physmem_alloc_top_down(RR_VMEM, _PAGE_TABLE_SIZE, _PAGE_TABLE_SIZE); 48bb1520d5SAlexander Gordeev memset64((u64 *)pte, _PAGE_INVALID, PTRS_PER_PTE); 49bb1520d5SAlexander Gordeev return pte; 50bb1520d5SAlexander Gordeev } 51bb1520d5SAlexander Gordeev 52e0e0a87bSAlexander Gordeev static unsigned long _pa(unsigned long addr, enum populate_mode mode) 53e0e0a87bSAlexander Gordeev { 54e0e0a87bSAlexander Gordeev switch (mode) { 558e9205d2SAlexander Gordeev case POPULATE_NONE: 568e9205d2SAlexander Gordeev return -1; 57e0e0a87bSAlexander Gordeev case POPULATE_ONE2ONE: 58e0e0a87bSAlexander Gordeev return addr; 592154e0b3SAlexander Gordeev case POPULATE_ABS_LOWCORE: 602154e0b3SAlexander Gordeev return __abs_lowcore_pa(addr); 61e0e0a87bSAlexander Gordeev default: 62e0e0a87bSAlexander Gordeev return -1; 63e0e0a87bSAlexander Gordeev } 64e0e0a87bSAlexander Gordeev } 65e0e0a87bSAlexander Gordeev 66bb1520d5SAlexander Gordeev static bool can_large_pud(pud_t *pu_dir, unsigned long addr, unsigned long end) 67bb1520d5SAlexander Gordeev { 68bb1520d5SAlexander Gordeev return machine.has_edat2 && 69bb1520d5SAlexander Gordeev IS_ALIGNED(addr, PUD_SIZE) && (end - addr) >= PUD_SIZE; 70bb1520d5SAlexander Gordeev } 71bb1520d5SAlexander Gordeev 72bb1520d5SAlexander Gordeev static bool can_large_pmd(pmd_t *pm_dir, unsigned long addr, unsigned long end) 73bb1520d5SAlexander Gordeev { 74bb1520d5SAlexander Gordeev return machine.has_edat1 && 75bb1520d5SAlexander Gordeev IS_ALIGNED(addr, PMD_SIZE) && (end - addr) >= PMD_SIZE; 76bb1520d5SAlexander Gordeev } 77bb1520d5SAlexander Gordeev 78e0e0a87bSAlexander Gordeev static void pgtable_pte_populate(pmd_t *pmd, unsigned long addr, unsigned long end, 79e0e0a87bSAlexander Gordeev enum populate_mode mode) 80bb1520d5SAlexander Gordeev { 81bb1520d5SAlexander Gordeev pte_t *pte, entry; 82bb1520d5SAlexander Gordeev 83bb1520d5SAlexander Gordeev pte = pte_offset_kernel(pmd, addr); 84bb1520d5SAlexander Gordeev for (; addr < end; addr += PAGE_SIZE, pte++) { 85bb1520d5SAlexander Gordeev if (pte_none(*pte)) { 86e0e0a87bSAlexander Gordeev entry = __pte(_pa(addr, mode)); 87bb1520d5SAlexander Gordeev entry = set_pte_bit(entry, PAGE_KERNEL_EXEC); 88bb1520d5SAlexander Gordeev set_pte(pte, entry); 89bb1520d5SAlexander Gordeev } 90bb1520d5SAlexander Gordeev } 91bb1520d5SAlexander Gordeev } 92bb1520d5SAlexander Gordeev 93e0e0a87bSAlexander Gordeev static void pgtable_pmd_populate(pud_t *pud, unsigned long addr, unsigned long end, 94e0e0a87bSAlexander Gordeev enum populate_mode mode) 95bb1520d5SAlexander Gordeev { 96bb1520d5SAlexander Gordeev unsigned long next; 97bb1520d5SAlexander Gordeev pmd_t *pmd, entry; 98bb1520d5SAlexander Gordeev pte_t *pte; 99bb1520d5SAlexander Gordeev 100bb1520d5SAlexander Gordeev pmd = pmd_offset(pud, addr); 101bb1520d5SAlexander Gordeev for (; addr < end; addr = next, pmd++) { 102bb1520d5SAlexander Gordeev next = pmd_addr_end(addr, end); 103bb1520d5SAlexander Gordeev if (pmd_none(*pmd)) { 104bb1520d5SAlexander Gordeev if (can_large_pmd(pmd, addr, next)) { 105e0e0a87bSAlexander Gordeev entry = __pmd(_pa(addr, mode)); 106bb1520d5SAlexander Gordeev entry = set_pmd_bit(entry, SEGMENT_KERNEL_EXEC); 107bb1520d5SAlexander Gordeev set_pmd(pmd, entry); 108bb1520d5SAlexander Gordeev continue; 109bb1520d5SAlexander Gordeev } 110bb1520d5SAlexander Gordeev pte = boot_pte_alloc(); 111bb1520d5SAlexander Gordeev pmd_populate(&init_mm, pmd, pte); 112bb1520d5SAlexander Gordeev } else if (pmd_large(*pmd)) { 113bb1520d5SAlexander Gordeev continue; 114bb1520d5SAlexander Gordeev } 115e0e0a87bSAlexander Gordeev pgtable_pte_populate(pmd, addr, next, mode); 116bb1520d5SAlexander Gordeev } 117bb1520d5SAlexander Gordeev } 118bb1520d5SAlexander Gordeev 119e0e0a87bSAlexander Gordeev static void pgtable_pud_populate(p4d_t *p4d, unsigned long addr, unsigned long end, 120e0e0a87bSAlexander Gordeev enum populate_mode mode) 121bb1520d5SAlexander Gordeev { 122bb1520d5SAlexander Gordeev unsigned long next; 123bb1520d5SAlexander Gordeev pud_t *pud, entry; 124bb1520d5SAlexander Gordeev pmd_t *pmd; 125bb1520d5SAlexander Gordeev 126bb1520d5SAlexander Gordeev pud = pud_offset(p4d, addr); 127bb1520d5SAlexander Gordeev for (; addr < end; addr = next, pud++) { 128bb1520d5SAlexander Gordeev next = pud_addr_end(addr, end); 129bb1520d5SAlexander Gordeev if (pud_none(*pud)) { 130bb1520d5SAlexander Gordeev if (can_large_pud(pud, addr, next)) { 131e0e0a87bSAlexander Gordeev entry = __pud(_pa(addr, mode)); 132bb1520d5SAlexander Gordeev entry = set_pud_bit(entry, REGION3_KERNEL_EXEC); 133bb1520d5SAlexander Gordeev set_pud(pud, entry); 134bb1520d5SAlexander Gordeev continue; 135bb1520d5SAlexander Gordeev } 136bb1520d5SAlexander Gordeev pmd = boot_crst_alloc(_SEGMENT_ENTRY_EMPTY); 137bb1520d5SAlexander Gordeev pud_populate(&init_mm, pud, pmd); 138bb1520d5SAlexander Gordeev } else if (pud_large(*pud)) { 139bb1520d5SAlexander Gordeev continue; 140bb1520d5SAlexander Gordeev } 141e0e0a87bSAlexander Gordeev pgtable_pmd_populate(pud, addr, next, mode); 142bb1520d5SAlexander Gordeev } 143bb1520d5SAlexander Gordeev } 144bb1520d5SAlexander Gordeev 145e0e0a87bSAlexander Gordeev static void pgtable_p4d_populate(pgd_t *pgd, unsigned long addr, unsigned long end, 146e0e0a87bSAlexander Gordeev enum populate_mode mode) 147bb1520d5SAlexander Gordeev { 148bb1520d5SAlexander Gordeev unsigned long next; 149bb1520d5SAlexander Gordeev p4d_t *p4d; 150bb1520d5SAlexander Gordeev pud_t *pud; 151bb1520d5SAlexander Gordeev 152bb1520d5SAlexander Gordeev p4d = p4d_offset(pgd, addr); 153bb1520d5SAlexander Gordeev for (; addr < end; addr = next, p4d++) { 154bb1520d5SAlexander Gordeev next = p4d_addr_end(addr, end); 155bb1520d5SAlexander Gordeev if (p4d_none(*p4d)) { 156bb1520d5SAlexander Gordeev pud = boot_crst_alloc(_REGION3_ENTRY_EMPTY); 157bb1520d5SAlexander Gordeev p4d_populate(&init_mm, p4d, pud); 158bb1520d5SAlexander Gordeev } 159e0e0a87bSAlexander Gordeev pgtable_pud_populate(p4d, addr, next, mode); 160bb1520d5SAlexander Gordeev } 161bb1520d5SAlexander Gordeev } 162bb1520d5SAlexander Gordeev 163e0e0a87bSAlexander Gordeev static void pgtable_populate(unsigned long addr, unsigned long end, enum populate_mode mode) 164bb1520d5SAlexander Gordeev { 165bb1520d5SAlexander Gordeev unsigned long next; 166bb1520d5SAlexander Gordeev pgd_t *pgd; 167bb1520d5SAlexander Gordeev p4d_t *p4d; 168bb1520d5SAlexander Gordeev 169bb1520d5SAlexander Gordeev pgd = pgd_offset(&init_mm, addr); 170bb1520d5SAlexander Gordeev for (; addr < end; addr = next, pgd++) { 171bb1520d5SAlexander Gordeev next = pgd_addr_end(addr, end); 172bb1520d5SAlexander Gordeev if (pgd_none(*pgd)) { 173bb1520d5SAlexander Gordeev p4d = boot_crst_alloc(_REGION2_ENTRY_EMPTY); 174bb1520d5SAlexander Gordeev pgd_populate(&init_mm, pgd, p4d); 175bb1520d5SAlexander Gordeev } 176e0e0a87bSAlexander Gordeev pgtable_p4d_populate(pgd, addr, next, mode); 177bb1520d5SAlexander Gordeev } 178bb1520d5SAlexander Gordeev } 179bb1520d5SAlexander Gordeev 180bf64f051SVasily Gorbik void setup_vmem(unsigned long asce_limit) 181bb1520d5SAlexander Gordeev { 182e966ccf8SVasily Gorbik unsigned long start, end; 183bb1520d5SAlexander Gordeev unsigned long asce_type; 184bb1520d5SAlexander Gordeev unsigned long asce_bits; 185e966ccf8SVasily Gorbik int i; 186bb1520d5SAlexander Gordeev 187bb1520d5SAlexander Gordeev if (asce_limit == _REGION1_SIZE) { 188bb1520d5SAlexander Gordeev asce_type = _REGION2_ENTRY_EMPTY; 189bb1520d5SAlexander Gordeev asce_bits = _ASCE_TYPE_REGION2 | _ASCE_TABLE_LENGTH; 190bb1520d5SAlexander Gordeev } else { 191bb1520d5SAlexander Gordeev asce_type = _REGION3_ENTRY_EMPTY; 192bb1520d5SAlexander Gordeev asce_bits = _ASCE_TYPE_REGION3 | _ASCE_TABLE_LENGTH; 193bb1520d5SAlexander Gordeev } 194bb1520d5SAlexander Gordeev s390_invalid_asce = invalid_pg_dir | _ASCE_TYPE_REGION3 | _ASCE_TABLE_LENGTH; 195bb1520d5SAlexander Gordeev 196bb1520d5SAlexander Gordeev crst_table_init((unsigned long *)swapper_pg_dir, asce_type); 197bb1520d5SAlexander Gordeev crst_table_init((unsigned long *)invalid_pg_dir, _REGION3_ENTRY_EMPTY); 198bb1520d5SAlexander Gordeev 199bb1520d5SAlexander Gordeev /* 200bb1520d5SAlexander Gordeev * To allow prefixing the lowcore must be mapped with 4KB pages. 201bb1520d5SAlexander Gordeev * To prevent creation of a large page at address 0 first map 202bb1520d5SAlexander Gordeev * the lowcore and create the identity mapping only afterwards. 203bb1520d5SAlexander Gordeev */ 204e0e0a87bSAlexander Gordeev pgtable_populate(0, sizeof(struct lowcore), POPULATE_ONE2ONE); 2058c37cb7dSVasily Gorbik for_each_physmem_usable_range(i, &start, &end) 206bf64f051SVasily Gorbik pgtable_populate(start, end, POPULATE_ONE2ONE); 2072154e0b3SAlexander Gordeev pgtable_populate(__abs_lowcore, __abs_lowcore + sizeof(struct lowcore), 2082154e0b3SAlexander Gordeev POPULATE_ABS_LOWCORE); 2098e9205d2SAlexander Gordeev pgtable_populate(__memcpy_real_area, __memcpy_real_area + PAGE_SIZE, 2108e9205d2SAlexander Gordeev POPULATE_NONE); 2118e9205d2SAlexander Gordeev memcpy_real_ptep = __virt_to_kpte(__memcpy_real_area); 212bb1520d5SAlexander Gordeev 213bb1520d5SAlexander Gordeev S390_lowcore.kernel_asce = swapper_pg_dir | asce_bits; 214bb1520d5SAlexander Gordeev S390_lowcore.user_asce = s390_invalid_asce; 215bb1520d5SAlexander Gordeev 216bb1520d5SAlexander Gordeev __ctl_load(S390_lowcore.kernel_asce, 1, 1); 217bb1520d5SAlexander Gordeev __ctl_load(S390_lowcore.user_asce, 7, 7); 218bb1520d5SAlexander Gordeev __ctl_load(S390_lowcore.kernel_asce, 13, 13); 219bb1520d5SAlexander Gordeev 220bb1520d5SAlexander Gordeev init_mm.context.asce = S390_lowcore.kernel_asce; 221bb1520d5SAlexander Gordeev } 222