150acfb2bSThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only 276d2a049SPalmer Dabbelt /* 376d2a049SPalmer Dabbelt * Copyright (C) 2012 Regents of the University of California 4671f9a3eSAnup Patel * Copyright (C) 2019 Western Digital Corporation or its affiliates. 5e53d2818SNick Kossifidis * Copyright (C) 2020 FORTH-ICS/CARV 6e53d2818SNick Kossifidis * Nick Kossifidis <mick@ics.forth.gr> 776d2a049SPalmer Dabbelt */ 876d2a049SPalmer Dabbelt 976d2a049SPalmer Dabbelt #include <linux/init.h> 1076d2a049SPalmer Dabbelt #include <linux/mm.h> 1176d2a049SPalmer Dabbelt #include <linux/memblock.h> 1257c8a661SMike Rapoport #include <linux/initrd.h> 1376d2a049SPalmer Dabbelt #include <linux/swap.h> 145ec9c4ffSChristoph Hellwig #include <linux/sizes.h> 150651c263SAnup Patel #include <linux/of_fdt.h> 1656409750SNick Kossifidis #include <linux/of_reserved_mem.h> 17922b0375SAlbert Ou #include <linux/libfdt.h> 18d27c3c90SZong Li #include <linux/set_memory.h> 19da815582SKefeng Wang #include <linux/dma-map-ops.h> 20e53d2818SNick Kossifidis #include <linux/crash_dump.h> 2176d2a049SPalmer Dabbelt 22f2c17aabSAnup Patel #include <asm/fixmap.h> 2376d2a049SPalmer Dabbelt #include <asm/tlbflush.h> 2476d2a049SPalmer Dabbelt #include <asm/sections.h> 252d268251SPalmer Dabbelt #include <asm/soc.h> 2676d2a049SPalmer Dabbelt #include <asm/io.h> 27b422d28bSZong Li #include <asm/ptdump.h> 284f0e8eefSAtish Patra #include <asm/numa.h> 2976d2a049SPalmer Dabbelt 30ffaee272SPaul Walmsley #include "../kernel/head.h" 31ffaee272SPaul Walmsley 322bfc6cd8SAlexandre Ghiti unsigned long kernel_virt_addr = KERNEL_LINK_ADDR; 332bfc6cd8SAlexandre Ghiti EXPORT_SYMBOL(kernel_virt_addr); 3444c92257SVitaly Wool #ifdef CONFIG_XIP_KERNEL 3544c92257SVitaly Wool #define kernel_virt_addr (*((unsigned long *)XIP_FIXUP(&kernel_virt_addr))) 3650bae95eSKefeng Wang extern char _xiprom[], _exiprom[]; 3744c92257SVitaly Wool #endif 382bfc6cd8SAlexandre Ghiti 39387181dcSAnup Patel unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)] 40387181dcSAnup Patel __page_aligned_bss; 41387181dcSAnup Patel EXPORT_SYMBOL(empty_zero_page); 42387181dcSAnup Patel 43d90d45d7SAnup Patel extern char _start[]; 448f3a2b4aSAnup Patel #define DTB_EARLY_BASE_VA PGDIR_SIZE 4544c92257SVitaly Wool void *_dtb_early_va __initdata; 4644c92257SVitaly Wool uintptr_t _dtb_early_pa __initdata; 47d90d45d7SAnup Patel 48e8dcb61fSAtish Patra struct pt_alloc_ops { 49e8dcb61fSAtish Patra pte_t *(*get_pte_virt)(phys_addr_t pa); 50e8dcb61fSAtish Patra phys_addr_t (*alloc_pte)(uintptr_t va); 51e8dcb61fSAtish Patra #ifndef __PAGETABLE_PMD_FOLDED 52e8dcb61fSAtish Patra pmd_t *(*get_pmd_virt)(phys_addr_t pa); 53e8dcb61fSAtish Patra phys_addr_t (*alloc_pmd)(uintptr_t va); 54e8dcb61fSAtish Patra #endif 55e8dcb61fSAtish Patra }; 5676d2a049SPalmer Dabbelt 57*01062356SJisheng Zhang static phys_addr_t dma32_phys_limit __initdata; 58da815582SKefeng Wang 5976d2a049SPalmer Dabbelt static void __init zone_sizes_init(void) 6076d2a049SPalmer Dabbelt { 615ec9c4ffSChristoph Hellwig unsigned long max_zone_pfns[MAX_NR_ZONES] = { 0, }; 6276d2a049SPalmer Dabbelt 63d5fad48cSZong Li #ifdef CONFIG_ZONE_DMA32 64da815582SKefeng Wang max_zone_pfns[ZONE_DMA32] = PFN_DOWN(dma32_phys_limit); 65d5fad48cSZong Li #endif 665ec9c4ffSChristoph Hellwig max_zone_pfns[ZONE_NORMAL] = max_low_pfn; 675ec9c4ffSChristoph Hellwig 689691a071SMike Rapoport free_area_init(max_zone_pfns); 6976d2a049SPalmer Dabbelt } 7076d2a049SPalmer Dabbelt 718fa3cdffSKefeng Wang #if defined(CONFIG_MMU) && defined(CONFIG_DEBUG_VM) 722cc6c4a0SYash Shah static inline void print_mlk(char *name, unsigned long b, unsigned long t) 732cc6c4a0SYash Shah { 742cc6c4a0SYash Shah pr_notice("%12s : 0x%08lx - 0x%08lx (%4ld kB)\n", name, b, t, 752cc6c4a0SYash Shah (((t) - (b)) >> 10)); 762cc6c4a0SYash Shah } 772cc6c4a0SYash Shah 782cc6c4a0SYash Shah static inline void print_mlm(char *name, unsigned long b, unsigned long t) 792cc6c4a0SYash Shah { 802cc6c4a0SYash Shah pr_notice("%12s : 0x%08lx - 0x%08lx (%4ld MB)\n", name, b, t, 812cc6c4a0SYash Shah (((t) - (b)) >> 20)); 822cc6c4a0SYash Shah } 832cc6c4a0SYash Shah 841987501bSJisheng Zhang static void __init print_vm_layout(void) 852cc6c4a0SYash Shah { 862cc6c4a0SYash Shah pr_notice("Virtual kernel memory layout:\n"); 872cc6c4a0SYash Shah print_mlk("fixmap", (unsigned long)FIXADDR_START, 882cc6c4a0SYash Shah (unsigned long)FIXADDR_TOP); 892cc6c4a0SYash Shah print_mlm("pci io", (unsigned long)PCI_IO_START, 902cc6c4a0SYash Shah (unsigned long)PCI_IO_END); 912cc6c4a0SYash Shah print_mlm("vmemmap", (unsigned long)VMEMMAP_START, 922cc6c4a0SYash Shah (unsigned long)VMEMMAP_END); 932cc6c4a0SYash Shah print_mlm("vmalloc", (unsigned long)VMALLOC_START, 942cc6c4a0SYash Shah (unsigned long)VMALLOC_END); 952cc6c4a0SYash Shah print_mlm("lowmem", (unsigned long)PAGE_OFFSET, 962cc6c4a0SYash Shah (unsigned long)high_memory); 972bfc6cd8SAlexandre Ghiti #ifdef CONFIG_64BIT 982bfc6cd8SAlexandre Ghiti print_mlm("kernel", (unsigned long)KERNEL_LINK_ADDR, 992bfc6cd8SAlexandre Ghiti (unsigned long)ADDRESS_SPACE_END); 1002bfc6cd8SAlexandre Ghiti #endif 1012cc6c4a0SYash Shah } 1022cc6c4a0SYash Shah #else 1032cc6c4a0SYash Shah static void print_vm_layout(void) { } 1042cc6c4a0SYash Shah #endif /* CONFIG_DEBUG_VM */ 1052cc6c4a0SYash Shah 10676d2a049SPalmer Dabbelt void __init mem_init(void) 10776d2a049SPalmer Dabbelt { 10876d2a049SPalmer Dabbelt #ifdef CONFIG_FLATMEM 10976d2a049SPalmer Dabbelt BUG_ON(!mem_map); 11076d2a049SPalmer Dabbelt #endif /* CONFIG_FLATMEM */ 11176d2a049SPalmer Dabbelt 11276d2a049SPalmer Dabbelt high_memory = (void *)(__va(PFN_PHYS(max_low_pfn))); 113c6ffc5caSMike Rapoport memblock_free_all(); 11476d2a049SPalmer Dabbelt 1152cc6c4a0SYash Shah print_vm_layout(); 11676d2a049SPalmer Dabbelt } 11776d2a049SPalmer Dabbelt 118f842f5ffSKefeng Wang static void __init setup_bootmem(void) 1190651c263SAnup Patel { 120ac51e005SZong Li phys_addr_t vmlinux_end = __pa_symbol(&_end); 121ac51e005SZong Li phys_addr_t vmlinux_start = __pa_symbol(&_start); 122dd2d082bSKefeng Wang phys_addr_t dram_end = memblock_end_of_DRAM(); 123abb8e86bSAtish Patra phys_addr_t max_mapped_addr = __pa(~(ulong)0); 1240651c263SAnup Patel 12544c92257SVitaly Wool #ifdef CONFIG_XIP_KERNEL 12644c92257SVitaly Wool vmlinux_start = __pa_symbol(&_sdata); 12744c92257SVitaly Wool #endif 12844c92257SVitaly Wool 129dd2d082bSKefeng Wang /* The maximal physical memory size is -PAGE_OFFSET. */ 130de043da0SAtish Patra memblock_enforce_memory_limit(-PAGE_OFFSET); 1310651c263SAnup Patel 1322bfc6cd8SAlexandre Ghiti /* 1332bfc6cd8SAlexandre Ghiti * Reserve from the start of the kernel to the end of the kernel 1348db6f937SGeert Uytterhoeven */ 1358db6f937SGeert Uytterhoeven #if defined(CONFIG_64BIT) && defined(CONFIG_STRICT_KERNEL_RWX) 1368db6f937SGeert Uytterhoeven /* 1378db6f937SGeert Uytterhoeven * Make sure we align the reservation on PMD_SIZE since we will 1382bfc6cd8SAlexandre Ghiti * map the kernel in the linear mapping as read-only: we do not want 1392bfc6cd8SAlexandre Ghiti * any allocation to happen between _end and the next pmd aligned page. 1402bfc6cd8SAlexandre Ghiti */ 1418db6f937SGeert Uytterhoeven vmlinux_end = (vmlinux_end + PMD_SIZE - 1) & PMD_MASK; 1428db6f937SGeert Uytterhoeven #endif 1438db6f937SGeert Uytterhoeven memblock_reserve(vmlinux_start, vmlinux_end - vmlinux_start); 144d90d45d7SAnup Patel 145abb8e86bSAtish Patra /* 146abb8e86bSAtish Patra * memblock allocator is not aware of the fact that last 4K bytes of 147abb8e86bSAtish Patra * the addressable memory can not be mapped because of IS_ERR_VALUE 148abb8e86bSAtish Patra * macro. Make sure that last 4k bytes are not usable by memblock 149abb8e86bSAtish Patra * if end of dram is equal to maximum addressable memory. 150abb8e86bSAtish Patra */ 151abb8e86bSAtish Patra if (max_mapped_addr == (dram_end - 1)) 152abb8e86bSAtish Patra memblock_set_current_limit(max_mapped_addr - 4096); 153abb8e86bSAtish Patra 154f6e5aedfSKefeng Wang min_low_pfn = PFN_UP(memblock_start_of_DRAM()); 155f6e5aedfSKefeng Wang max_low_pfn = max_pfn = PFN_DOWN(dram_end); 156f6e5aedfSKefeng Wang 157da815582SKefeng Wang dma32_phys_limit = min(4UL * SZ_1G, (unsigned long)PFN_PHYS(max_low_pfn)); 158336e8eb2SGuo Ren set_max_mapnr(max_low_pfn - ARCH_PFN_OFFSET); 1590651c263SAnup Patel 160aec33b54SKefeng Wang reserve_initrd_mem(); 161922b0375SAlbert Ou /* 162f105aa94SVitaly Wool * If DTB is built in, no need to reserve its memblock. 163f105aa94SVitaly Wool * Otherwise, do reserve it but avoid using 164f105aa94SVitaly Wool * early_init_fdt_reserve_self() since __pa() does 165922b0375SAlbert Ou * not work for DTB pointers that are fixmap addresses 166922b0375SAlbert Ou */ 167f105aa94SVitaly Wool if (!IS_ENABLED(CONFIG_BUILTIN_DTB)) 168922b0375SAlbert Ou memblock_reserve(dtb_early_pa, fdt_totalsize(dtb_early_va)); 169922b0375SAlbert Ou 1700651c263SAnup Patel early_init_fdt_scan_reserved_mem(); 171da815582SKefeng Wang dma_contiguous_reserve(dma32_phys_limit); 1720651c263SAnup Patel memblock_allow_resize(); 1730651c263SAnup Patel } 1746f1e9e94SAnup Patel 1756bd33e1eSChristoph Hellwig #ifdef CONFIG_MMU 176*01062356SJisheng Zhang static struct pt_alloc_ops _pt_ops __initdata; 17744c92257SVitaly Wool 17844c92257SVitaly Wool #ifdef CONFIG_XIP_KERNEL 17944c92257SVitaly Wool #define pt_ops (*(struct pt_alloc_ops *)XIP_FIXUP(&_pt_ops)) 18044c92257SVitaly Wool #else 18144c92257SVitaly Wool #define pt_ops _pt_ops 18244c92257SVitaly Wool #endif 183e8dcb61fSAtish Patra 1842bfc6cd8SAlexandre Ghiti /* Offset between linear mapping virtual address and kernel load address */ 185de31ea4aSJisheng Zhang unsigned long va_pa_offset __ro_after_init; 186387181dcSAnup Patel EXPORT_SYMBOL(va_pa_offset); 18744c92257SVitaly Wool #ifdef CONFIG_XIP_KERNEL 18844c92257SVitaly Wool #define va_pa_offset (*((unsigned long *)XIP_FIXUP(&va_pa_offset))) 18944c92257SVitaly Wool #endif 1902bfc6cd8SAlexandre Ghiti /* Offset between kernel mapping virtual address and kernel load address */ 19144c92257SVitaly Wool #ifdef CONFIG_64BIT 192*01062356SJisheng Zhang unsigned long va_kernel_pa_offset __ro_after_init; 1932bfc6cd8SAlexandre Ghiti EXPORT_SYMBOL(va_kernel_pa_offset); 1942bfc6cd8SAlexandre Ghiti #endif 19544c92257SVitaly Wool #ifdef CONFIG_XIP_KERNEL 19644c92257SVitaly Wool #define va_kernel_pa_offset (*((unsigned long *)XIP_FIXUP(&va_kernel_pa_offset))) 19744c92257SVitaly Wool #endif 198*01062356SJisheng Zhang unsigned long va_kernel_xip_pa_offset __ro_after_init; 19944c92257SVitaly Wool EXPORT_SYMBOL(va_kernel_xip_pa_offset); 20044c92257SVitaly Wool #ifdef CONFIG_XIP_KERNEL 20144c92257SVitaly Wool #define va_kernel_xip_pa_offset (*((unsigned long *)XIP_FIXUP(&va_kernel_xip_pa_offset))) 20244c92257SVitaly Wool #endif 203de31ea4aSJisheng Zhang unsigned long pfn_base __ro_after_init; 204387181dcSAnup Patel EXPORT_SYMBOL(pfn_base); 205387181dcSAnup Patel 2066f1e9e94SAnup Patel pgd_t swapper_pg_dir[PTRS_PER_PGD] __page_aligned_bss; 207671f9a3eSAnup Patel pgd_t trampoline_pg_dir[PTRS_PER_PGD] __page_aligned_bss; 208*01062356SJisheng Zhang static pte_t fixmap_pte[PTRS_PER_PTE] __page_aligned_bss; 209671f9a3eSAnup Patel 210671f9a3eSAnup Patel pgd_t early_pg_dir[PTRS_PER_PGD] __initdata __aligned(PAGE_SIZE); 211f2c17aabSAnup Patel 21244c92257SVitaly Wool #ifdef CONFIG_XIP_KERNEL 21344c92257SVitaly Wool #define trampoline_pg_dir ((pgd_t *)XIP_FIXUP(trampoline_pg_dir)) 21444c92257SVitaly Wool #define fixmap_pte ((pte_t *)XIP_FIXUP(fixmap_pte)) 21544c92257SVitaly Wool #define early_pg_dir ((pgd_t *)XIP_FIXUP(early_pg_dir)) 21644c92257SVitaly Wool #endif /* CONFIG_XIP_KERNEL */ 21744c92257SVitaly Wool 218f2c17aabSAnup Patel void __set_fixmap(enum fixed_addresses idx, phys_addr_t phys, pgprot_t prot) 219f2c17aabSAnup Patel { 220f2c17aabSAnup Patel unsigned long addr = __fix_to_virt(idx); 221f2c17aabSAnup Patel pte_t *ptep; 222f2c17aabSAnup Patel 223f2c17aabSAnup Patel BUG_ON(idx <= FIX_HOLE || idx >= __end_of_fixed_addresses); 224f2c17aabSAnup Patel 225f2c17aabSAnup Patel ptep = &fixmap_pte[pte_index(addr)]; 226f2c17aabSAnup Patel 22721190b74SGreentime Hu if (pgprot_val(prot)) 228f2c17aabSAnup Patel set_pte(ptep, pfn_pte(phys >> PAGE_SHIFT, prot)); 22921190b74SGreentime Hu else 230f2c17aabSAnup Patel pte_clear(&init_mm, addr, ptep); 231f2c17aabSAnup Patel local_flush_tlb_page(addr); 232f2c17aabSAnup Patel } 233f2c17aabSAnup Patel 234e8dcb61fSAtish Patra static inline pte_t *__init get_pte_virt_early(phys_addr_t pa) 235671f9a3eSAnup Patel { 236671f9a3eSAnup Patel return (pte_t *)((uintptr_t)pa); 237671f9a3eSAnup Patel } 238e8dcb61fSAtish Patra 239e8dcb61fSAtish Patra static inline pte_t *__init get_pte_virt_fixmap(phys_addr_t pa) 240e8dcb61fSAtish Patra { 241e8dcb61fSAtish Patra clear_fixmap(FIX_PTE); 242e8dcb61fSAtish Patra return (pte_t *)set_fixmap_offset(FIX_PTE, pa); 243671f9a3eSAnup Patel } 244671f9a3eSAnup Patel 245*01062356SJisheng Zhang static inline pte_t *__init get_pte_virt_late(phys_addr_t pa) 246e8dcb61fSAtish Patra { 247e8dcb61fSAtish Patra return (pte_t *) __va(pa); 248e8dcb61fSAtish Patra } 249e8dcb61fSAtish Patra 250e8dcb61fSAtish Patra static inline phys_addr_t __init alloc_pte_early(uintptr_t va) 251671f9a3eSAnup Patel { 252671f9a3eSAnup Patel /* 253671f9a3eSAnup Patel * We only create PMD or PGD early mappings so we 254671f9a3eSAnup Patel * should never reach here with MMU disabled. 255671f9a3eSAnup Patel */ 256e8dcb61fSAtish Patra BUG(); 257e8dcb61fSAtish Patra } 258671f9a3eSAnup Patel 259e8dcb61fSAtish Patra static inline phys_addr_t __init alloc_pte_fixmap(uintptr_t va) 260e8dcb61fSAtish Patra { 261671f9a3eSAnup Patel return memblock_phys_alloc(PAGE_SIZE, PAGE_SIZE); 262671f9a3eSAnup Patel } 263671f9a3eSAnup Patel 264*01062356SJisheng Zhang static phys_addr_t __init alloc_pte_late(uintptr_t va) 265e8dcb61fSAtish Patra { 266e8dcb61fSAtish Patra unsigned long vaddr; 267e8dcb61fSAtish Patra 268e8dcb61fSAtish Patra vaddr = __get_free_page(GFP_KERNEL); 269e75e6bf4Szhouchuangao BUG_ON(!vaddr || !pgtable_pte_page_ctor(virt_to_page(vaddr))); 270e75e6bf4Szhouchuangao 271e8dcb61fSAtish Patra return __pa(vaddr); 272e8dcb61fSAtish Patra } 273e8dcb61fSAtish Patra 274671f9a3eSAnup Patel static void __init create_pte_mapping(pte_t *ptep, 275671f9a3eSAnup Patel uintptr_t va, phys_addr_t pa, 276671f9a3eSAnup Patel phys_addr_t sz, pgprot_t prot) 277671f9a3eSAnup Patel { 278974b9b2cSMike Rapoport uintptr_t pte_idx = pte_index(va); 279671f9a3eSAnup Patel 280671f9a3eSAnup Patel BUG_ON(sz != PAGE_SIZE); 281671f9a3eSAnup Patel 282974b9b2cSMike Rapoport if (pte_none(ptep[pte_idx])) 283974b9b2cSMike Rapoport ptep[pte_idx] = pfn_pte(PFN_DOWN(pa), prot); 284671f9a3eSAnup Patel } 285671f9a3eSAnup Patel 286671f9a3eSAnup Patel #ifndef __PAGETABLE_PMD_FOLDED 287671f9a3eSAnup Patel 288*01062356SJisheng Zhang static pmd_t trampoline_pmd[PTRS_PER_PMD] __page_aligned_bss; 289*01062356SJisheng Zhang static pmd_t fixmap_pmd[PTRS_PER_PMD] __page_aligned_bss; 290*01062356SJisheng Zhang static pmd_t early_pmd[PTRS_PER_PMD] __initdata __aligned(PAGE_SIZE); 291*01062356SJisheng Zhang static pmd_t early_dtb_pmd[PTRS_PER_PMD] __initdata __aligned(PAGE_SIZE); 292671f9a3eSAnup Patel 29344c92257SVitaly Wool #ifdef CONFIG_XIP_KERNEL 29444c92257SVitaly Wool #define trampoline_pmd ((pmd_t *)XIP_FIXUP(trampoline_pmd)) 29544c92257SVitaly Wool #define fixmap_pmd ((pmd_t *)XIP_FIXUP(fixmap_pmd)) 29644c92257SVitaly Wool #define early_pmd ((pmd_t *)XIP_FIXUP(early_pmd)) 29744c92257SVitaly Wool #endif /* CONFIG_XIP_KERNEL */ 29844c92257SVitaly Wool 299e8dcb61fSAtish Patra static pmd_t *__init get_pmd_virt_early(phys_addr_t pa) 300671f9a3eSAnup Patel { 301e8dcb61fSAtish Patra /* Before MMU is enabled */ 302671f9a3eSAnup Patel return (pmd_t *)((uintptr_t)pa); 303671f9a3eSAnup Patel } 304e8dcb61fSAtish Patra 305e8dcb61fSAtish Patra static pmd_t *__init get_pmd_virt_fixmap(phys_addr_t pa) 306e8dcb61fSAtish Patra { 307e8dcb61fSAtish Patra clear_fixmap(FIX_PMD); 308e8dcb61fSAtish Patra return (pmd_t *)set_fixmap_offset(FIX_PMD, pa); 309671f9a3eSAnup Patel } 310671f9a3eSAnup Patel 311*01062356SJisheng Zhang static pmd_t *__init get_pmd_virt_late(phys_addr_t pa) 312e8dcb61fSAtish Patra { 313e8dcb61fSAtish Patra return (pmd_t *) __va(pa); 314e8dcb61fSAtish Patra } 315e8dcb61fSAtish Patra 316e8dcb61fSAtish Patra static phys_addr_t __init alloc_pmd_early(uintptr_t va) 317671f9a3eSAnup Patel { 3182bfc6cd8SAlexandre Ghiti BUG_ON((va - kernel_virt_addr) >> PGDIR_SHIFT); 319671f9a3eSAnup Patel 3200f02de44SAlexandre Ghiti return (uintptr_t)early_pmd; 321671f9a3eSAnup Patel } 322671f9a3eSAnup Patel 323e8dcb61fSAtish Patra static phys_addr_t __init alloc_pmd_fixmap(uintptr_t va) 324e8dcb61fSAtish Patra { 325e8dcb61fSAtish Patra return memblock_phys_alloc(PAGE_SIZE, PAGE_SIZE); 326e8dcb61fSAtish Patra } 327e8dcb61fSAtish Patra 328*01062356SJisheng Zhang static phys_addr_t __init alloc_pmd_late(uintptr_t va) 329e8dcb61fSAtish Patra { 330e8dcb61fSAtish Patra unsigned long vaddr; 331e8dcb61fSAtish Patra 332e8dcb61fSAtish Patra vaddr = __get_free_page(GFP_KERNEL); 333e8dcb61fSAtish Patra BUG_ON(!vaddr); 334e8dcb61fSAtish Patra return __pa(vaddr); 335e8dcb61fSAtish Patra } 336e8dcb61fSAtish Patra 337671f9a3eSAnup Patel static void __init create_pmd_mapping(pmd_t *pmdp, 338671f9a3eSAnup Patel uintptr_t va, phys_addr_t pa, 339671f9a3eSAnup Patel phys_addr_t sz, pgprot_t prot) 340671f9a3eSAnup Patel { 341671f9a3eSAnup Patel pte_t *ptep; 342671f9a3eSAnup Patel phys_addr_t pte_phys; 343974b9b2cSMike Rapoport uintptr_t pmd_idx = pmd_index(va); 344671f9a3eSAnup Patel 345671f9a3eSAnup Patel if (sz == PMD_SIZE) { 346974b9b2cSMike Rapoport if (pmd_none(pmdp[pmd_idx])) 347974b9b2cSMike Rapoport pmdp[pmd_idx] = pfn_pmd(PFN_DOWN(pa), prot); 348671f9a3eSAnup Patel return; 349671f9a3eSAnup Patel } 350671f9a3eSAnup Patel 351974b9b2cSMike Rapoport if (pmd_none(pmdp[pmd_idx])) { 352e8dcb61fSAtish Patra pte_phys = pt_ops.alloc_pte(va); 353974b9b2cSMike Rapoport pmdp[pmd_idx] = pfn_pmd(PFN_DOWN(pte_phys), PAGE_TABLE); 354e8dcb61fSAtish Patra ptep = pt_ops.get_pte_virt(pte_phys); 355671f9a3eSAnup Patel memset(ptep, 0, PAGE_SIZE); 356671f9a3eSAnup Patel } else { 357974b9b2cSMike Rapoport pte_phys = PFN_PHYS(_pmd_pfn(pmdp[pmd_idx])); 358e8dcb61fSAtish Patra ptep = pt_ops.get_pte_virt(pte_phys); 359671f9a3eSAnup Patel } 360671f9a3eSAnup Patel 361671f9a3eSAnup Patel create_pte_mapping(ptep, va, pa, sz, prot); 362671f9a3eSAnup Patel } 363671f9a3eSAnup Patel 364671f9a3eSAnup Patel #define pgd_next_t pmd_t 365e8dcb61fSAtish Patra #define alloc_pgd_next(__va) pt_ops.alloc_pmd(__va) 366e8dcb61fSAtish Patra #define get_pgd_next_virt(__pa) pt_ops.get_pmd_virt(__pa) 367671f9a3eSAnup Patel #define create_pgd_next_mapping(__nextp, __va, __pa, __sz, __prot) \ 368671f9a3eSAnup Patel create_pmd_mapping(__nextp, __va, __pa, __sz, __prot) 369671f9a3eSAnup Patel #define fixmap_pgd_next fixmap_pmd 370671f9a3eSAnup Patel #else 371671f9a3eSAnup Patel #define pgd_next_t pte_t 372e8dcb61fSAtish Patra #define alloc_pgd_next(__va) pt_ops.alloc_pte(__va) 373e8dcb61fSAtish Patra #define get_pgd_next_virt(__pa) pt_ops.get_pte_virt(__pa) 374671f9a3eSAnup Patel #define create_pgd_next_mapping(__nextp, __va, __pa, __sz, __prot) \ 375671f9a3eSAnup Patel create_pte_mapping(__nextp, __va, __pa, __sz, __prot) 376671f9a3eSAnup Patel #define fixmap_pgd_next fixmap_pte 377671f9a3eSAnup Patel #endif 378671f9a3eSAnup Patel 379b91540d5SAtish Patra void __init create_pgd_mapping(pgd_t *pgdp, 380671f9a3eSAnup Patel uintptr_t va, phys_addr_t pa, 381671f9a3eSAnup Patel phys_addr_t sz, pgprot_t prot) 382671f9a3eSAnup Patel { 383671f9a3eSAnup Patel pgd_next_t *nextp; 384671f9a3eSAnup Patel phys_addr_t next_phys; 385974b9b2cSMike Rapoport uintptr_t pgd_idx = pgd_index(va); 386671f9a3eSAnup Patel 387671f9a3eSAnup Patel if (sz == PGDIR_SIZE) { 388974b9b2cSMike Rapoport if (pgd_val(pgdp[pgd_idx]) == 0) 389974b9b2cSMike Rapoport pgdp[pgd_idx] = pfn_pgd(PFN_DOWN(pa), prot); 390671f9a3eSAnup Patel return; 391671f9a3eSAnup Patel } 392671f9a3eSAnup Patel 393974b9b2cSMike Rapoport if (pgd_val(pgdp[pgd_idx]) == 0) { 394671f9a3eSAnup Patel next_phys = alloc_pgd_next(va); 395974b9b2cSMike Rapoport pgdp[pgd_idx] = pfn_pgd(PFN_DOWN(next_phys), PAGE_TABLE); 396671f9a3eSAnup Patel nextp = get_pgd_next_virt(next_phys); 397671f9a3eSAnup Patel memset(nextp, 0, PAGE_SIZE); 398671f9a3eSAnup Patel } else { 399974b9b2cSMike Rapoport next_phys = PFN_PHYS(_pgd_pfn(pgdp[pgd_idx])); 400671f9a3eSAnup Patel nextp = get_pgd_next_virt(next_phys); 401671f9a3eSAnup Patel } 402671f9a3eSAnup Patel 403671f9a3eSAnup Patel create_pgd_next_mapping(nextp, va, pa, sz, prot); 404671f9a3eSAnup Patel } 405671f9a3eSAnup Patel 406671f9a3eSAnup Patel static uintptr_t __init best_map_size(phys_addr_t base, phys_addr_t size) 407671f9a3eSAnup Patel { 4080fdc636cSZong Li /* Upgrade to PMD_SIZE mappings whenever possible */ 4090fdc636cSZong Li if ((base & (PMD_SIZE - 1)) || (size & (PMD_SIZE - 1))) 4100fdc636cSZong Li return PAGE_SIZE; 411671f9a3eSAnup Patel 4120fdc636cSZong Li return PMD_SIZE; 413671f9a3eSAnup Patel } 414671f9a3eSAnup Patel 41544c92257SVitaly Wool #ifdef CONFIG_XIP_KERNEL 41644c92257SVitaly Wool /* called from head.S with MMU off */ 41744c92257SVitaly Wool asmlinkage void __init __copy_data(void) 41844c92257SVitaly Wool { 41944c92257SVitaly Wool void *from = (void *)(&_sdata); 42044c92257SVitaly Wool void *end = (void *)(&_end); 42144c92257SVitaly Wool void *to = (void *)CONFIG_PHYS_RAM_BASE; 42244c92257SVitaly Wool size_t sz = (size_t)(end - from + 1); 42344c92257SVitaly Wool 42444c92257SVitaly Wool memcpy(to, from, sz); 42544c92257SVitaly Wool } 42644c92257SVitaly Wool #endif 42744c92257SVitaly Wool 428387181dcSAnup Patel /* 429387181dcSAnup Patel * setup_vm() is called from head.S with MMU-off. 430387181dcSAnup Patel * 431387181dcSAnup Patel * Following requirements should be honoured for setup_vm() to work 432387181dcSAnup Patel * correctly: 433387181dcSAnup Patel * 1) It should use PC-relative addressing for accessing kernel symbols. 434387181dcSAnup Patel * To achieve this we always use GCC cmodel=medany. 435387181dcSAnup Patel * 2) The compiler instrumentation for FTRACE will not work for setup_vm() 436387181dcSAnup Patel * so disable compiler instrumentation when FTRACE is enabled. 437387181dcSAnup Patel * 438387181dcSAnup Patel * Currently, the above requirements are honoured by using custom CFLAGS 439387181dcSAnup Patel * for init.o in mm/Makefile. 440387181dcSAnup Patel */ 441387181dcSAnup Patel 442387181dcSAnup Patel #ifndef __riscv_cmodel_medany 4436a527b67SPaul Walmsley #error "setup_vm() is called from head.S before relocate so it should not use absolute addressing." 444387181dcSAnup Patel #endif 445387181dcSAnup Patel 446*01062356SJisheng Zhang static uintptr_t load_pa __initdata; 447*01062356SJisheng Zhang static uintptr_t load_sz __initdata; 44844c92257SVitaly Wool #ifdef CONFIG_XIP_KERNEL 44944c92257SVitaly Wool #define load_pa (*((uintptr_t *)XIP_FIXUP(&load_pa))) 45044c92257SVitaly Wool #define load_sz (*((uintptr_t *)XIP_FIXUP(&load_sz))) 45144c92257SVitaly Wool #endif 4522bfc6cd8SAlexandre Ghiti 45344c92257SVitaly Wool #ifdef CONFIG_XIP_KERNEL 454*01062356SJisheng Zhang static uintptr_t xiprom __inidata; 455*01062356SJisheng Zhang static uintptr_t xiprom_sz __initdata; 45644c92257SVitaly Wool #define xiprom_sz (*((uintptr_t *)XIP_FIXUP(&xiprom_sz))) 45744c92257SVitaly Wool #define xiprom (*((uintptr_t *)XIP_FIXUP(&xiprom))) 45844c92257SVitaly Wool 45944c92257SVitaly Wool static void __init create_kernel_page_table(pgd_t *pgdir, uintptr_t map_size) 46044c92257SVitaly Wool { 46144c92257SVitaly Wool uintptr_t va, end_va; 46244c92257SVitaly Wool 46344c92257SVitaly Wool /* Map the flash resident part */ 46444c92257SVitaly Wool end_va = kernel_virt_addr + xiprom_sz; 46544c92257SVitaly Wool for (va = kernel_virt_addr; va < end_va; va += map_size) 46644c92257SVitaly Wool create_pgd_mapping(pgdir, va, 46744c92257SVitaly Wool xiprom + (va - kernel_virt_addr), 46844c92257SVitaly Wool map_size, PAGE_KERNEL_EXEC); 46944c92257SVitaly Wool 47044c92257SVitaly Wool /* Map the data in RAM */ 47144c92257SVitaly Wool end_va = kernel_virt_addr + XIP_OFFSET + load_sz; 47244c92257SVitaly Wool for (va = kernel_virt_addr + XIP_OFFSET; va < end_va; va += map_size) 47344c92257SVitaly Wool create_pgd_mapping(pgdir, va, 47444c92257SVitaly Wool load_pa + (va - (kernel_virt_addr + XIP_OFFSET)), 47544c92257SVitaly Wool map_size, PAGE_KERNEL); 47644c92257SVitaly Wool } 47744c92257SVitaly Wool #else 4782bfc6cd8SAlexandre Ghiti static void __init create_kernel_page_table(pgd_t *pgdir, uintptr_t map_size) 4792bfc6cd8SAlexandre Ghiti { 4802bfc6cd8SAlexandre Ghiti uintptr_t va, end_va; 4812bfc6cd8SAlexandre Ghiti 4822bfc6cd8SAlexandre Ghiti end_va = kernel_virt_addr + load_sz; 4832bfc6cd8SAlexandre Ghiti for (va = kernel_virt_addr; va < end_va; va += map_size) 4842bfc6cd8SAlexandre Ghiti create_pgd_mapping(pgdir, va, 4852bfc6cd8SAlexandre Ghiti load_pa + (va - kernel_virt_addr), 4862bfc6cd8SAlexandre Ghiti map_size, PAGE_KERNEL_EXEC); 4872bfc6cd8SAlexandre Ghiti } 48844c92257SVitaly Wool #endif 4892bfc6cd8SAlexandre Ghiti 490671f9a3eSAnup Patel asmlinkage void __init setup_vm(uintptr_t dtb_pa) 4916f1e9e94SAnup Patel { 49244c92257SVitaly Wool uintptr_t __maybe_unused pa; 4930f02de44SAlexandre Ghiti uintptr_t map_size; 4946262f661SAtish Patra #ifndef __PAGETABLE_PMD_FOLDED 4956262f661SAtish Patra pmd_t fix_bmap_spmd, fix_bmap_epmd; 4966262f661SAtish Patra #endif 4976f1e9e94SAnup Patel 49844c92257SVitaly Wool #ifdef CONFIG_XIP_KERNEL 49944c92257SVitaly Wool xiprom = (uintptr_t)CONFIG_XIP_PHYS_ADDR; 50044c92257SVitaly Wool xiprom_sz = (uintptr_t)(&_exiprom) - (uintptr_t)(&_xiprom); 50144c92257SVitaly Wool 50244c92257SVitaly Wool load_pa = (uintptr_t)CONFIG_PHYS_RAM_BASE; 50344c92257SVitaly Wool load_sz = (uintptr_t)(&_end) - (uintptr_t)(&_sdata); 50444c92257SVitaly Wool 50544c92257SVitaly Wool va_kernel_xip_pa_offset = kernel_virt_addr - xiprom; 50644c92257SVitaly Wool #else 5072bfc6cd8SAlexandre Ghiti load_pa = (uintptr_t)(&_start); 5082bfc6cd8SAlexandre Ghiti load_sz = (uintptr_t)(&_end) - load_pa; 50944c92257SVitaly Wool #endif 5106f1e9e94SAnup Patel 511671f9a3eSAnup Patel va_pa_offset = PAGE_OFFSET - load_pa; 5122bfc6cd8SAlexandre Ghiti #ifdef CONFIG_64BIT 5132bfc6cd8SAlexandre Ghiti va_kernel_pa_offset = kernel_virt_addr - load_pa; 5142bfc6cd8SAlexandre Ghiti #endif 5152bfc6cd8SAlexandre Ghiti 516671f9a3eSAnup Patel pfn_base = PFN_DOWN(load_pa); 517671f9a3eSAnup Patel 518671f9a3eSAnup Patel /* 519671f9a3eSAnup Patel * Enforce boot alignment requirements of RV32 and 520671f9a3eSAnup Patel * RV64 by only allowing PMD or PGD mappings. 521671f9a3eSAnup Patel */ 5220f02de44SAlexandre Ghiti map_size = PMD_SIZE; 5236f1e9e94SAnup Patel 5246f1e9e94SAnup Patel /* Sanity check alignment and size */ 5256f1e9e94SAnup Patel BUG_ON((PAGE_OFFSET % PGDIR_SIZE) != 0); 526671f9a3eSAnup Patel BUG_ON((load_pa % map_size) != 0); 527671f9a3eSAnup Patel 528e8dcb61fSAtish Patra pt_ops.alloc_pte = alloc_pte_early; 529e8dcb61fSAtish Patra pt_ops.get_pte_virt = get_pte_virt_early; 530e8dcb61fSAtish Patra #ifndef __PAGETABLE_PMD_FOLDED 531e8dcb61fSAtish Patra pt_ops.alloc_pmd = alloc_pmd_early; 532e8dcb61fSAtish Patra pt_ops.get_pmd_virt = get_pmd_virt_early; 533e8dcb61fSAtish Patra #endif 534671f9a3eSAnup Patel /* Setup early PGD for fixmap */ 535671f9a3eSAnup Patel create_pgd_mapping(early_pg_dir, FIXADDR_START, 536671f9a3eSAnup Patel (uintptr_t)fixmap_pgd_next, PGDIR_SIZE, PAGE_TABLE); 5376f1e9e94SAnup Patel 5386f1e9e94SAnup Patel #ifndef __PAGETABLE_PMD_FOLDED 539671f9a3eSAnup Patel /* Setup fixmap PMD */ 540671f9a3eSAnup Patel create_pmd_mapping(fixmap_pmd, FIXADDR_START, 541671f9a3eSAnup Patel (uintptr_t)fixmap_pte, PMD_SIZE, PAGE_TABLE); 542671f9a3eSAnup Patel /* Setup trampoline PGD and PMD */ 5432bfc6cd8SAlexandre Ghiti create_pgd_mapping(trampoline_pg_dir, kernel_virt_addr, 544671f9a3eSAnup Patel (uintptr_t)trampoline_pmd, PGDIR_SIZE, PAGE_TABLE); 54544c92257SVitaly Wool #ifdef CONFIG_XIP_KERNEL 54644c92257SVitaly Wool create_pmd_mapping(trampoline_pmd, kernel_virt_addr, 54744c92257SVitaly Wool xiprom, PMD_SIZE, PAGE_KERNEL_EXEC); 54844c92257SVitaly Wool #else 5492bfc6cd8SAlexandre Ghiti create_pmd_mapping(trampoline_pmd, kernel_virt_addr, 550671f9a3eSAnup Patel load_pa, PMD_SIZE, PAGE_KERNEL_EXEC); 55144c92257SVitaly Wool #endif 5526f1e9e94SAnup Patel #else 553671f9a3eSAnup Patel /* Setup trampoline PGD */ 5542bfc6cd8SAlexandre Ghiti create_pgd_mapping(trampoline_pg_dir, kernel_virt_addr, 555671f9a3eSAnup Patel load_pa, PGDIR_SIZE, PAGE_KERNEL_EXEC); 556671f9a3eSAnup Patel #endif 5576f1e9e94SAnup Patel 558671f9a3eSAnup Patel /* 5592bfc6cd8SAlexandre Ghiti * Setup early PGD covering entire kernel which will allow 560671f9a3eSAnup Patel * us to reach paging_init(). We map all memory banks later 561671f9a3eSAnup Patel * in setup_vm_final() below. 562671f9a3eSAnup Patel */ 5632bfc6cd8SAlexandre Ghiti create_kernel_page_table(early_pg_dir, map_size); 564f2c17aabSAnup Patel 5651074dd44SAnup Patel #ifndef __PAGETABLE_PMD_FOLDED 5661074dd44SAnup Patel /* Setup early PMD for DTB */ 5671074dd44SAnup Patel create_pgd_mapping(early_pg_dir, DTB_EARLY_BASE_VA, 5681074dd44SAnup Patel (uintptr_t)early_dtb_pmd, PGDIR_SIZE, PAGE_TABLE); 569f105aa94SVitaly Wool #ifndef CONFIG_BUILTIN_DTB 5701074dd44SAnup Patel /* Create two consecutive PMD mappings for FDT early scan */ 5711074dd44SAnup Patel pa = dtb_pa & ~(PMD_SIZE - 1); 5721074dd44SAnup Patel create_pmd_mapping(early_dtb_pmd, DTB_EARLY_BASE_VA, 5731074dd44SAnup Patel pa, PMD_SIZE, PAGE_KERNEL); 5741074dd44SAnup Patel create_pmd_mapping(early_dtb_pmd, DTB_EARLY_BASE_VA + PMD_SIZE, 5751074dd44SAnup Patel pa + PMD_SIZE, PMD_SIZE, PAGE_KERNEL); 5761074dd44SAnup Patel dtb_early_va = (void *)DTB_EARLY_BASE_VA + (dtb_pa & (PMD_SIZE - 1)); 577f105aa94SVitaly Wool #else /* CONFIG_BUILTIN_DTB */ 5782bfc6cd8SAlexandre Ghiti #ifdef CONFIG_64BIT 5792bfc6cd8SAlexandre Ghiti /* 5802bfc6cd8SAlexandre Ghiti * __va can't be used since it would return a linear mapping address 5812bfc6cd8SAlexandre Ghiti * whereas dtb_early_va will be used before setup_vm_final installs 5822bfc6cd8SAlexandre Ghiti * the linear mapping. 5832bfc6cd8SAlexandre Ghiti */ 58444c92257SVitaly Wool dtb_early_va = kernel_mapping_pa_to_va(XIP_FIXUP(dtb_pa)); 5852bfc6cd8SAlexandre Ghiti #else 586f105aa94SVitaly Wool dtb_early_va = __va(dtb_pa); 5872bfc6cd8SAlexandre Ghiti #endif /* CONFIG_64BIT */ 588f105aa94SVitaly Wool #endif /* CONFIG_BUILTIN_DTB */ 5891074dd44SAnup Patel #else 590f105aa94SVitaly Wool #ifndef CONFIG_BUILTIN_DTB 5918f3a2b4aSAnup Patel /* Create two consecutive PGD mappings for FDT early scan */ 5928f3a2b4aSAnup Patel pa = dtb_pa & ~(PGDIR_SIZE - 1); 5938f3a2b4aSAnup Patel create_pgd_mapping(early_pg_dir, DTB_EARLY_BASE_VA, 5948f3a2b4aSAnup Patel pa, PGDIR_SIZE, PAGE_KERNEL); 5958f3a2b4aSAnup Patel create_pgd_mapping(early_pg_dir, DTB_EARLY_BASE_VA + PGDIR_SIZE, 5968f3a2b4aSAnup Patel pa + PGDIR_SIZE, PGDIR_SIZE, PAGE_KERNEL); 5978f3a2b4aSAnup Patel dtb_early_va = (void *)DTB_EARLY_BASE_VA + (dtb_pa & (PGDIR_SIZE - 1)); 598f105aa94SVitaly Wool #else /* CONFIG_BUILTIN_DTB */ 5992bfc6cd8SAlexandre Ghiti #ifdef CONFIG_64BIT 60044c92257SVitaly Wool dtb_early_va = kernel_mapping_pa_to_va(XIP_FIXUP(dtb_pa)); 6012bfc6cd8SAlexandre Ghiti #else 602f105aa94SVitaly Wool dtb_early_va = __va(dtb_pa); 6032bfc6cd8SAlexandre Ghiti #endif /* CONFIG_64BIT */ 604f105aa94SVitaly Wool #endif /* CONFIG_BUILTIN_DTB */ 6051074dd44SAnup Patel #endif 606922b0375SAlbert Ou dtb_early_pa = dtb_pa; 6076262f661SAtish Patra 6086262f661SAtish Patra /* 6096262f661SAtish Patra * Bootime fixmap only can handle PMD_SIZE mapping. Thus, boot-ioremap 6106262f661SAtish Patra * range can not span multiple pmds. 6116262f661SAtish Patra */ 6126262f661SAtish Patra BUILD_BUG_ON((__fix_to_virt(FIX_BTMAP_BEGIN) >> PMD_SHIFT) 6136262f661SAtish Patra != (__fix_to_virt(FIX_BTMAP_END) >> PMD_SHIFT)); 6146262f661SAtish Patra 6156262f661SAtish Patra #ifndef __PAGETABLE_PMD_FOLDED 6166262f661SAtish Patra /* 6176262f661SAtish Patra * Early ioremap fixmap is already created as it lies within first 2MB 6186262f661SAtish Patra * of fixmap region. We always map PMD_SIZE. Thus, both FIX_BTMAP_END 6196262f661SAtish Patra * FIX_BTMAP_BEGIN should lie in the same pmd. Verify that and warn 6206262f661SAtish Patra * the user if not. 6216262f661SAtish Patra */ 6226262f661SAtish Patra fix_bmap_spmd = fixmap_pmd[pmd_index(__fix_to_virt(FIX_BTMAP_BEGIN))]; 6236262f661SAtish Patra fix_bmap_epmd = fixmap_pmd[pmd_index(__fix_to_virt(FIX_BTMAP_END))]; 6246262f661SAtish Patra if (pmd_val(fix_bmap_spmd) != pmd_val(fix_bmap_epmd)) { 6256262f661SAtish Patra WARN_ON(1); 6266262f661SAtish Patra pr_warn("fixmap btmap start [%08lx] != end [%08lx]\n", 6276262f661SAtish Patra pmd_val(fix_bmap_spmd), pmd_val(fix_bmap_epmd)); 6286262f661SAtish Patra pr_warn("fix_to_virt(FIX_BTMAP_BEGIN): %08lx\n", 6296262f661SAtish Patra fix_to_virt(FIX_BTMAP_BEGIN)); 6306262f661SAtish Patra pr_warn("fix_to_virt(FIX_BTMAP_END): %08lx\n", 6316262f661SAtish Patra fix_to_virt(FIX_BTMAP_END)); 6326262f661SAtish Patra 6336262f661SAtish Patra pr_warn("FIX_BTMAP_END: %d\n", FIX_BTMAP_END); 6346262f661SAtish Patra pr_warn("FIX_BTMAP_BEGIN: %d\n", FIX_BTMAP_BEGIN); 6356262f661SAtish Patra } 6366262f661SAtish Patra #endif 6376f1e9e94SAnup Patel } 638f2c17aabSAnup Patel 6398d91b097SGeert Uytterhoeven #if defined(CONFIG_64BIT) && defined(CONFIG_STRICT_KERNEL_RWX) 640*01062356SJisheng Zhang void __init protect_kernel_linear_mapping_text_rodata(void) 6412bfc6cd8SAlexandre Ghiti { 6422bfc6cd8SAlexandre Ghiti unsigned long text_start = (unsigned long)lm_alias(_start); 6432bfc6cd8SAlexandre Ghiti unsigned long init_text_start = (unsigned long)lm_alias(__init_text_begin); 6442bfc6cd8SAlexandre Ghiti unsigned long rodata_start = (unsigned long)lm_alias(__start_rodata); 6452bfc6cd8SAlexandre Ghiti unsigned long data_start = (unsigned long)lm_alias(_data); 6462bfc6cd8SAlexandre Ghiti 6472bfc6cd8SAlexandre Ghiti set_memory_ro(text_start, (init_text_start - text_start) >> PAGE_SHIFT); 6482bfc6cd8SAlexandre Ghiti set_memory_nx(text_start, (init_text_start - text_start) >> PAGE_SHIFT); 6492bfc6cd8SAlexandre Ghiti 6502bfc6cd8SAlexandre Ghiti set_memory_ro(rodata_start, (data_start - rodata_start) >> PAGE_SHIFT); 6512bfc6cd8SAlexandre Ghiti set_memory_nx(rodata_start, (data_start - rodata_start) >> PAGE_SHIFT); 6522bfc6cd8SAlexandre Ghiti } 6532bfc6cd8SAlexandre Ghiti #endif 6542bfc6cd8SAlexandre Ghiti 655671f9a3eSAnup Patel static void __init setup_vm_final(void) 656671f9a3eSAnup Patel { 657671f9a3eSAnup Patel uintptr_t va, map_size; 658671f9a3eSAnup Patel phys_addr_t pa, start, end; 659b10d6bcaSMike Rapoport u64 i; 660671f9a3eSAnup Patel 661e8dcb61fSAtish Patra /** 662e8dcb61fSAtish Patra * MMU is enabled at this point. But page table setup is not complete yet. 663e8dcb61fSAtish Patra * fixmap page table alloc functions should be used at this point 664e8dcb61fSAtish Patra */ 665e8dcb61fSAtish Patra pt_ops.alloc_pte = alloc_pte_fixmap; 666e8dcb61fSAtish Patra pt_ops.get_pte_virt = get_pte_virt_fixmap; 667e8dcb61fSAtish Patra #ifndef __PAGETABLE_PMD_FOLDED 668e8dcb61fSAtish Patra pt_ops.alloc_pmd = alloc_pmd_fixmap; 669e8dcb61fSAtish Patra pt_ops.get_pmd_virt = get_pmd_virt_fixmap; 670e8dcb61fSAtish Patra #endif 671671f9a3eSAnup Patel /* Setup swapper PGD for fixmap */ 672671f9a3eSAnup Patel create_pgd_mapping(swapper_pg_dir, FIXADDR_START, 673ac51e005SZong Li __pa_symbol(fixmap_pgd_next), 674671f9a3eSAnup Patel PGDIR_SIZE, PAGE_TABLE); 675671f9a3eSAnup Patel 6762bfc6cd8SAlexandre Ghiti /* Map all memory banks in the linear mapping */ 677b10d6bcaSMike Rapoport for_each_mem_range(i, &start, &end) { 678671f9a3eSAnup Patel if (start >= end) 679671f9a3eSAnup Patel break; 680671f9a3eSAnup Patel if (start <= __pa(PAGE_OFFSET) && 681671f9a3eSAnup Patel __pa(PAGE_OFFSET) < end) 682671f9a3eSAnup Patel start = __pa(PAGE_OFFSET); 683671f9a3eSAnup Patel 684671f9a3eSAnup Patel map_size = best_map_size(start, end - start); 685671f9a3eSAnup Patel for (pa = start; pa < end; pa += map_size) { 686671f9a3eSAnup Patel va = (uintptr_t)__va(pa); 687671f9a3eSAnup Patel create_pgd_mapping(swapper_pg_dir, va, pa, 6882bfc6cd8SAlexandre Ghiti map_size, 6892bfc6cd8SAlexandre Ghiti #ifdef CONFIG_64BIT 6902bfc6cd8SAlexandre Ghiti PAGE_KERNEL 6912bfc6cd8SAlexandre Ghiti #else 6922bfc6cd8SAlexandre Ghiti PAGE_KERNEL_EXEC 6932bfc6cd8SAlexandre Ghiti #endif 6942bfc6cd8SAlexandre Ghiti ); 6952bfc6cd8SAlexandre Ghiti 696671f9a3eSAnup Patel } 697671f9a3eSAnup Patel } 698671f9a3eSAnup Patel 6992bfc6cd8SAlexandre Ghiti #ifdef CONFIG_64BIT 7002bfc6cd8SAlexandre Ghiti /* Map the kernel */ 7012bfc6cd8SAlexandre Ghiti create_kernel_page_table(swapper_pg_dir, PMD_SIZE); 7022bfc6cd8SAlexandre Ghiti #endif 7032bfc6cd8SAlexandre Ghiti 704671f9a3eSAnup Patel /* Clear fixmap PTE and PMD mappings */ 705671f9a3eSAnup Patel clear_fixmap(FIX_PTE); 706671f9a3eSAnup Patel clear_fixmap(FIX_PMD); 707671f9a3eSAnup Patel 708671f9a3eSAnup Patel /* Move to swapper page table */ 709ac51e005SZong Li csr_write(CSR_SATP, PFN_DOWN(__pa_symbol(swapper_pg_dir)) | SATP_MODE); 710671f9a3eSAnup Patel local_flush_tlb_all(); 711e8dcb61fSAtish Patra 712e8dcb61fSAtish Patra /* generic page allocation functions must be used to setup page table */ 713e8dcb61fSAtish Patra pt_ops.alloc_pte = alloc_pte_late; 714e8dcb61fSAtish Patra pt_ops.get_pte_virt = get_pte_virt_late; 715e8dcb61fSAtish Patra #ifndef __PAGETABLE_PMD_FOLDED 716e8dcb61fSAtish Patra pt_ops.alloc_pmd = alloc_pmd_late; 717e8dcb61fSAtish Patra pt_ops.get_pmd_virt = get_pmd_virt_late; 718e8dcb61fSAtish Patra #endif 719671f9a3eSAnup Patel } 7206bd33e1eSChristoph Hellwig #else 7216bd33e1eSChristoph Hellwig asmlinkage void __init setup_vm(uintptr_t dtb_pa) 7226bd33e1eSChristoph Hellwig { 7236bd33e1eSChristoph Hellwig dtb_early_va = (void *)dtb_pa; 724a78c6f59SAtish Patra dtb_early_pa = dtb_pa; 7256bd33e1eSChristoph Hellwig } 7266bd33e1eSChristoph Hellwig 7276bd33e1eSChristoph Hellwig static inline void setup_vm_final(void) 7286bd33e1eSChristoph Hellwig { 7296bd33e1eSChristoph Hellwig } 7306bd33e1eSChristoph Hellwig #endif /* CONFIG_MMU */ 731671f9a3eSAnup Patel 732d27c3c90SZong Li #ifdef CONFIG_STRICT_KERNEL_RWX 7331987501bSJisheng Zhang void __init protect_kernel_text_data(void) 734d27c3c90SZong Li { 73519a00869SAtish Patra unsigned long text_start = (unsigned long)_start; 73619a00869SAtish Patra unsigned long init_text_start = (unsigned long)__init_text_begin; 73719a00869SAtish Patra unsigned long init_data_start = (unsigned long)__init_data_begin; 738d27c3c90SZong Li unsigned long rodata_start = (unsigned long)__start_rodata; 739d27c3c90SZong Li unsigned long data_start = (unsigned long)_data; 740d27c3c90SZong Li unsigned long max_low = (unsigned long)(__va(PFN_PHYS(max_low_pfn))); 741d27c3c90SZong Li 74219a00869SAtish Patra set_memory_ro(text_start, (init_text_start - text_start) >> PAGE_SHIFT); 74319a00869SAtish Patra set_memory_ro(init_text_start, (init_data_start - init_text_start) >> PAGE_SHIFT); 74419a00869SAtish Patra set_memory_nx(init_data_start, (rodata_start - init_data_start) >> PAGE_SHIFT); 74519a00869SAtish Patra /* rodata section is marked readonly in mark_rodata_ro */ 746d27c3c90SZong Li set_memory_nx(rodata_start, (data_start - rodata_start) >> PAGE_SHIFT); 747d27c3c90SZong Li set_memory_nx(data_start, (max_low - data_start) >> PAGE_SHIFT); 74819a00869SAtish Patra } 74919a00869SAtish Patra 75019a00869SAtish Patra void mark_rodata_ro(void) 75119a00869SAtish Patra { 75219a00869SAtish Patra unsigned long rodata_start = (unsigned long)__start_rodata; 75319a00869SAtish Patra unsigned long data_start = (unsigned long)_data; 75419a00869SAtish Patra 75519a00869SAtish Patra set_memory_ro(rodata_start, (data_start - rodata_start) >> PAGE_SHIFT); 756b422d28bSZong Li 757b422d28bSZong Li debug_checkwx(); 758d27c3c90SZong Li } 759d27c3c90SZong Li #endif 760d27c3c90SZong Li 761e53d2818SNick Kossifidis #ifdef CONFIG_KEXEC_CORE 762e53d2818SNick Kossifidis /* 763e53d2818SNick Kossifidis * reserve_crashkernel() - reserves memory for crash kernel 764e53d2818SNick Kossifidis * 765e53d2818SNick Kossifidis * This function reserves memory area given in "crashkernel=" kernel command 766e53d2818SNick Kossifidis * line parameter. The memory reserved is used by dump capture kernel when 767e53d2818SNick Kossifidis * primary kernel is crashing. 768e53d2818SNick Kossifidis */ 769e53d2818SNick Kossifidis static void __init reserve_crashkernel(void) 770e53d2818SNick Kossifidis { 771e53d2818SNick Kossifidis unsigned long long crash_base = 0; 772e53d2818SNick Kossifidis unsigned long long crash_size = 0; 773e53d2818SNick Kossifidis unsigned long search_start = memblock_start_of_DRAM(); 774e53d2818SNick Kossifidis unsigned long search_end = memblock_end_of_DRAM(); 775e53d2818SNick Kossifidis 776e53d2818SNick Kossifidis int ret = 0; 777e53d2818SNick Kossifidis 77856409750SNick Kossifidis /* 77956409750SNick Kossifidis * Don't reserve a region for a crash kernel on a crash kernel 78056409750SNick Kossifidis * since it doesn't make much sense and we have limited memory 78156409750SNick Kossifidis * resources. 78256409750SNick Kossifidis */ 78356409750SNick Kossifidis #ifdef CONFIG_CRASH_DUMP 78456409750SNick Kossifidis if (is_kdump_kernel()) { 78556409750SNick Kossifidis pr_info("crashkernel: ignoring reservation request\n"); 78656409750SNick Kossifidis return; 78756409750SNick Kossifidis } 78856409750SNick Kossifidis #endif 78956409750SNick Kossifidis 790e53d2818SNick Kossifidis ret = parse_crashkernel(boot_command_line, memblock_phys_mem_size(), 791e53d2818SNick Kossifidis &crash_size, &crash_base); 792e53d2818SNick Kossifidis if (ret || !crash_size) 793e53d2818SNick Kossifidis return; 794e53d2818SNick Kossifidis 795e53d2818SNick Kossifidis crash_size = PAGE_ALIGN(crash_size); 796e53d2818SNick Kossifidis 797e53d2818SNick Kossifidis if (crash_base == 0) { 798e53d2818SNick Kossifidis /* 799e53d2818SNick Kossifidis * Current riscv boot protocol requires 2MB alignment for 800e53d2818SNick Kossifidis * RV64 and 4MB alignment for RV32 (hugepage size) 801e53d2818SNick Kossifidis */ 802e53d2818SNick Kossifidis crash_base = memblock_find_in_range(search_start, search_end, 803e53d2818SNick Kossifidis crash_size, PMD_SIZE); 804e53d2818SNick Kossifidis 805e53d2818SNick Kossifidis if (crash_base == 0) { 806e53d2818SNick Kossifidis pr_warn("crashkernel: couldn't allocate %lldKB\n", 807e53d2818SNick Kossifidis crash_size >> 10); 808e53d2818SNick Kossifidis return; 809e53d2818SNick Kossifidis } 810e53d2818SNick Kossifidis } else { 811e53d2818SNick Kossifidis /* User specifies base address explicitly. */ 812e53d2818SNick Kossifidis if (!memblock_is_region_memory(crash_base, crash_size)) { 813e53d2818SNick Kossifidis pr_warn("crashkernel: requested region is not memory\n"); 814e53d2818SNick Kossifidis return; 815e53d2818SNick Kossifidis } 816e53d2818SNick Kossifidis 817e53d2818SNick Kossifidis if (memblock_is_region_reserved(crash_base, crash_size)) { 818e53d2818SNick Kossifidis pr_warn("crashkernel: requested region is reserved\n"); 819e53d2818SNick Kossifidis return; 820e53d2818SNick Kossifidis } 821e53d2818SNick Kossifidis 822e53d2818SNick Kossifidis 823e53d2818SNick Kossifidis if (!IS_ALIGNED(crash_base, PMD_SIZE)) { 824e53d2818SNick Kossifidis pr_warn("crashkernel: requested region is misaligned\n"); 825e53d2818SNick Kossifidis return; 826e53d2818SNick Kossifidis } 827e53d2818SNick Kossifidis } 828e53d2818SNick Kossifidis memblock_reserve(crash_base, crash_size); 829e53d2818SNick Kossifidis 830e53d2818SNick Kossifidis pr_info("crashkernel: reserved 0x%016llx - 0x%016llx (%lld MB)\n", 831e53d2818SNick Kossifidis crash_base, crash_base + crash_size, crash_size >> 20); 832e53d2818SNick Kossifidis 833e53d2818SNick Kossifidis crashk_res.start = crash_base; 834e53d2818SNick Kossifidis crashk_res.end = crash_base + crash_size - 1; 835e53d2818SNick Kossifidis } 836e53d2818SNick Kossifidis #endif /* CONFIG_KEXEC_CORE */ 837e53d2818SNick Kossifidis 83856409750SNick Kossifidis #ifdef CONFIG_CRASH_DUMP 83956409750SNick Kossifidis /* 84056409750SNick Kossifidis * We keep track of the ELF core header of the crashed 84156409750SNick Kossifidis * kernel with a reserved-memory region with compatible 84256409750SNick Kossifidis * string "linux,elfcorehdr". Here we register a callback 84356409750SNick Kossifidis * to populate elfcorehdr_addr/size when this region is 84456409750SNick Kossifidis * present. Note that this region will be marked as 84556409750SNick Kossifidis * reserved once we call early_init_fdt_scan_reserved_mem() 84656409750SNick Kossifidis * later on. 84756409750SNick Kossifidis */ 848*01062356SJisheng Zhang static int __init elfcore_hdr_setup(struct reserved_mem *rmem) 84956409750SNick Kossifidis { 85056409750SNick Kossifidis elfcorehdr_addr = rmem->base; 85156409750SNick Kossifidis elfcorehdr_size = rmem->size; 85256409750SNick Kossifidis return 0; 85356409750SNick Kossifidis } 85456409750SNick Kossifidis 85556409750SNick Kossifidis RESERVEDMEM_OF_DECLARE(elfcorehdr, "linux,elfcorehdr", elfcore_hdr_setup); 85656409750SNick Kossifidis #endif 85756409750SNick Kossifidis 858671f9a3eSAnup Patel void __init paging_init(void) 859671f9a3eSAnup Patel { 860f842f5ffSKefeng Wang setup_bootmem(); 861671f9a3eSAnup Patel setup_vm_final(); 862cbd34f4bSAtish Patra } 863cbd34f4bSAtish Patra 864cbd34f4bSAtish Patra void __init misc_mem_init(void) 865cbd34f4bSAtish Patra { 866f6e5aedfSKefeng Wang early_memtest(min_low_pfn << PAGE_SHIFT, max_low_pfn << PAGE_SHIFT); 8674f0e8eefSAtish Patra arch_numa_init(); 868cbd34f4bSAtish Patra sparse_init(); 869671f9a3eSAnup Patel zone_sizes_init(); 870e53d2818SNick Kossifidis #ifdef CONFIG_KEXEC_CORE 871e53d2818SNick Kossifidis reserve_crashkernel(); 872e53d2818SNick Kossifidis #endif 8734f0e8eefSAtish Patra memblock_dump_all(); 8746f1e9e94SAnup Patel } 875d95f1a54SLogan Gunthorpe 8769fe57d8cSKefeng Wang #ifdef CONFIG_SPARSEMEM_VMEMMAP 877d95f1a54SLogan Gunthorpe int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node, 878d95f1a54SLogan Gunthorpe struct vmem_altmap *altmap) 879d95f1a54SLogan Gunthorpe { 8801d9cfee7SAnshuman Khandual return vmemmap_populate_basepages(start, end, node, NULL); 881d95f1a54SLogan Gunthorpe } 882d95f1a54SLogan Gunthorpe #endif 883