150acfb2bSThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only 276d2a049SPalmer Dabbelt /* 376d2a049SPalmer Dabbelt * Copyright (C) 2012 Regents of the University of California 4671f9a3eSAnup Patel * Copyright (C) 2019 Western Digital Corporation or its affiliates. 5e53d2818SNick Kossifidis * Copyright (C) 2020 FORTH-ICS/CARV 6e53d2818SNick Kossifidis * Nick Kossifidis <mick@ics.forth.gr> 776d2a049SPalmer Dabbelt */ 876d2a049SPalmer Dabbelt 976d2a049SPalmer Dabbelt #include <linux/init.h> 1076d2a049SPalmer Dabbelt #include <linux/mm.h> 1176d2a049SPalmer Dabbelt #include <linux/memblock.h> 1257c8a661SMike Rapoport #include <linux/initrd.h> 1376d2a049SPalmer Dabbelt #include <linux/swap.h> 14ce3aca04SKefeng Wang #include <linux/swiotlb.h> 155ec9c4ffSChristoph Hellwig #include <linux/sizes.h> 160651c263SAnup Patel #include <linux/of_fdt.h> 1756409750SNick Kossifidis #include <linux/of_reserved_mem.h> 18922b0375SAlbert Ou #include <linux/libfdt.h> 19d27c3c90SZong Li #include <linux/set_memory.h> 20da815582SKefeng Wang #include <linux/dma-map-ops.h> 21e53d2818SNick Kossifidis #include <linux/crash_dump.h> 2276d2a049SPalmer Dabbelt 23f2c17aabSAnup Patel #include <asm/fixmap.h> 2476d2a049SPalmer Dabbelt #include <asm/tlbflush.h> 2576d2a049SPalmer Dabbelt #include <asm/sections.h> 262d268251SPalmer Dabbelt #include <asm/soc.h> 2776d2a049SPalmer Dabbelt #include <asm/io.h> 28b422d28bSZong Li #include <asm/ptdump.h> 294f0e8eefSAtish Patra #include <asm/numa.h> 3076d2a049SPalmer Dabbelt 31ffaee272SPaul Walmsley #include "../kernel/head.h" 32ffaee272SPaul Walmsley 332bfc6cd8SAlexandre Ghiti unsigned long kernel_virt_addr = KERNEL_LINK_ADDR; 342bfc6cd8SAlexandre Ghiti EXPORT_SYMBOL(kernel_virt_addr); 3544c92257SVitaly Wool #ifdef CONFIG_XIP_KERNEL 3644c92257SVitaly Wool #define kernel_virt_addr (*((unsigned long *)XIP_FIXUP(&kernel_virt_addr))) 3750bae95eSKefeng Wang extern char _xiprom[], _exiprom[]; 3844c92257SVitaly Wool #endif 392bfc6cd8SAlexandre Ghiti 40387181dcSAnup Patel unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)] 41387181dcSAnup Patel __page_aligned_bss; 42387181dcSAnup Patel EXPORT_SYMBOL(empty_zero_page); 43387181dcSAnup Patel 44d90d45d7SAnup Patel extern char _start[]; 458f3a2b4aSAnup Patel #define DTB_EARLY_BASE_VA PGDIR_SIZE 4644c92257SVitaly Wool void *_dtb_early_va __initdata; 4744c92257SVitaly Wool uintptr_t _dtb_early_pa __initdata; 48d90d45d7SAnup Patel 49e8dcb61fSAtish Patra struct pt_alloc_ops { 50e8dcb61fSAtish Patra pte_t *(*get_pte_virt)(phys_addr_t pa); 51e8dcb61fSAtish Patra phys_addr_t (*alloc_pte)(uintptr_t va); 52e8dcb61fSAtish Patra #ifndef __PAGETABLE_PMD_FOLDED 53e8dcb61fSAtish Patra pmd_t *(*get_pmd_virt)(phys_addr_t pa); 54e8dcb61fSAtish Patra phys_addr_t (*alloc_pmd)(uintptr_t va); 55e8dcb61fSAtish Patra #endif 56e8dcb61fSAtish Patra }; 5776d2a049SPalmer Dabbelt 5801062356SJisheng Zhang static phys_addr_t dma32_phys_limit __initdata; 59da815582SKefeng Wang 6076d2a049SPalmer Dabbelt static void __init zone_sizes_init(void) 6176d2a049SPalmer Dabbelt { 625ec9c4ffSChristoph Hellwig unsigned long max_zone_pfns[MAX_NR_ZONES] = { 0, }; 6376d2a049SPalmer Dabbelt 64d5fad48cSZong Li #ifdef CONFIG_ZONE_DMA32 65da815582SKefeng Wang max_zone_pfns[ZONE_DMA32] = PFN_DOWN(dma32_phys_limit); 66d5fad48cSZong Li #endif 675ec9c4ffSChristoph Hellwig max_zone_pfns[ZONE_NORMAL] = max_low_pfn; 685ec9c4ffSChristoph Hellwig 699691a071SMike Rapoport free_area_init(max_zone_pfns); 7076d2a049SPalmer Dabbelt } 7176d2a049SPalmer Dabbelt 728fa3cdffSKefeng Wang #if defined(CONFIG_MMU) && defined(CONFIG_DEBUG_VM) 732cc6c4a0SYash Shah static inline void print_mlk(char *name, unsigned long b, unsigned long t) 742cc6c4a0SYash Shah { 752cc6c4a0SYash Shah pr_notice("%12s : 0x%08lx - 0x%08lx (%4ld kB)\n", name, b, t, 762cc6c4a0SYash Shah (((t) - (b)) >> 10)); 772cc6c4a0SYash Shah } 782cc6c4a0SYash Shah 792cc6c4a0SYash Shah static inline void print_mlm(char *name, unsigned long b, unsigned long t) 802cc6c4a0SYash Shah { 812cc6c4a0SYash Shah pr_notice("%12s : 0x%08lx - 0x%08lx (%4ld MB)\n", name, b, t, 822cc6c4a0SYash Shah (((t) - (b)) >> 20)); 832cc6c4a0SYash Shah } 842cc6c4a0SYash Shah 851987501bSJisheng Zhang static void __init print_vm_layout(void) 862cc6c4a0SYash Shah { 872cc6c4a0SYash Shah pr_notice("Virtual kernel memory layout:\n"); 882cc6c4a0SYash Shah print_mlk("fixmap", (unsigned long)FIXADDR_START, 892cc6c4a0SYash Shah (unsigned long)FIXADDR_TOP); 902cc6c4a0SYash Shah print_mlm("pci io", (unsigned long)PCI_IO_START, 912cc6c4a0SYash Shah (unsigned long)PCI_IO_END); 922cc6c4a0SYash Shah print_mlm("vmemmap", (unsigned long)VMEMMAP_START, 932cc6c4a0SYash Shah (unsigned long)VMEMMAP_END); 942cc6c4a0SYash Shah print_mlm("vmalloc", (unsigned long)VMALLOC_START, 952cc6c4a0SYash Shah (unsigned long)VMALLOC_END); 962cc6c4a0SYash Shah print_mlm("lowmem", (unsigned long)PAGE_OFFSET, 972cc6c4a0SYash Shah (unsigned long)high_memory); 982bfc6cd8SAlexandre Ghiti #ifdef CONFIG_64BIT 992bfc6cd8SAlexandre Ghiti print_mlm("kernel", (unsigned long)KERNEL_LINK_ADDR, 1002bfc6cd8SAlexandre Ghiti (unsigned long)ADDRESS_SPACE_END); 1012bfc6cd8SAlexandre Ghiti #endif 1022cc6c4a0SYash Shah } 1032cc6c4a0SYash Shah #else 1042cc6c4a0SYash Shah static void print_vm_layout(void) { } 1052cc6c4a0SYash Shah #endif /* CONFIG_DEBUG_VM */ 1062cc6c4a0SYash Shah 10776d2a049SPalmer Dabbelt void __init mem_init(void) 10876d2a049SPalmer Dabbelt { 10976d2a049SPalmer Dabbelt #ifdef CONFIG_FLATMEM 11076d2a049SPalmer Dabbelt BUG_ON(!mem_map); 11176d2a049SPalmer Dabbelt #endif /* CONFIG_FLATMEM */ 11276d2a049SPalmer Dabbelt 113ce3aca04SKefeng Wang #ifdef CONFIG_SWIOTLB 114ce3aca04SKefeng Wang if (swiotlb_force == SWIOTLB_FORCE || 115ce3aca04SKefeng Wang max_pfn > PFN_DOWN(dma32_phys_limit)) 116ce3aca04SKefeng Wang swiotlb_init(1); 117ce3aca04SKefeng Wang else 118ce3aca04SKefeng Wang swiotlb_force = SWIOTLB_NO_FORCE; 119ce3aca04SKefeng Wang #endif 12076d2a049SPalmer Dabbelt high_memory = (void *)(__va(PFN_PHYS(max_low_pfn))); 121c6ffc5caSMike Rapoport memblock_free_all(); 12276d2a049SPalmer Dabbelt 1232cc6c4a0SYash Shah print_vm_layout(); 12476d2a049SPalmer Dabbelt } 12576d2a049SPalmer Dabbelt 126*c9811e37SKefeng Wang /* 127*c9811e37SKefeng Wang * The default maximal physical memory size is -PAGE_OFFSET, 128*c9811e37SKefeng Wang * limit the memory size via mem. 129*c9811e37SKefeng Wang */ 130*c9811e37SKefeng Wang static phys_addr_t memory_limit = -PAGE_OFFSET; 131*c9811e37SKefeng Wang 132*c9811e37SKefeng Wang static int __init early_mem(char *p) 133*c9811e37SKefeng Wang { 134*c9811e37SKefeng Wang u64 size; 135*c9811e37SKefeng Wang 136*c9811e37SKefeng Wang if (!p) 137*c9811e37SKefeng Wang return 1; 138*c9811e37SKefeng Wang 139*c9811e37SKefeng Wang size = memparse(p, &p) & PAGE_MASK; 140*c9811e37SKefeng Wang memory_limit = min_t(u64, size, memory_limit); 141*c9811e37SKefeng Wang 142*c9811e37SKefeng Wang pr_notice("Memory limited to %lldMB\n", (u64)memory_limit >> 20); 143*c9811e37SKefeng Wang 144*c9811e37SKefeng Wang return 0; 145*c9811e37SKefeng Wang } 146*c9811e37SKefeng Wang early_param("mem", early_mem); 147*c9811e37SKefeng Wang 148f842f5ffSKefeng Wang static void __init setup_bootmem(void) 1490651c263SAnup Patel { 150ac51e005SZong Li phys_addr_t vmlinux_end = __pa_symbol(&_end); 151ac51e005SZong Li phys_addr_t vmlinux_start = __pa_symbol(&_start); 152abb8e86bSAtish Patra phys_addr_t max_mapped_addr = __pa(~(ulong)0); 153*c9811e37SKefeng Wang phys_addr_t dram_end; 1540651c263SAnup Patel 15544c92257SVitaly Wool #ifdef CONFIG_XIP_KERNEL 15644c92257SVitaly Wool vmlinux_start = __pa_symbol(&_sdata); 15744c92257SVitaly Wool #endif 15844c92257SVitaly Wool 159*c9811e37SKefeng Wang memblock_enforce_memory_limit(memory_limit); 1600651c263SAnup Patel 1612bfc6cd8SAlexandre Ghiti /* 1622bfc6cd8SAlexandre Ghiti * Reserve from the start of the kernel to the end of the kernel 1638db6f937SGeert Uytterhoeven */ 1648db6f937SGeert Uytterhoeven #if defined(CONFIG_64BIT) && defined(CONFIG_STRICT_KERNEL_RWX) 1658db6f937SGeert Uytterhoeven /* 1668db6f937SGeert Uytterhoeven * Make sure we align the reservation on PMD_SIZE since we will 1672bfc6cd8SAlexandre Ghiti * map the kernel in the linear mapping as read-only: we do not want 1682bfc6cd8SAlexandre Ghiti * any allocation to happen between _end and the next pmd aligned page. 1692bfc6cd8SAlexandre Ghiti */ 1708db6f937SGeert Uytterhoeven vmlinux_end = (vmlinux_end + PMD_SIZE - 1) & PMD_MASK; 1718db6f937SGeert Uytterhoeven #endif 1728db6f937SGeert Uytterhoeven memblock_reserve(vmlinux_start, vmlinux_end - vmlinux_start); 173d90d45d7SAnup Patel 174*c9811e37SKefeng Wang dram_end = memblock_end_of_DRAM(); 175abb8e86bSAtish Patra /* 176abb8e86bSAtish Patra * memblock allocator is not aware of the fact that last 4K bytes of 177abb8e86bSAtish Patra * the addressable memory can not be mapped because of IS_ERR_VALUE 178abb8e86bSAtish Patra * macro. Make sure that last 4k bytes are not usable by memblock 179abb8e86bSAtish Patra * if end of dram is equal to maximum addressable memory. 180abb8e86bSAtish Patra */ 181abb8e86bSAtish Patra if (max_mapped_addr == (dram_end - 1)) 182abb8e86bSAtish Patra memblock_set_current_limit(max_mapped_addr - 4096); 183abb8e86bSAtish Patra 184f6e5aedfSKefeng Wang min_low_pfn = PFN_UP(memblock_start_of_DRAM()); 185f6e5aedfSKefeng Wang max_low_pfn = max_pfn = PFN_DOWN(dram_end); 186f6e5aedfSKefeng Wang 187da815582SKefeng Wang dma32_phys_limit = min(4UL * SZ_1G, (unsigned long)PFN_PHYS(max_low_pfn)); 188336e8eb2SGuo Ren set_max_mapnr(max_low_pfn - ARCH_PFN_OFFSET); 1890651c263SAnup Patel 190aec33b54SKefeng Wang reserve_initrd_mem(); 191922b0375SAlbert Ou /* 192f105aa94SVitaly Wool * If DTB is built in, no need to reserve its memblock. 193f105aa94SVitaly Wool * Otherwise, do reserve it but avoid using 194f105aa94SVitaly Wool * early_init_fdt_reserve_self() since __pa() does 195922b0375SAlbert Ou * not work for DTB pointers that are fixmap addresses 196922b0375SAlbert Ou */ 197f105aa94SVitaly Wool if (!IS_ENABLED(CONFIG_BUILTIN_DTB)) 198922b0375SAlbert Ou memblock_reserve(dtb_early_pa, fdt_totalsize(dtb_early_va)); 199922b0375SAlbert Ou 2000651c263SAnup Patel early_init_fdt_scan_reserved_mem(); 201da815582SKefeng Wang dma_contiguous_reserve(dma32_phys_limit); 2020651c263SAnup Patel memblock_allow_resize(); 2030651c263SAnup Patel } 2046f1e9e94SAnup Patel 2056bd33e1eSChristoph Hellwig #ifdef CONFIG_MMU 20601062356SJisheng Zhang static struct pt_alloc_ops _pt_ops __initdata; 20744c92257SVitaly Wool 20844c92257SVitaly Wool #ifdef CONFIG_XIP_KERNEL 20944c92257SVitaly Wool #define pt_ops (*(struct pt_alloc_ops *)XIP_FIXUP(&_pt_ops)) 21044c92257SVitaly Wool #else 21144c92257SVitaly Wool #define pt_ops _pt_ops 21244c92257SVitaly Wool #endif 213e8dcb61fSAtish Patra 2142bfc6cd8SAlexandre Ghiti /* Offset between linear mapping virtual address and kernel load address */ 215de31ea4aSJisheng Zhang unsigned long va_pa_offset __ro_after_init; 216387181dcSAnup Patel EXPORT_SYMBOL(va_pa_offset); 21744c92257SVitaly Wool #ifdef CONFIG_XIP_KERNEL 21844c92257SVitaly Wool #define va_pa_offset (*((unsigned long *)XIP_FIXUP(&va_pa_offset))) 21944c92257SVitaly Wool #endif 2202bfc6cd8SAlexandre Ghiti /* Offset between kernel mapping virtual address and kernel load address */ 22144c92257SVitaly Wool #ifdef CONFIG_64BIT 22201062356SJisheng Zhang unsigned long va_kernel_pa_offset __ro_after_init; 2232bfc6cd8SAlexandre Ghiti EXPORT_SYMBOL(va_kernel_pa_offset); 2242bfc6cd8SAlexandre Ghiti #endif 22544c92257SVitaly Wool #ifdef CONFIG_XIP_KERNEL 22644c92257SVitaly Wool #define va_kernel_pa_offset (*((unsigned long *)XIP_FIXUP(&va_kernel_pa_offset))) 22744c92257SVitaly Wool #endif 22801062356SJisheng Zhang unsigned long va_kernel_xip_pa_offset __ro_after_init; 22944c92257SVitaly Wool EXPORT_SYMBOL(va_kernel_xip_pa_offset); 23044c92257SVitaly Wool #ifdef CONFIG_XIP_KERNEL 23144c92257SVitaly Wool #define va_kernel_xip_pa_offset (*((unsigned long *)XIP_FIXUP(&va_kernel_xip_pa_offset))) 23244c92257SVitaly Wool #endif 233de31ea4aSJisheng Zhang unsigned long pfn_base __ro_after_init; 234387181dcSAnup Patel EXPORT_SYMBOL(pfn_base); 235387181dcSAnup Patel 2366f1e9e94SAnup Patel pgd_t swapper_pg_dir[PTRS_PER_PGD] __page_aligned_bss; 237671f9a3eSAnup Patel pgd_t trampoline_pg_dir[PTRS_PER_PGD] __page_aligned_bss; 23801062356SJisheng Zhang static pte_t fixmap_pte[PTRS_PER_PTE] __page_aligned_bss; 239671f9a3eSAnup Patel 240671f9a3eSAnup Patel pgd_t early_pg_dir[PTRS_PER_PGD] __initdata __aligned(PAGE_SIZE); 241f2c17aabSAnup Patel 24244c92257SVitaly Wool #ifdef CONFIG_XIP_KERNEL 24344c92257SVitaly Wool #define trampoline_pg_dir ((pgd_t *)XIP_FIXUP(trampoline_pg_dir)) 24444c92257SVitaly Wool #define fixmap_pte ((pte_t *)XIP_FIXUP(fixmap_pte)) 24544c92257SVitaly Wool #define early_pg_dir ((pgd_t *)XIP_FIXUP(early_pg_dir)) 24644c92257SVitaly Wool #endif /* CONFIG_XIP_KERNEL */ 24744c92257SVitaly Wool 248f2c17aabSAnup Patel void __set_fixmap(enum fixed_addresses idx, phys_addr_t phys, pgprot_t prot) 249f2c17aabSAnup Patel { 250f2c17aabSAnup Patel unsigned long addr = __fix_to_virt(idx); 251f2c17aabSAnup Patel pte_t *ptep; 252f2c17aabSAnup Patel 253f2c17aabSAnup Patel BUG_ON(idx <= FIX_HOLE || idx >= __end_of_fixed_addresses); 254f2c17aabSAnup Patel 255f2c17aabSAnup Patel ptep = &fixmap_pte[pte_index(addr)]; 256f2c17aabSAnup Patel 25721190b74SGreentime Hu if (pgprot_val(prot)) 258f2c17aabSAnup Patel set_pte(ptep, pfn_pte(phys >> PAGE_SHIFT, prot)); 25921190b74SGreentime Hu else 260f2c17aabSAnup Patel pte_clear(&init_mm, addr, ptep); 261f2c17aabSAnup Patel local_flush_tlb_page(addr); 262f2c17aabSAnup Patel } 263f2c17aabSAnup Patel 264e8dcb61fSAtish Patra static inline pte_t *__init get_pte_virt_early(phys_addr_t pa) 265671f9a3eSAnup Patel { 266671f9a3eSAnup Patel return (pte_t *)((uintptr_t)pa); 267671f9a3eSAnup Patel } 268e8dcb61fSAtish Patra 269e8dcb61fSAtish Patra static inline pte_t *__init get_pte_virt_fixmap(phys_addr_t pa) 270e8dcb61fSAtish Patra { 271e8dcb61fSAtish Patra clear_fixmap(FIX_PTE); 272e8dcb61fSAtish Patra return (pte_t *)set_fixmap_offset(FIX_PTE, pa); 273671f9a3eSAnup Patel } 274671f9a3eSAnup Patel 27501062356SJisheng Zhang static inline pte_t *__init get_pte_virt_late(phys_addr_t pa) 276e8dcb61fSAtish Patra { 277e8dcb61fSAtish Patra return (pte_t *) __va(pa); 278e8dcb61fSAtish Patra } 279e8dcb61fSAtish Patra 280e8dcb61fSAtish Patra static inline phys_addr_t __init alloc_pte_early(uintptr_t va) 281671f9a3eSAnup Patel { 282671f9a3eSAnup Patel /* 283671f9a3eSAnup Patel * We only create PMD or PGD early mappings so we 284671f9a3eSAnup Patel * should never reach here with MMU disabled. 285671f9a3eSAnup Patel */ 286e8dcb61fSAtish Patra BUG(); 287e8dcb61fSAtish Patra } 288671f9a3eSAnup Patel 289e8dcb61fSAtish Patra static inline phys_addr_t __init alloc_pte_fixmap(uintptr_t va) 290e8dcb61fSAtish Patra { 291671f9a3eSAnup Patel return memblock_phys_alloc(PAGE_SIZE, PAGE_SIZE); 292671f9a3eSAnup Patel } 293671f9a3eSAnup Patel 29401062356SJisheng Zhang static phys_addr_t __init alloc_pte_late(uintptr_t va) 295e8dcb61fSAtish Patra { 296e8dcb61fSAtish Patra unsigned long vaddr; 297e8dcb61fSAtish Patra 298e8dcb61fSAtish Patra vaddr = __get_free_page(GFP_KERNEL); 299e75e6bf4Szhouchuangao BUG_ON(!vaddr || !pgtable_pte_page_ctor(virt_to_page(vaddr))); 300e75e6bf4Szhouchuangao 301e8dcb61fSAtish Patra return __pa(vaddr); 302e8dcb61fSAtish Patra } 303e8dcb61fSAtish Patra 304671f9a3eSAnup Patel static void __init create_pte_mapping(pte_t *ptep, 305671f9a3eSAnup Patel uintptr_t va, phys_addr_t pa, 306671f9a3eSAnup Patel phys_addr_t sz, pgprot_t prot) 307671f9a3eSAnup Patel { 308974b9b2cSMike Rapoport uintptr_t pte_idx = pte_index(va); 309671f9a3eSAnup Patel 310671f9a3eSAnup Patel BUG_ON(sz != PAGE_SIZE); 311671f9a3eSAnup Patel 312974b9b2cSMike Rapoport if (pte_none(ptep[pte_idx])) 313974b9b2cSMike Rapoport ptep[pte_idx] = pfn_pte(PFN_DOWN(pa), prot); 314671f9a3eSAnup Patel } 315671f9a3eSAnup Patel 316671f9a3eSAnup Patel #ifndef __PAGETABLE_PMD_FOLDED 317671f9a3eSAnup Patel 31801062356SJisheng Zhang static pmd_t trampoline_pmd[PTRS_PER_PMD] __page_aligned_bss; 31901062356SJisheng Zhang static pmd_t fixmap_pmd[PTRS_PER_PMD] __page_aligned_bss; 32001062356SJisheng Zhang static pmd_t early_pmd[PTRS_PER_PMD] __initdata __aligned(PAGE_SIZE); 32101062356SJisheng Zhang static pmd_t early_dtb_pmd[PTRS_PER_PMD] __initdata __aligned(PAGE_SIZE); 322671f9a3eSAnup Patel 32344c92257SVitaly Wool #ifdef CONFIG_XIP_KERNEL 32444c92257SVitaly Wool #define trampoline_pmd ((pmd_t *)XIP_FIXUP(trampoline_pmd)) 32544c92257SVitaly Wool #define fixmap_pmd ((pmd_t *)XIP_FIXUP(fixmap_pmd)) 32644c92257SVitaly Wool #define early_pmd ((pmd_t *)XIP_FIXUP(early_pmd)) 32744c92257SVitaly Wool #endif /* CONFIG_XIP_KERNEL */ 32844c92257SVitaly Wool 329e8dcb61fSAtish Patra static pmd_t *__init get_pmd_virt_early(phys_addr_t pa) 330671f9a3eSAnup Patel { 331e8dcb61fSAtish Patra /* Before MMU is enabled */ 332671f9a3eSAnup Patel return (pmd_t *)((uintptr_t)pa); 333671f9a3eSAnup Patel } 334e8dcb61fSAtish Patra 335e8dcb61fSAtish Patra static pmd_t *__init get_pmd_virt_fixmap(phys_addr_t pa) 336e8dcb61fSAtish Patra { 337e8dcb61fSAtish Patra clear_fixmap(FIX_PMD); 338e8dcb61fSAtish Patra return (pmd_t *)set_fixmap_offset(FIX_PMD, pa); 339671f9a3eSAnup Patel } 340671f9a3eSAnup Patel 34101062356SJisheng Zhang static pmd_t *__init get_pmd_virt_late(phys_addr_t pa) 342e8dcb61fSAtish Patra { 343e8dcb61fSAtish Patra return (pmd_t *) __va(pa); 344e8dcb61fSAtish Patra } 345e8dcb61fSAtish Patra 346e8dcb61fSAtish Patra static phys_addr_t __init alloc_pmd_early(uintptr_t va) 347671f9a3eSAnup Patel { 3482bfc6cd8SAlexandre Ghiti BUG_ON((va - kernel_virt_addr) >> PGDIR_SHIFT); 349671f9a3eSAnup Patel 3500f02de44SAlexandre Ghiti return (uintptr_t)early_pmd; 351671f9a3eSAnup Patel } 352671f9a3eSAnup Patel 353e8dcb61fSAtish Patra static phys_addr_t __init alloc_pmd_fixmap(uintptr_t va) 354e8dcb61fSAtish Patra { 355e8dcb61fSAtish Patra return memblock_phys_alloc(PAGE_SIZE, PAGE_SIZE); 356e8dcb61fSAtish Patra } 357e8dcb61fSAtish Patra 35801062356SJisheng Zhang static phys_addr_t __init alloc_pmd_late(uintptr_t va) 359e8dcb61fSAtish Patra { 360e8dcb61fSAtish Patra unsigned long vaddr; 361e8dcb61fSAtish Patra 362e8dcb61fSAtish Patra vaddr = __get_free_page(GFP_KERNEL); 363e8dcb61fSAtish Patra BUG_ON(!vaddr); 364e8dcb61fSAtish Patra return __pa(vaddr); 365e8dcb61fSAtish Patra } 366e8dcb61fSAtish Patra 367671f9a3eSAnup Patel static void __init create_pmd_mapping(pmd_t *pmdp, 368671f9a3eSAnup Patel uintptr_t va, phys_addr_t pa, 369671f9a3eSAnup Patel phys_addr_t sz, pgprot_t prot) 370671f9a3eSAnup Patel { 371671f9a3eSAnup Patel pte_t *ptep; 372671f9a3eSAnup Patel phys_addr_t pte_phys; 373974b9b2cSMike Rapoport uintptr_t pmd_idx = pmd_index(va); 374671f9a3eSAnup Patel 375671f9a3eSAnup Patel if (sz == PMD_SIZE) { 376974b9b2cSMike Rapoport if (pmd_none(pmdp[pmd_idx])) 377974b9b2cSMike Rapoport pmdp[pmd_idx] = pfn_pmd(PFN_DOWN(pa), prot); 378671f9a3eSAnup Patel return; 379671f9a3eSAnup Patel } 380671f9a3eSAnup Patel 381974b9b2cSMike Rapoport if (pmd_none(pmdp[pmd_idx])) { 382e8dcb61fSAtish Patra pte_phys = pt_ops.alloc_pte(va); 383974b9b2cSMike Rapoport pmdp[pmd_idx] = pfn_pmd(PFN_DOWN(pte_phys), PAGE_TABLE); 384e8dcb61fSAtish Patra ptep = pt_ops.get_pte_virt(pte_phys); 385671f9a3eSAnup Patel memset(ptep, 0, PAGE_SIZE); 386671f9a3eSAnup Patel } else { 387974b9b2cSMike Rapoport pte_phys = PFN_PHYS(_pmd_pfn(pmdp[pmd_idx])); 388e8dcb61fSAtish Patra ptep = pt_ops.get_pte_virt(pte_phys); 389671f9a3eSAnup Patel } 390671f9a3eSAnup Patel 391671f9a3eSAnup Patel create_pte_mapping(ptep, va, pa, sz, prot); 392671f9a3eSAnup Patel } 393671f9a3eSAnup Patel 394671f9a3eSAnup Patel #define pgd_next_t pmd_t 395e8dcb61fSAtish Patra #define alloc_pgd_next(__va) pt_ops.alloc_pmd(__va) 396e8dcb61fSAtish Patra #define get_pgd_next_virt(__pa) pt_ops.get_pmd_virt(__pa) 397671f9a3eSAnup Patel #define create_pgd_next_mapping(__nextp, __va, __pa, __sz, __prot) \ 398671f9a3eSAnup Patel create_pmd_mapping(__nextp, __va, __pa, __sz, __prot) 399671f9a3eSAnup Patel #define fixmap_pgd_next fixmap_pmd 400671f9a3eSAnup Patel #else 401671f9a3eSAnup Patel #define pgd_next_t pte_t 402e8dcb61fSAtish Patra #define alloc_pgd_next(__va) pt_ops.alloc_pte(__va) 403e8dcb61fSAtish Patra #define get_pgd_next_virt(__pa) pt_ops.get_pte_virt(__pa) 404671f9a3eSAnup Patel #define create_pgd_next_mapping(__nextp, __va, __pa, __sz, __prot) \ 405671f9a3eSAnup Patel create_pte_mapping(__nextp, __va, __pa, __sz, __prot) 406671f9a3eSAnup Patel #define fixmap_pgd_next fixmap_pte 407671f9a3eSAnup Patel #endif 408671f9a3eSAnup Patel 409b91540d5SAtish Patra void __init create_pgd_mapping(pgd_t *pgdp, 410671f9a3eSAnup Patel uintptr_t va, phys_addr_t pa, 411671f9a3eSAnup Patel phys_addr_t sz, pgprot_t prot) 412671f9a3eSAnup Patel { 413671f9a3eSAnup Patel pgd_next_t *nextp; 414671f9a3eSAnup Patel phys_addr_t next_phys; 415974b9b2cSMike Rapoport uintptr_t pgd_idx = pgd_index(va); 416671f9a3eSAnup Patel 417671f9a3eSAnup Patel if (sz == PGDIR_SIZE) { 418974b9b2cSMike Rapoport if (pgd_val(pgdp[pgd_idx]) == 0) 419974b9b2cSMike Rapoport pgdp[pgd_idx] = pfn_pgd(PFN_DOWN(pa), prot); 420671f9a3eSAnup Patel return; 421671f9a3eSAnup Patel } 422671f9a3eSAnup Patel 423974b9b2cSMike Rapoport if (pgd_val(pgdp[pgd_idx]) == 0) { 424671f9a3eSAnup Patel next_phys = alloc_pgd_next(va); 425974b9b2cSMike Rapoport pgdp[pgd_idx] = pfn_pgd(PFN_DOWN(next_phys), PAGE_TABLE); 426671f9a3eSAnup Patel nextp = get_pgd_next_virt(next_phys); 427671f9a3eSAnup Patel memset(nextp, 0, PAGE_SIZE); 428671f9a3eSAnup Patel } else { 429974b9b2cSMike Rapoport next_phys = PFN_PHYS(_pgd_pfn(pgdp[pgd_idx])); 430671f9a3eSAnup Patel nextp = get_pgd_next_virt(next_phys); 431671f9a3eSAnup Patel } 432671f9a3eSAnup Patel 433671f9a3eSAnup Patel create_pgd_next_mapping(nextp, va, pa, sz, prot); 434671f9a3eSAnup Patel } 435671f9a3eSAnup Patel 436671f9a3eSAnup Patel static uintptr_t __init best_map_size(phys_addr_t base, phys_addr_t size) 437671f9a3eSAnup Patel { 4380fdc636cSZong Li /* Upgrade to PMD_SIZE mappings whenever possible */ 4390fdc636cSZong Li if ((base & (PMD_SIZE - 1)) || (size & (PMD_SIZE - 1))) 4400fdc636cSZong Li return PAGE_SIZE; 441671f9a3eSAnup Patel 4420fdc636cSZong Li return PMD_SIZE; 443671f9a3eSAnup Patel } 444671f9a3eSAnup Patel 44544c92257SVitaly Wool #ifdef CONFIG_XIP_KERNEL 44644c92257SVitaly Wool /* called from head.S with MMU off */ 44744c92257SVitaly Wool asmlinkage void __init __copy_data(void) 44844c92257SVitaly Wool { 44944c92257SVitaly Wool void *from = (void *)(&_sdata); 45044c92257SVitaly Wool void *end = (void *)(&_end); 45144c92257SVitaly Wool void *to = (void *)CONFIG_PHYS_RAM_BASE; 45244c92257SVitaly Wool size_t sz = (size_t)(end - from + 1); 45344c92257SVitaly Wool 45444c92257SVitaly Wool memcpy(to, from, sz); 45544c92257SVitaly Wool } 45644c92257SVitaly Wool #endif 45744c92257SVitaly Wool 458387181dcSAnup Patel /* 459387181dcSAnup Patel * setup_vm() is called from head.S with MMU-off. 460387181dcSAnup Patel * 461387181dcSAnup Patel * Following requirements should be honoured for setup_vm() to work 462387181dcSAnup Patel * correctly: 463387181dcSAnup Patel * 1) It should use PC-relative addressing for accessing kernel symbols. 464387181dcSAnup Patel * To achieve this we always use GCC cmodel=medany. 465387181dcSAnup Patel * 2) The compiler instrumentation for FTRACE will not work for setup_vm() 466387181dcSAnup Patel * so disable compiler instrumentation when FTRACE is enabled. 467387181dcSAnup Patel * 468387181dcSAnup Patel * Currently, the above requirements are honoured by using custom CFLAGS 469387181dcSAnup Patel * for init.o in mm/Makefile. 470387181dcSAnup Patel */ 471387181dcSAnup Patel 472387181dcSAnup Patel #ifndef __riscv_cmodel_medany 4736a527b67SPaul Walmsley #error "setup_vm() is called from head.S before relocate so it should not use absolute addressing." 474387181dcSAnup Patel #endif 475387181dcSAnup Patel 47601062356SJisheng Zhang static uintptr_t load_pa __initdata; 47701062356SJisheng Zhang static uintptr_t load_sz __initdata; 47844c92257SVitaly Wool #ifdef CONFIG_XIP_KERNEL 47944c92257SVitaly Wool #define load_pa (*((uintptr_t *)XIP_FIXUP(&load_pa))) 48044c92257SVitaly Wool #define load_sz (*((uintptr_t *)XIP_FIXUP(&load_sz))) 48144c92257SVitaly Wool #endif 4822bfc6cd8SAlexandre Ghiti 48344c92257SVitaly Wool #ifdef CONFIG_XIP_KERNEL 484ae3d69bcSVitaly Wool static uintptr_t xiprom __initdata; 48501062356SJisheng Zhang static uintptr_t xiprom_sz __initdata; 48644c92257SVitaly Wool #define xiprom_sz (*((uintptr_t *)XIP_FIXUP(&xiprom_sz))) 48744c92257SVitaly Wool #define xiprom (*((uintptr_t *)XIP_FIXUP(&xiprom))) 48844c92257SVitaly Wool 48944c92257SVitaly Wool static void __init create_kernel_page_table(pgd_t *pgdir, uintptr_t map_size) 49044c92257SVitaly Wool { 49144c92257SVitaly Wool uintptr_t va, end_va; 49244c92257SVitaly Wool 49344c92257SVitaly Wool /* Map the flash resident part */ 49444c92257SVitaly Wool end_va = kernel_virt_addr + xiprom_sz; 49544c92257SVitaly Wool for (va = kernel_virt_addr; va < end_va; va += map_size) 49644c92257SVitaly Wool create_pgd_mapping(pgdir, va, 49744c92257SVitaly Wool xiprom + (va - kernel_virt_addr), 49844c92257SVitaly Wool map_size, PAGE_KERNEL_EXEC); 49944c92257SVitaly Wool 50044c92257SVitaly Wool /* Map the data in RAM */ 50144c92257SVitaly Wool end_va = kernel_virt_addr + XIP_OFFSET + load_sz; 50244c92257SVitaly Wool for (va = kernel_virt_addr + XIP_OFFSET; va < end_va; va += map_size) 50344c92257SVitaly Wool create_pgd_mapping(pgdir, va, 50444c92257SVitaly Wool load_pa + (va - (kernel_virt_addr + XIP_OFFSET)), 50544c92257SVitaly Wool map_size, PAGE_KERNEL); 50644c92257SVitaly Wool } 50744c92257SVitaly Wool #else 5082bfc6cd8SAlexandre Ghiti static void __init create_kernel_page_table(pgd_t *pgdir, uintptr_t map_size) 5092bfc6cd8SAlexandre Ghiti { 5102bfc6cd8SAlexandre Ghiti uintptr_t va, end_va; 5112bfc6cd8SAlexandre Ghiti 5122bfc6cd8SAlexandre Ghiti end_va = kernel_virt_addr + load_sz; 5132bfc6cd8SAlexandre Ghiti for (va = kernel_virt_addr; va < end_va; va += map_size) 5142bfc6cd8SAlexandre Ghiti create_pgd_mapping(pgdir, va, 5152bfc6cd8SAlexandre Ghiti load_pa + (va - kernel_virt_addr), 5162bfc6cd8SAlexandre Ghiti map_size, PAGE_KERNEL_EXEC); 5172bfc6cd8SAlexandre Ghiti } 51844c92257SVitaly Wool #endif 5192bfc6cd8SAlexandre Ghiti 520671f9a3eSAnup Patel asmlinkage void __init setup_vm(uintptr_t dtb_pa) 5216f1e9e94SAnup Patel { 52244c92257SVitaly Wool uintptr_t __maybe_unused pa; 5230f02de44SAlexandre Ghiti uintptr_t map_size; 5246262f661SAtish Patra #ifndef __PAGETABLE_PMD_FOLDED 5256262f661SAtish Patra pmd_t fix_bmap_spmd, fix_bmap_epmd; 5266262f661SAtish Patra #endif 5276f1e9e94SAnup Patel 52844c92257SVitaly Wool #ifdef CONFIG_XIP_KERNEL 52944c92257SVitaly Wool xiprom = (uintptr_t)CONFIG_XIP_PHYS_ADDR; 53044c92257SVitaly Wool xiprom_sz = (uintptr_t)(&_exiprom) - (uintptr_t)(&_xiprom); 53144c92257SVitaly Wool 53244c92257SVitaly Wool load_pa = (uintptr_t)CONFIG_PHYS_RAM_BASE; 53344c92257SVitaly Wool load_sz = (uintptr_t)(&_end) - (uintptr_t)(&_sdata); 53444c92257SVitaly Wool 53544c92257SVitaly Wool va_kernel_xip_pa_offset = kernel_virt_addr - xiprom; 53644c92257SVitaly Wool #else 5372bfc6cd8SAlexandre Ghiti load_pa = (uintptr_t)(&_start); 5382bfc6cd8SAlexandre Ghiti load_sz = (uintptr_t)(&_end) - load_pa; 53944c92257SVitaly Wool #endif 5406f1e9e94SAnup Patel 541671f9a3eSAnup Patel va_pa_offset = PAGE_OFFSET - load_pa; 5422bfc6cd8SAlexandre Ghiti #ifdef CONFIG_64BIT 5432bfc6cd8SAlexandre Ghiti va_kernel_pa_offset = kernel_virt_addr - load_pa; 5442bfc6cd8SAlexandre Ghiti #endif 5452bfc6cd8SAlexandre Ghiti 546671f9a3eSAnup Patel pfn_base = PFN_DOWN(load_pa); 547671f9a3eSAnup Patel 548671f9a3eSAnup Patel /* 549671f9a3eSAnup Patel * Enforce boot alignment requirements of RV32 and 550671f9a3eSAnup Patel * RV64 by only allowing PMD or PGD mappings. 551671f9a3eSAnup Patel */ 5520f02de44SAlexandre Ghiti map_size = PMD_SIZE; 5536f1e9e94SAnup Patel 5546f1e9e94SAnup Patel /* Sanity check alignment and size */ 5556f1e9e94SAnup Patel BUG_ON((PAGE_OFFSET % PGDIR_SIZE) != 0); 556671f9a3eSAnup Patel BUG_ON((load_pa % map_size) != 0); 557671f9a3eSAnup Patel 558e8dcb61fSAtish Patra pt_ops.alloc_pte = alloc_pte_early; 559e8dcb61fSAtish Patra pt_ops.get_pte_virt = get_pte_virt_early; 560e8dcb61fSAtish Patra #ifndef __PAGETABLE_PMD_FOLDED 561e8dcb61fSAtish Patra pt_ops.alloc_pmd = alloc_pmd_early; 562e8dcb61fSAtish Patra pt_ops.get_pmd_virt = get_pmd_virt_early; 563e8dcb61fSAtish Patra #endif 564671f9a3eSAnup Patel /* Setup early PGD for fixmap */ 565671f9a3eSAnup Patel create_pgd_mapping(early_pg_dir, FIXADDR_START, 566671f9a3eSAnup Patel (uintptr_t)fixmap_pgd_next, PGDIR_SIZE, PAGE_TABLE); 5676f1e9e94SAnup Patel 5686f1e9e94SAnup Patel #ifndef __PAGETABLE_PMD_FOLDED 569671f9a3eSAnup Patel /* Setup fixmap PMD */ 570671f9a3eSAnup Patel create_pmd_mapping(fixmap_pmd, FIXADDR_START, 571671f9a3eSAnup Patel (uintptr_t)fixmap_pte, PMD_SIZE, PAGE_TABLE); 572671f9a3eSAnup Patel /* Setup trampoline PGD and PMD */ 5732bfc6cd8SAlexandre Ghiti create_pgd_mapping(trampoline_pg_dir, kernel_virt_addr, 574671f9a3eSAnup Patel (uintptr_t)trampoline_pmd, PGDIR_SIZE, PAGE_TABLE); 57544c92257SVitaly Wool #ifdef CONFIG_XIP_KERNEL 57644c92257SVitaly Wool create_pmd_mapping(trampoline_pmd, kernel_virt_addr, 57744c92257SVitaly Wool xiprom, PMD_SIZE, PAGE_KERNEL_EXEC); 57844c92257SVitaly Wool #else 5792bfc6cd8SAlexandre Ghiti create_pmd_mapping(trampoline_pmd, kernel_virt_addr, 580671f9a3eSAnup Patel load_pa, PMD_SIZE, PAGE_KERNEL_EXEC); 58144c92257SVitaly Wool #endif 5826f1e9e94SAnup Patel #else 583671f9a3eSAnup Patel /* Setup trampoline PGD */ 5842bfc6cd8SAlexandre Ghiti create_pgd_mapping(trampoline_pg_dir, kernel_virt_addr, 585671f9a3eSAnup Patel load_pa, PGDIR_SIZE, PAGE_KERNEL_EXEC); 586671f9a3eSAnup Patel #endif 5876f1e9e94SAnup Patel 588671f9a3eSAnup Patel /* 5892bfc6cd8SAlexandre Ghiti * Setup early PGD covering entire kernel which will allow 590671f9a3eSAnup Patel * us to reach paging_init(). We map all memory banks later 591671f9a3eSAnup Patel * in setup_vm_final() below. 592671f9a3eSAnup Patel */ 5932bfc6cd8SAlexandre Ghiti create_kernel_page_table(early_pg_dir, map_size); 594f2c17aabSAnup Patel 5951074dd44SAnup Patel #ifndef __PAGETABLE_PMD_FOLDED 5961074dd44SAnup Patel /* Setup early PMD for DTB */ 5971074dd44SAnup Patel create_pgd_mapping(early_pg_dir, DTB_EARLY_BASE_VA, 5981074dd44SAnup Patel (uintptr_t)early_dtb_pmd, PGDIR_SIZE, PAGE_TABLE); 599f105aa94SVitaly Wool #ifndef CONFIG_BUILTIN_DTB 6001074dd44SAnup Patel /* Create two consecutive PMD mappings for FDT early scan */ 6011074dd44SAnup Patel pa = dtb_pa & ~(PMD_SIZE - 1); 6021074dd44SAnup Patel create_pmd_mapping(early_dtb_pmd, DTB_EARLY_BASE_VA, 6031074dd44SAnup Patel pa, PMD_SIZE, PAGE_KERNEL); 6041074dd44SAnup Patel create_pmd_mapping(early_dtb_pmd, DTB_EARLY_BASE_VA + PMD_SIZE, 6051074dd44SAnup Patel pa + PMD_SIZE, PMD_SIZE, PAGE_KERNEL); 6061074dd44SAnup Patel dtb_early_va = (void *)DTB_EARLY_BASE_VA + (dtb_pa & (PMD_SIZE - 1)); 607f105aa94SVitaly Wool #else /* CONFIG_BUILTIN_DTB */ 6082bfc6cd8SAlexandre Ghiti #ifdef CONFIG_64BIT 6092bfc6cd8SAlexandre Ghiti /* 6102bfc6cd8SAlexandre Ghiti * __va can't be used since it would return a linear mapping address 6112bfc6cd8SAlexandre Ghiti * whereas dtb_early_va will be used before setup_vm_final installs 6122bfc6cd8SAlexandre Ghiti * the linear mapping. 6132bfc6cd8SAlexandre Ghiti */ 61444c92257SVitaly Wool dtb_early_va = kernel_mapping_pa_to_va(XIP_FIXUP(dtb_pa)); 6152bfc6cd8SAlexandre Ghiti #else 616f105aa94SVitaly Wool dtb_early_va = __va(dtb_pa); 6172bfc6cd8SAlexandre Ghiti #endif /* CONFIG_64BIT */ 618f105aa94SVitaly Wool #endif /* CONFIG_BUILTIN_DTB */ 6191074dd44SAnup Patel #else 620f105aa94SVitaly Wool #ifndef CONFIG_BUILTIN_DTB 6218f3a2b4aSAnup Patel /* Create two consecutive PGD mappings for FDT early scan */ 6228f3a2b4aSAnup Patel pa = dtb_pa & ~(PGDIR_SIZE - 1); 6238f3a2b4aSAnup Patel create_pgd_mapping(early_pg_dir, DTB_EARLY_BASE_VA, 6248f3a2b4aSAnup Patel pa, PGDIR_SIZE, PAGE_KERNEL); 6258f3a2b4aSAnup Patel create_pgd_mapping(early_pg_dir, DTB_EARLY_BASE_VA + PGDIR_SIZE, 6268f3a2b4aSAnup Patel pa + PGDIR_SIZE, PGDIR_SIZE, PAGE_KERNEL); 6278f3a2b4aSAnup Patel dtb_early_va = (void *)DTB_EARLY_BASE_VA + (dtb_pa & (PGDIR_SIZE - 1)); 628f105aa94SVitaly Wool #else /* CONFIG_BUILTIN_DTB */ 6292bfc6cd8SAlexandre Ghiti #ifdef CONFIG_64BIT 63044c92257SVitaly Wool dtb_early_va = kernel_mapping_pa_to_va(XIP_FIXUP(dtb_pa)); 6312bfc6cd8SAlexandre Ghiti #else 632f105aa94SVitaly Wool dtb_early_va = __va(dtb_pa); 6332bfc6cd8SAlexandre Ghiti #endif /* CONFIG_64BIT */ 634f105aa94SVitaly Wool #endif /* CONFIG_BUILTIN_DTB */ 6351074dd44SAnup Patel #endif 636922b0375SAlbert Ou dtb_early_pa = dtb_pa; 6376262f661SAtish Patra 6386262f661SAtish Patra /* 6396262f661SAtish Patra * Bootime fixmap only can handle PMD_SIZE mapping. Thus, boot-ioremap 6406262f661SAtish Patra * range can not span multiple pmds. 6416262f661SAtish Patra */ 6426262f661SAtish Patra BUILD_BUG_ON((__fix_to_virt(FIX_BTMAP_BEGIN) >> PMD_SHIFT) 6436262f661SAtish Patra != (__fix_to_virt(FIX_BTMAP_END) >> PMD_SHIFT)); 6446262f661SAtish Patra 6456262f661SAtish Patra #ifndef __PAGETABLE_PMD_FOLDED 6466262f661SAtish Patra /* 6476262f661SAtish Patra * Early ioremap fixmap is already created as it lies within first 2MB 6486262f661SAtish Patra * of fixmap region. We always map PMD_SIZE. Thus, both FIX_BTMAP_END 6496262f661SAtish Patra * FIX_BTMAP_BEGIN should lie in the same pmd. Verify that and warn 6506262f661SAtish Patra * the user if not. 6516262f661SAtish Patra */ 6526262f661SAtish Patra fix_bmap_spmd = fixmap_pmd[pmd_index(__fix_to_virt(FIX_BTMAP_BEGIN))]; 6536262f661SAtish Patra fix_bmap_epmd = fixmap_pmd[pmd_index(__fix_to_virt(FIX_BTMAP_END))]; 6546262f661SAtish Patra if (pmd_val(fix_bmap_spmd) != pmd_val(fix_bmap_epmd)) { 6556262f661SAtish Patra WARN_ON(1); 6566262f661SAtish Patra pr_warn("fixmap btmap start [%08lx] != end [%08lx]\n", 6576262f661SAtish Patra pmd_val(fix_bmap_spmd), pmd_val(fix_bmap_epmd)); 6586262f661SAtish Patra pr_warn("fix_to_virt(FIX_BTMAP_BEGIN): %08lx\n", 6596262f661SAtish Patra fix_to_virt(FIX_BTMAP_BEGIN)); 6606262f661SAtish Patra pr_warn("fix_to_virt(FIX_BTMAP_END): %08lx\n", 6616262f661SAtish Patra fix_to_virt(FIX_BTMAP_END)); 6626262f661SAtish Patra 6636262f661SAtish Patra pr_warn("FIX_BTMAP_END: %d\n", FIX_BTMAP_END); 6646262f661SAtish Patra pr_warn("FIX_BTMAP_BEGIN: %d\n", FIX_BTMAP_BEGIN); 6656262f661SAtish Patra } 6666262f661SAtish Patra #endif 6676f1e9e94SAnup Patel } 668f2c17aabSAnup Patel 6698d91b097SGeert Uytterhoeven #if defined(CONFIG_64BIT) && defined(CONFIG_STRICT_KERNEL_RWX) 67001062356SJisheng Zhang void __init protect_kernel_linear_mapping_text_rodata(void) 6712bfc6cd8SAlexandre Ghiti { 6722bfc6cd8SAlexandre Ghiti unsigned long text_start = (unsigned long)lm_alias(_start); 6732bfc6cd8SAlexandre Ghiti unsigned long init_text_start = (unsigned long)lm_alias(__init_text_begin); 6742bfc6cd8SAlexandre Ghiti unsigned long rodata_start = (unsigned long)lm_alias(__start_rodata); 6752bfc6cd8SAlexandre Ghiti unsigned long data_start = (unsigned long)lm_alias(_data); 6762bfc6cd8SAlexandre Ghiti 6772bfc6cd8SAlexandre Ghiti set_memory_ro(text_start, (init_text_start - text_start) >> PAGE_SHIFT); 6782bfc6cd8SAlexandre Ghiti set_memory_nx(text_start, (init_text_start - text_start) >> PAGE_SHIFT); 6792bfc6cd8SAlexandre Ghiti 6802bfc6cd8SAlexandre Ghiti set_memory_ro(rodata_start, (data_start - rodata_start) >> PAGE_SHIFT); 6812bfc6cd8SAlexandre Ghiti set_memory_nx(rodata_start, (data_start - rodata_start) >> PAGE_SHIFT); 6822bfc6cd8SAlexandre Ghiti } 6832bfc6cd8SAlexandre Ghiti #endif 6842bfc6cd8SAlexandre Ghiti 685671f9a3eSAnup Patel static void __init setup_vm_final(void) 686671f9a3eSAnup Patel { 687671f9a3eSAnup Patel uintptr_t va, map_size; 688671f9a3eSAnup Patel phys_addr_t pa, start, end; 689b10d6bcaSMike Rapoport u64 i; 690671f9a3eSAnup Patel 691e8dcb61fSAtish Patra /** 692e8dcb61fSAtish Patra * MMU is enabled at this point. But page table setup is not complete yet. 693e8dcb61fSAtish Patra * fixmap page table alloc functions should be used at this point 694e8dcb61fSAtish Patra */ 695e8dcb61fSAtish Patra pt_ops.alloc_pte = alloc_pte_fixmap; 696e8dcb61fSAtish Patra pt_ops.get_pte_virt = get_pte_virt_fixmap; 697e8dcb61fSAtish Patra #ifndef __PAGETABLE_PMD_FOLDED 698e8dcb61fSAtish Patra pt_ops.alloc_pmd = alloc_pmd_fixmap; 699e8dcb61fSAtish Patra pt_ops.get_pmd_virt = get_pmd_virt_fixmap; 700e8dcb61fSAtish Patra #endif 701671f9a3eSAnup Patel /* Setup swapper PGD for fixmap */ 702671f9a3eSAnup Patel create_pgd_mapping(swapper_pg_dir, FIXADDR_START, 703ac51e005SZong Li __pa_symbol(fixmap_pgd_next), 704671f9a3eSAnup Patel PGDIR_SIZE, PAGE_TABLE); 705671f9a3eSAnup Patel 7062bfc6cd8SAlexandre Ghiti /* Map all memory banks in the linear mapping */ 707b10d6bcaSMike Rapoport for_each_mem_range(i, &start, &end) { 708671f9a3eSAnup Patel if (start >= end) 709671f9a3eSAnup Patel break; 710671f9a3eSAnup Patel if (start <= __pa(PAGE_OFFSET) && 711671f9a3eSAnup Patel __pa(PAGE_OFFSET) < end) 712671f9a3eSAnup Patel start = __pa(PAGE_OFFSET); 713671f9a3eSAnup Patel 714671f9a3eSAnup Patel map_size = best_map_size(start, end - start); 715671f9a3eSAnup Patel for (pa = start; pa < end; pa += map_size) { 716671f9a3eSAnup Patel va = (uintptr_t)__va(pa); 717671f9a3eSAnup Patel create_pgd_mapping(swapper_pg_dir, va, pa, 7182bfc6cd8SAlexandre Ghiti map_size, 7192bfc6cd8SAlexandre Ghiti #ifdef CONFIG_64BIT 7202bfc6cd8SAlexandre Ghiti PAGE_KERNEL 7212bfc6cd8SAlexandre Ghiti #else 7222bfc6cd8SAlexandre Ghiti PAGE_KERNEL_EXEC 7232bfc6cd8SAlexandre Ghiti #endif 7242bfc6cd8SAlexandre Ghiti ); 7252bfc6cd8SAlexandre Ghiti 726671f9a3eSAnup Patel } 727671f9a3eSAnup Patel } 728671f9a3eSAnup Patel 7292bfc6cd8SAlexandre Ghiti #ifdef CONFIG_64BIT 7302bfc6cd8SAlexandre Ghiti /* Map the kernel */ 7312bfc6cd8SAlexandre Ghiti create_kernel_page_table(swapper_pg_dir, PMD_SIZE); 7322bfc6cd8SAlexandre Ghiti #endif 7332bfc6cd8SAlexandre Ghiti 734671f9a3eSAnup Patel /* Clear fixmap PTE and PMD mappings */ 735671f9a3eSAnup Patel clear_fixmap(FIX_PTE); 736671f9a3eSAnup Patel clear_fixmap(FIX_PMD); 737671f9a3eSAnup Patel 738671f9a3eSAnup Patel /* Move to swapper page table */ 739ac51e005SZong Li csr_write(CSR_SATP, PFN_DOWN(__pa_symbol(swapper_pg_dir)) | SATP_MODE); 740671f9a3eSAnup Patel local_flush_tlb_all(); 741e8dcb61fSAtish Patra 742e8dcb61fSAtish Patra /* generic page allocation functions must be used to setup page table */ 743e8dcb61fSAtish Patra pt_ops.alloc_pte = alloc_pte_late; 744e8dcb61fSAtish Patra pt_ops.get_pte_virt = get_pte_virt_late; 745e8dcb61fSAtish Patra #ifndef __PAGETABLE_PMD_FOLDED 746e8dcb61fSAtish Patra pt_ops.alloc_pmd = alloc_pmd_late; 747e8dcb61fSAtish Patra pt_ops.get_pmd_virt = get_pmd_virt_late; 748e8dcb61fSAtish Patra #endif 749671f9a3eSAnup Patel } 7506bd33e1eSChristoph Hellwig #else 7516bd33e1eSChristoph Hellwig asmlinkage void __init setup_vm(uintptr_t dtb_pa) 7526bd33e1eSChristoph Hellwig { 7536bd33e1eSChristoph Hellwig dtb_early_va = (void *)dtb_pa; 754a78c6f59SAtish Patra dtb_early_pa = dtb_pa; 7556bd33e1eSChristoph Hellwig } 7566bd33e1eSChristoph Hellwig 7576bd33e1eSChristoph Hellwig static inline void setup_vm_final(void) 7586bd33e1eSChristoph Hellwig { 7596bd33e1eSChristoph Hellwig } 7606bd33e1eSChristoph Hellwig #endif /* CONFIG_MMU */ 761671f9a3eSAnup Patel 762d27c3c90SZong Li #ifdef CONFIG_STRICT_KERNEL_RWX 7631987501bSJisheng Zhang void __init protect_kernel_text_data(void) 764d27c3c90SZong Li { 76519a00869SAtish Patra unsigned long text_start = (unsigned long)_start; 76619a00869SAtish Patra unsigned long init_text_start = (unsigned long)__init_text_begin; 76719a00869SAtish Patra unsigned long init_data_start = (unsigned long)__init_data_begin; 768d27c3c90SZong Li unsigned long rodata_start = (unsigned long)__start_rodata; 769d27c3c90SZong Li unsigned long data_start = (unsigned long)_data; 770d27c3c90SZong Li unsigned long max_low = (unsigned long)(__va(PFN_PHYS(max_low_pfn))); 771d27c3c90SZong Li 77219a00869SAtish Patra set_memory_ro(text_start, (init_text_start - text_start) >> PAGE_SHIFT); 77319a00869SAtish Patra set_memory_ro(init_text_start, (init_data_start - init_text_start) >> PAGE_SHIFT); 77419a00869SAtish Patra set_memory_nx(init_data_start, (rodata_start - init_data_start) >> PAGE_SHIFT); 77519a00869SAtish Patra /* rodata section is marked readonly in mark_rodata_ro */ 776d27c3c90SZong Li set_memory_nx(rodata_start, (data_start - rodata_start) >> PAGE_SHIFT); 777d27c3c90SZong Li set_memory_nx(data_start, (max_low - data_start) >> PAGE_SHIFT); 77819a00869SAtish Patra } 77919a00869SAtish Patra 78019a00869SAtish Patra void mark_rodata_ro(void) 78119a00869SAtish Patra { 78219a00869SAtish Patra unsigned long rodata_start = (unsigned long)__start_rodata; 78319a00869SAtish Patra unsigned long data_start = (unsigned long)_data; 78419a00869SAtish Patra 78519a00869SAtish Patra set_memory_ro(rodata_start, (data_start - rodata_start) >> PAGE_SHIFT); 786b422d28bSZong Li 787b422d28bSZong Li debug_checkwx(); 788d27c3c90SZong Li } 789d27c3c90SZong Li #endif 790d27c3c90SZong Li 791e53d2818SNick Kossifidis #ifdef CONFIG_KEXEC_CORE 792e53d2818SNick Kossifidis /* 793e53d2818SNick Kossifidis * reserve_crashkernel() - reserves memory for crash kernel 794e53d2818SNick Kossifidis * 795e53d2818SNick Kossifidis * This function reserves memory area given in "crashkernel=" kernel command 796e53d2818SNick Kossifidis * line parameter. The memory reserved is used by dump capture kernel when 797e53d2818SNick Kossifidis * primary kernel is crashing. 798e53d2818SNick Kossifidis */ 799e53d2818SNick Kossifidis static void __init reserve_crashkernel(void) 800e53d2818SNick Kossifidis { 801e53d2818SNick Kossifidis unsigned long long crash_base = 0; 802e53d2818SNick Kossifidis unsigned long long crash_size = 0; 803e53d2818SNick Kossifidis unsigned long search_start = memblock_start_of_DRAM(); 804e53d2818SNick Kossifidis unsigned long search_end = memblock_end_of_DRAM(); 805e53d2818SNick Kossifidis 806e53d2818SNick Kossifidis int ret = 0; 807e53d2818SNick Kossifidis 80856409750SNick Kossifidis /* 80956409750SNick Kossifidis * Don't reserve a region for a crash kernel on a crash kernel 81056409750SNick Kossifidis * since it doesn't make much sense and we have limited memory 81156409750SNick Kossifidis * resources. 81256409750SNick Kossifidis */ 81356409750SNick Kossifidis #ifdef CONFIG_CRASH_DUMP 81456409750SNick Kossifidis if (is_kdump_kernel()) { 81556409750SNick Kossifidis pr_info("crashkernel: ignoring reservation request\n"); 81656409750SNick Kossifidis return; 81756409750SNick Kossifidis } 81856409750SNick Kossifidis #endif 81956409750SNick Kossifidis 820e53d2818SNick Kossifidis ret = parse_crashkernel(boot_command_line, memblock_phys_mem_size(), 821e53d2818SNick Kossifidis &crash_size, &crash_base); 822e53d2818SNick Kossifidis if (ret || !crash_size) 823e53d2818SNick Kossifidis return; 824e53d2818SNick Kossifidis 825e53d2818SNick Kossifidis crash_size = PAGE_ALIGN(crash_size); 826e53d2818SNick Kossifidis 827e53d2818SNick Kossifidis if (crash_base == 0) { 828e53d2818SNick Kossifidis /* 829e53d2818SNick Kossifidis * Current riscv boot protocol requires 2MB alignment for 830e53d2818SNick Kossifidis * RV64 and 4MB alignment for RV32 (hugepage size) 831e53d2818SNick Kossifidis */ 832e53d2818SNick Kossifidis crash_base = memblock_find_in_range(search_start, search_end, 833e53d2818SNick Kossifidis crash_size, PMD_SIZE); 834e53d2818SNick Kossifidis 835e53d2818SNick Kossifidis if (crash_base == 0) { 836e53d2818SNick Kossifidis pr_warn("crashkernel: couldn't allocate %lldKB\n", 837e53d2818SNick Kossifidis crash_size >> 10); 838e53d2818SNick Kossifidis return; 839e53d2818SNick Kossifidis } 840e53d2818SNick Kossifidis } else { 841e53d2818SNick Kossifidis /* User specifies base address explicitly. */ 842e53d2818SNick Kossifidis if (!memblock_is_region_memory(crash_base, crash_size)) { 843e53d2818SNick Kossifidis pr_warn("crashkernel: requested region is not memory\n"); 844e53d2818SNick Kossifidis return; 845e53d2818SNick Kossifidis } 846e53d2818SNick Kossifidis 847e53d2818SNick Kossifidis if (memblock_is_region_reserved(crash_base, crash_size)) { 848e53d2818SNick Kossifidis pr_warn("crashkernel: requested region is reserved\n"); 849e53d2818SNick Kossifidis return; 850e53d2818SNick Kossifidis } 851e53d2818SNick Kossifidis 852e53d2818SNick Kossifidis 853e53d2818SNick Kossifidis if (!IS_ALIGNED(crash_base, PMD_SIZE)) { 854e53d2818SNick Kossifidis pr_warn("crashkernel: requested region is misaligned\n"); 855e53d2818SNick Kossifidis return; 856e53d2818SNick Kossifidis } 857e53d2818SNick Kossifidis } 858e53d2818SNick Kossifidis memblock_reserve(crash_base, crash_size); 859e53d2818SNick Kossifidis 860e53d2818SNick Kossifidis pr_info("crashkernel: reserved 0x%016llx - 0x%016llx (%lld MB)\n", 861e53d2818SNick Kossifidis crash_base, crash_base + crash_size, crash_size >> 20); 862e53d2818SNick Kossifidis 863e53d2818SNick Kossifidis crashk_res.start = crash_base; 864e53d2818SNick Kossifidis crashk_res.end = crash_base + crash_size - 1; 865e53d2818SNick Kossifidis } 866e53d2818SNick Kossifidis #endif /* CONFIG_KEXEC_CORE */ 867e53d2818SNick Kossifidis 86856409750SNick Kossifidis #ifdef CONFIG_CRASH_DUMP 86956409750SNick Kossifidis /* 87056409750SNick Kossifidis * We keep track of the ELF core header of the crashed 87156409750SNick Kossifidis * kernel with a reserved-memory region with compatible 87256409750SNick Kossifidis * string "linux,elfcorehdr". Here we register a callback 87356409750SNick Kossifidis * to populate elfcorehdr_addr/size when this region is 87456409750SNick Kossifidis * present. Note that this region will be marked as 87556409750SNick Kossifidis * reserved once we call early_init_fdt_scan_reserved_mem() 87656409750SNick Kossifidis * later on. 87756409750SNick Kossifidis */ 87801062356SJisheng Zhang static int __init elfcore_hdr_setup(struct reserved_mem *rmem) 87956409750SNick Kossifidis { 88056409750SNick Kossifidis elfcorehdr_addr = rmem->base; 88156409750SNick Kossifidis elfcorehdr_size = rmem->size; 88256409750SNick Kossifidis return 0; 88356409750SNick Kossifidis } 88456409750SNick Kossifidis 88556409750SNick Kossifidis RESERVEDMEM_OF_DECLARE(elfcorehdr, "linux,elfcorehdr", elfcore_hdr_setup); 88656409750SNick Kossifidis #endif 88756409750SNick Kossifidis 888671f9a3eSAnup Patel void __init paging_init(void) 889671f9a3eSAnup Patel { 890f842f5ffSKefeng Wang setup_bootmem(); 891671f9a3eSAnup Patel setup_vm_final(); 892cbd34f4bSAtish Patra } 893cbd34f4bSAtish Patra 894cbd34f4bSAtish Patra void __init misc_mem_init(void) 895cbd34f4bSAtish Patra { 896f6e5aedfSKefeng Wang early_memtest(min_low_pfn << PAGE_SHIFT, max_low_pfn << PAGE_SHIFT); 8974f0e8eefSAtish Patra arch_numa_init(); 898cbd34f4bSAtish Patra sparse_init(); 899671f9a3eSAnup Patel zone_sizes_init(); 900e53d2818SNick Kossifidis #ifdef CONFIG_KEXEC_CORE 901e53d2818SNick Kossifidis reserve_crashkernel(); 902e53d2818SNick Kossifidis #endif 9034f0e8eefSAtish Patra memblock_dump_all(); 9046f1e9e94SAnup Patel } 905d95f1a54SLogan Gunthorpe 9069fe57d8cSKefeng Wang #ifdef CONFIG_SPARSEMEM_VMEMMAP 907d95f1a54SLogan Gunthorpe int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node, 908d95f1a54SLogan Gunthorpe struct vmem_altmap *altmap) 909d95f1a54SLogan Gunthorpe { 9101d9cfee7SAnshuman Khandual return vmemmap_populate_basepages(start, end, node, NULL); 911d95f1a54SLogan Gunthorpe } 912d95f1a54SLogan Gunthorpe #endif 913