1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2012 Regents of the University of California 4 * Copyright (C) 2019 Western Digital Corporation or its affiliates. 5 */ 6 7 #include <linux/init.h> 8 #include <linux/mm.h> 9 #include <linux/memblock.h> 10 #include <linux/initrd.h> 11 #include <linux/swap.h> 12 #include <linux/sizes.h> 13 #include <linux/of_fdt.h> 14 15 #include <asm/fixmap.h> 16 #include <asm/tlbflush.h> 17 #include <asm/sections.h> 18 #include <asm/pgtable.h> 19 #include <asm/io.h> 20 21 unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)] 22 __page_aligned_bss; 23 EXPORT_SYMBOL(empty_zero_page); 24 25 extern char _start[]; 26 27 static void __init zone_sizes_init(void) 28 { 29 unsigned long max_zone_pfns[MAX_NR_ZONES] = { 0, }; 30 31 #ifdef CONFIG_ZONE_DMA32 32 max_zone_pfns[ZONE_DMA32] = PFN_DOWN(min(4UL * SZ_1G, 33 (unsigned long) PFN_PHYS(max_low_pfn))); 34 #endif 35 max_zone_pfns[ZONE_NORMAL] = max_low_pfn; 36 37 free_area_init_nodes(max_zone_pfns); 38 } 39 40 void setup_zero_page(void) 41 { 42 memset((void *)empty_zero_page, 0, PAGE_SIZE); 43 } 44 45 void __init mem_init(void) 46 { 47 #ifdef CONFIG_FLATMEM 48 BUG_ON(!mem_map); 49 #endif /* CONFIG_FLATMEM */ 50 51 high_memory = (void *)(__va(PFN_PHYS(max_low_pfn))); 52 memblock_free_all(); 53 54 mem_init_print_info(NULL); 55 } 56 57 #ifdef CONFIG_BLK_DEV_INITRD 58 static void __init setup_initrd(void) 59 { 60 unsigned long size; 61 62 if (initrd_start >= initrd_end) { 63 pr_info("initrd not found or empty"); 64 goto disable; 65 } 66 if (__pa(initrd_end) > PFN_PHYS(max_low_pfn)) { 67 pr_err("initrd extends beyond end of memory"); 68 goto disable; 69 } 70 71 size = initrd_end - initrd_start; 72 memblock_reserve(__pa(initrd_start), size); 73 initrd_below_start_ok = 1; 74 75 pr_info("Initial ramdisk at: 0x%p (%lu bytes)\n", 76 (void *)(initrd_start), size); 77 return; 78 disable: 79 pr_cont(" - disabling initrd\n"); 80 initrd_start = 0; 81 initrd_end = 0; 82 } 83 #endif /* CONFIG_BLK_DEV_INITRD */ 84 85 void __init setup_bootmem(void) 86 { 87 struct memblock_region *reg; 88 phys_addr_t mem_size = 0; 89 phys_addr_t vmlinux_end = __pa(&_end); 90 phys_addr_t vmlinux_start = __pa(&_start); 91 92 /* Find the memory region containing the kernel */ 93 for_each_memblock(memory, reg) { 94 phys_addr_t end = reg->base + reg->size; 95 96 if (reg->base <= vmlinux_end && vmlinux_end <= end) { 97 mem_size = min(reg->size, (phys_addr_t)-PAGE_OFFSET); 98 99 /* 100 * Remove memblock from the end of usable area to the 101 * end of region 102 */ 103 if (reg->base + mem_size < end) 104 memblock_remove(reg->base + mem_size, 105 end - reg->base - mem_size); 106 } 107 } 108 BUG_ON(mem_size == 0); 109 110 /* Reserve from the start of the kernel to the end of the kernel */ 111 memblock_reserve(vmlinux_start, vmlinux_end - vmlinux_start); 112 113 set_max_mapnr(PFN_DOWN(mem_size)); 114 max_low_pfn = PFN_DOWN(memblock_end_of_DRAM()); 115 116 #ifdef CONFIG_BLK_DEV_INITRD 117 setup_initrd(); 118 #endif /* CONFIG_BLK_DEV_INITRD */ 119 120 early_init_fdt_reserve_self(); 121 early_init_fdt_scan_reserved_mem(); 122 memblock_allow_resize(); 123 memblock_dump_all(); 124 125 for_each_memblock(memory, reg) { 126 unsigned long start_pfn = memblock_region_memory_base_pfn(reg); 127 unsigned long end_pfn = memblock_region_memory_end_pfn(reg); 128 129 memblock_set_node(PFN_PHYS(start_pfn), 130 PFN_PHYS(end_pfn - start_pfn), 131 &memblock.memory, 0); 132 } 133 } 134 135 unsigned long va_pa_offset; 136 EXPORT_SYMBOL(va_pa_offset); 137 unsigned long pfn_base; 138 EXPORT_SYMBOL(pfn_base); 139 140 void *dtb_early_va; 141 pgd_t swapper_pg_dir[PTRS_PER_PGD] __page_aligned_bss; 142 pgd_t trampoline_pg_dir[PTRS_PER_PGD] __page_aligned_bss; 143 pte_t fixmap_pte[PTRS_PER_PTE] __page_aligned_bss; 144 static bool mmu_enabled; 145 146 #define MAX_EARLY_MAPPING_SIZE SZ_128M 147 148 pgd_t early_pg_dir[PTRS_PER_PGD] __initdata __aligned(PAGE_SIZE); 149 150 void __set_fixmap(enum fixed_addresses idx, phys_addr_t phys, pgprot_t prot) 151 { 152 unsigned long addr = __fix_to_virt(idx); 153 pte_t *ptep; 154 155 BUG_ON(idx <= FIX_HOLE || idx >= __end_of_fixed_addresses); 156 157 ptep = &fixmap_pte[pte_index(addr)]; 158 159 if (pgprot_val(prot)) { 160 set_pte(ptep, pfn_pte(phys >> PAGE_SHIFT, prot)); 161 } else { 162 pte_clear(&init_mm, addr, ptep); 163 local_flush_tlb_page(addr); 164 } 165 } 166 167 static pte_t *__init get_pte_virt(phys_addr_t pa) 168 { 169 if (mmu_enabled) { 170 clear_fixmap(FIX_PTE); 171 return (pte_t *)set_fixmap_offset(FIX_PTE, pa); 172 } else { 173 return (pte_t *)((uintptr_t)pa); 174 } 175 } 176 177 static phys_addr_t __init alloc_pte(uintptr_t va) 178 { 179 /* 180 * We only create PMD or PGD early mappings so we 181 * should never reach here with MMU disabled. 182 */ 183 BUG_ON(!mmu_enabled); 184 185 return memblock_phys_alloc(PAGE_SIZE, PAGE_SIZE); 186 } 187 188 static void __init create_pte_mapping(pte_t *ptep, 189 uintptr_t va, phys_addr_t pa, 190 phys_addr_t sz, pgprot_t prot) 191 { 192 uintptr_t pte_index = pte_index(va); 193 194 BUG_ON(sz != PAGE_SIZE); 195 196 if (pte_none(ptep[pte_index])) 197 ptep[pte_index] = pfn_pte(PFN_DOWN(pa), prot); 198 } 199 200 #ifndef __PAGETABLE_PMD_FOLDED 201 202 pmd_t trampoline_pmd[PTRS_PER_PMD] __page_aligned_bss; 203 pmd_t fixmap_pmd[PTRS_PER_PMD] __page_aligned_bss; 204 205 #if MAX_EARLY_MAPPING_SIZE < PGDIR_SIZE 206 #define NUM_EARLY_PMDS 1UL 207 #else 208 #define NUM_EARLY_PMDS (1UL + MAX_EARLY_MAPPING_SIZE / PGDIR_SIZE) 209 #endif 210 pmd_t early_pmd[PTRS_PER_PMD * NUM_EARLY_PMDS] __initdata __aligned(PAGE_SIZE); 211 212 static pmd_t *__init get_pmd_virt(phys_addr_t pa) 213 { 214 if (mmu_enabled) { 215 clear_fixmap(FIX_PMD); 216 return (pmd_t *)set_fixmap_offset(FIX_PMD, pa); 217 } else { 218 return (pmd_t *)((uintptr_t)pa); 219 } 220 } 221 222 static phys_addr_t __init alloc_pmd(uintptr_t va) 223 { 224 uintptr_t pmd_num; 225 226 if (mmu_enabled) 227 return memblock_phys_alloc(PAGE_SIZE, PAGE_SIZE); 228 229 pmd_num = (va - PAGE_OFFSET) >> PGDIR_SHIFT; 230 BUG_ON(pmd_num >= NUM_EARLY_PMDS); 231 return (uintptr_t)&early_pmd[pmd_num * PTRS_PER_PMD]; 232 } 233 234 static void __init create_pmd_mapping(pmd_t *pmdp, 235 uintptr_t va, phys_addr_t pa, 236 phys_addr_t sz, pgprot_t prot) 237 { 238 pte_t *ptep; 239 phys_addr_t pte_phys; 240 uintptr_t pmd_index = pmd_index(va); 241 242 if (sz == PMD_SIZE) { 243 if (pmd_none(pmdp[pmd_index])) 244 pmdp[pmd_index] = pfn_pmd(PFN_DOWN(pa), prot); 245 return; 246 } 247 248 if (pmd_none(pmdp[pmd_index])) { 249 pte_phys = alloc_pte(va); 250 pmdp[pmd_index] = pfn_pmd(PFN_DOWN(pte_phys), PAGE_TABLE); 251 ptep = get_pte_virt(pte_phys); 252 memset(ptep, 0, PAGE_SIZE); 253 } else { 254 pte_phys = PFN_PHYS(_pmd_pfn(pmdp[pmd_index])); 255 ptep = get_pte_virt(pte_phys); 256 } 257 258 create_pte_mapping(ptep, va, pa, sz, prot); 259 } 260 261 #define pgd_next_t pmd_t 262 #define alloc_pgd_next(__va) alloc_pmd(__va) 263 #define get_pgd_next_virt(__pa) get_pmd_virt(__pa) 264 #define create_pgd_next_mapping(__nextp, __va, __pa, __sz, __prot) \ 265 create_pmd_mapping(__nextp, __va, __pa, __sz, __prot) 266 #define PTE_PARENT_SIZE PMD_SIZE 267 #define fixmap_pgd_next fixmap_pmd 268 #else 269 #define pgd_next_t pte_t 270 #define alloc_pgd_next(__va) alloc_pte(__va) 271 #define get_pgd_next_virt(__pa) get_pte_virt(__pa) 272 #define create_pgd_next_mapping(__nextp, __va, __pa, __sz, __prot) \ 273 create_pte_mapping(__nextp, __va, __pa, __sz, __prot) 274 #define PTE_PARENT_SIZE PGDIR_SIZE 275 #define fixmap_pgd_next fixmap_pte 276 #endif 277 278 static void __init create_pgd_mapping(pgd_t *pgdp, 279 uintptr_t va, phys_addr_t pa, 280 phys_addr_t sz, pgprot_t prot) 281 { 282 pgd_next_t *nextp; 283 phys_addr_t next_phys; 284 uintptr_t pgd_index = pgd_index(va); 285 286 if (sz == PGDIR_SIZE) { 287 if (pgd_val(pgdp[pgd_index]) == 0) 288 pgdp[pgd_index] = pfn_pgd(PFN_DOWN(pa), prot); 289 return; 290 } 291 292 if (pgd_val(pgdp[pgd_index]) == 0) { 293 next_phys = alloc_pgd_next(va); 294 pgdp[pgd_index] = pfn_pgd(PFN_DOWN(next_phys), PAGE_TABLE); 295 nextp = get_pgd_next_virt(next_phys); 296 memset(nextp, 0, PAGE_SIZE); 297 } else { 298 next_phys = PFN_PHYS(_pgd_pfn(pgdp[pgd_index])); 299 nextp = get_pgd_next_virt(next_phys); 300 } 301 302 create_pgd_next_mapping(nextp, va, pa, sz, prot); 303 } 304 305 static uintptr_t __init best_map_size(phys_addr_t base, phys_addr_t size) 306 { 307 uintptr_t map_size = PAGE_SIZE; 308 309 /* Upgrade to PMD/PGDIR mappings whenever possible */ 310 if (!(base & (PTE_PARENT_SIZE - 1)) && 311 !(size & (PTE_PARENT_SIZE - 1))) 312 map_size = PTE_PARENT_SIZE; 313 314 return map_size; 315 } 316 317 /* 318 * setup_vm() is called from head.S with MMU-off. 319 * 320 * Following requirements should be honoured for setup_vm() to work 321 * correctly: 322 * 1) It should use PC-relative addressing for accessing kernel symbols. 323 * To achieve this we always use GCC cmodel=medany. 324 * 2) The compiler instrumentation for FTRACE will not work for setup_vm() 325 * so disable compiler instrumentation when FTRACE is enabled. 326 * 327 * Currently, the above requirements are honoured by using custom CFLAGS 328 * for init.o in mm/Makefile. 329 */ 330 331 #ifndef __riscv_cmodel_medany 332 #error "setup_vm() is called from head.S before relocate so it should " 333 "not use absolute addressing." 334 #endif 335 336 asmlinkage void __init setup_vm(uintptr_t dtb_pa) 337 { 338 uintptr_t va, end_va; 339 uintptr_t load_pa = (uintptr_t)(&_start); 340 uintptr_t load_sz = (uintptr_t)(&_end) - load_pa; 341 uintptr_t map_size = best_map_size(load_pa, MAX_EARLY_MAPPING_SIZE); 342 343 va_pa_offset = PAGE_OFFSET - load_pa; 344 pfn_base = PFN_DOWN(load_pa); 345 346 /* 347 * Enforce boot alignment requirements of RV32 and 348 * RV64 by only allowing PMD or PGD mappings. 349 */ 350 BUG_ON(map_size == PAGE_SIZE); 351 352 /* Sanity check alignment and size */ 353 BUG_ON((PAGE_OFFSET % PGDIR_SIZE) != 0); 354 BUG_ON((load_pa % map_size) != 0); 355 BUG_ON(load_sz > MAX_EARLY_MAPPING_SIZE); 356 357 /* Setup early PGD for fixmap */ 358 create_pgd_mapping(early_pg_dir, FIXADDR_START, 359 (uintptr_t)fixmap_pgd_next, PGDIR_SIZE, PAGE_TABLE); 360 361 #ifndef __PAGETABLE_PMD_FOLDED 362 /* Setup fixmap PMD */ 363 create_pmd_mapping(fixmap_pmd, FIXADDR_START, 364 (uintptr_t)fixmap_pte, PMD_SIZE, PAGE_TABLE); 365 /* Setup trampoline PGD and PMD */ 366 create_pgd_mapping(trampoline_pg_dir, PAGE_OFFSET, 367 (uintptr_t)trampoline_pmd, PGDIR_SIZE, PAGE_TABLE); 368 create_pmd_mapping(trampoline_pmd, PAGE_OFFSET, 369 load_pa, PMD_SIZE, PAGE_KERNEL_EXEC); 370 #else 371 /* Setup trampoline PGD */ 372 create_pgd_mapping(trampoline_pg_dir, PAGE_OFFSET, 373 load_pa, PGDIR_SIZE, PAGE_KERNEL_EXEC); 374 #endif 375 376 /* 377 * Setup early PGD covering entire kernel which will allows 378 * us to reach paging_init(). We map all memory banks later 379 * in setup_vm_final() below. 380 */ 381 end_va = PAGE_OFFSET + load_sz; 382 for (va = PAGE_OFFSET; va < end_va; va += map_size) 383 create_pgd_mapping(early_pg_dir, va, 384 load_pa + (va - PAGE_OFFSET), 385 map_size, PAGE_KERNEL_EXEC); 386 387 /* Create fixed mapping for early FDT parsing */ 388 end_va = __fix_to_virt(FIX_FDT) + FIX_FDT_SIZE; 389 for (va = __fix_to_virt(FIX_FDT); va < end_va; va += PAGE_SIZE) 390 create_pte_mapping(fixmap_pte, va, 391 dtb_pa + (va - __fix_to_virt(FIX_FDT)), 392 PAGE_SIZE, PAGE_KERNEL); 393 394 /* Save pointer to DTB for early FDT parsing */ 395 dtb_early_va = (void *)fix_to_virt(FIX_FDT) + (dtb_pa & ~PAGE_MASK); 396 } 397 398 static void __init setup_vm_final(void) 399 { 400 uintptr_t va, map_size; 401 phys_addr_t pa, start, end; 402 struct memblock_region *reg; 403 404 /* Set mmu_enabled flag */ 405 mmu_enabled = true; 406 407 /* Setup swapper PGD for fixmap */ 408 create_pgd_mapping(swapper_pg_dir, FIXADDR_START, 409 __pa(fixmap_pgd_next), 410 PGDIR_SIZE, PAGE_TABLE); 411 412 /* Map all memory banks */ 413 for_each_memblock(memory, reg) { 414 start = reg->base; 415 end = start + reg->size; 416 417 if (start >= end) 418 break; 419 if (memblock_is_nomap(reg)) 420 continue; 421 if (start <= __pa(PAGE_OFFSET) && 422 __pa(PAGE_OFFSET) < end) 423 start = __pa(PAGE_OFFSET); 424 425 map_size = best_map_size(start, end - start); 426 for (pa = start; pa < end; pa += map_size) { 427 va = (uintptr_t)__va(pa); 428 create_pgd_mapping(swapper_pg_dir, va, pa, 429 map_size, PAGE_KERNEL_EXEC); 430 } 431 } 432 433 /* Clear fixmap PTE and PMD mappings */ 434 clear_fixmap(FIX_PTE); 435 clear_fixmap(FIX_PMD); 436 437 /* Move to swapper page table */ 438 csr_write(sptbr, PFN_DOWN(__pa(swapper_pg_dir)) | SATP_MODE); 439 local_flush_tlb_all(); 440 } 441 442 void __init paging_init(void) 443 { 444 setup_vm_final(); 445 setup_zero_page(); 446 zone_sizes_init(); 447 } 448