1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2012 Regents of the University of California 4 * Copyright (C) 2019 Western Digital Corporation or its affiliates. 5 */ 6 7 #include <linux/init.h> 8 #include <linux/mm.h> 9 #include <linux/memblock.h> 10 #include <linux/initrd.h> 11 #include <linux/swap.h> 12 #include <linux/sizes.h> 13 #include <linux/of_fdt.h> 14 #include <linux/libfdt.h> 15 #include <linux/set_memory.h> 16 17 #include <asm/fixmap.h> 18 #include <asm/tlbflush.h> 19 #include <asm/sections.h> 20 #include <asm/soc.h> 21 #include <asm/pgtable.h> 22 #include <asm/io.h> 23 #include <asm/ptdump.h> 24 25 #include "../kernel/head.h" 26 27 unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)] 28 __page_aligned_bss; 29 EXPORT_SYMBOL(empty_zero_page); 30 31 extern char _start[]; 32 void *dtb_early_va; 33 34 static void __init zone_sizes_init(void) 35 { 36 unsigned long max_zone_pfns[MAX_NR_ZONES] = { 0, }; 37 38 #ifdef CONFIG_ZONE_DMA32 39 max_zone_pfns[ZONE_DMA32] = PFN_DOWN(min(4UL * SZ_1G, 40 (unsigned long) PFN_PHYS(max_low_pfn))); 41 #endif 42 max_zone_pfns[ZONE_NORMAL] = max_low_pfn; 43 44 free_area_init(max_zone_pfns); 45 } 46 47 static void setup_zero_page(void) 48 { 49 memset((void *)empty_zero_page, 0, PAGE_SIZE); 50 } 51 52 #if defined(CONFIG_MMU) && defined(CONFIG_DEBUG_VM) 53 static inline void print_mlk(char *name, unsigned long b, unsigned long t) 54 { 55 pr_notice("%12s : 0x%08lx - 0x%08lx (%4ld kB)\n", name, b, t, 56 (((t) - (b)) >> 10)); 57 } 58 59 static inline void print_mlm(char *name, unsigned long b, unsigned long t) 60 { 61 pr_notice("%12s : 0x%08lx - 0x%08lx (%4ld MB)\n", name, b, t, 62 (((t) - (b)) >> 20)); 63 } 64 65 static void print_vm_layout(void) 66 { 67 pr_notice("Virtual kernel memory layout:\n"); 68 print_mlk("fixmap", (unsigned long)FIXADDR_START, 69 (unsigned long)FIXADDR_TOP); 70 print_mlm("pci io", (unsigned long)PCI_IO_START, 71 (unsigned long)PCI_IO_END); 72 print_mlm("vmemmap", (unsigned long)VMEMMAP_START, 73 (unsigned long)VMEMMAP_END); 74 print_mlm("vmalloc", (unsigned long)VMALLOC_START, 75 (unsigned long)VMALLOC_END); 76 print_mlm("lowmem", (unsigned long)PAGE_OFFSET, 77 (unsigned long)high_memory); 78 } 79 #else 80 static void print_vm_layout(void) { } 81 #endif /* CONFIG_DEBUG_VM */ 82 83 void __init mem_init(void) 84 { 85 #ifdef CONFIG_FLATMEM 86 BUG_ON(!mem_map); 87 #endif /* CONFIG_FLATMEM */ 88 89 high_memory = (void *)(__va(PFN_PHYS(max_low_pfn))); 90 memblock_free_all(); 91 92 mem_init_print_info(NULL); 93 print_vm_layout(); 94 } 95 96 #ifdef CONFIG_BLK_DEV_INITRD 97 static void __init setup_initrd(void) 98 { 99 unsigned long size; 100 101 if (initrd_start >= initrd_end) { 102 pr_info("initrd not found or empty"); 103 goto disable; 104 } 105 if (__pa_symbol(initrd_end) > PFN_PHYS(max_low_pfn)) { 106 pr_err("initrd extends beyond end of memory"); 107 goto disable; 108 } 109 110 size = initrd_end - initrd_start; 111 memblock_reserve(__pa_symbol(initrd_start), size); 112 initrd_below_start_ok = 1; 113 114 pr_info("Initial ramdisk at: 0x%p (%lu bytes)\n", 115 (void *)(initrd_start), size); 116 return; 117 disable: 118 pr_cont(" - disabling initrd\n"); 119 initrd_start = 0; 120 initrd_end = 0; 121 } 122 #endif /* CONFIG_BLK_DEV_INITRD */ 123 124 static phys_addr_t dtb_early_pa __initdata; 125 126 void __init setup_bootmem(void) 127 { 128 struct memblock_region *reg; 129 phys_addr_t mem_size = 0; 130 phys_addr_t vmlinux_end = __pa_symbol(&_end); 131 phys_addr_t vmlinux_start = __pa_symbol(&_start); 132 133 /* Find the memory region containing the kernel */ 134 for_each_memblock(memory, reg) { 135 phys_addr_t end = reg->base + reg->size; 136 137 if (reg->base <= vmlinux_start && vmlinux_end <= end) { 138 mem_size = min(reg->size, (phys_addr_t)-PAGE_OFFSET); 139 140 /* 141 * Remove memblock from the end of usable area to the 142 * end of region 143 */ 144 if (reg->base + mem_size < end) 145 memblock_remove(reg->base + mem_size, 146 end - reg->base - mem_size); 147 } 148 } 149 BUG_ON(mem_size == 0); 150 151 /* Reserve from the start of the kernel to the end of the kernel */ 152 memblock_reserve(vmlinux_start, vmlinux_end - vmlinux_start); 153 154 set_max_mapnr(PFN_DOWN(mem_size)); 155 max_pfn = PFN_DOWN(memblock_end_of_DRAM()); 156 max_low_pfn = max_pfn; 157 158 #ifdef CONFIG_BLK_DEV_INITRD 159 setup_initrd(); 160 #endif /* CONFIG_BLK_DEV_INITRD */ 161 162 /* 163 * Avoid using early_init_fdt_reserve_self() since __pa() does 164 * not work for DTB pointers that are fixmap addresses 165 */ 166 memblock_reserve(dtb_early_pa, fdt_totalsize(dtb_early_va)); 167 168 early_init_fdt_scan_reserved_mem(); 169 memblock_allow_resize(); 170 memblock_dump_all(); 171 172 for_each_memblock(memory, reg) { 173 unsigned long start_pfn = memblock_region_memory_base_pfn(reg); 174 unsigned long end_pfn = memblock_region_memory_end_pfn(reg); 175 176 memblock_set_node(PFN_PHYS(start_pfn), 177 PFN_PHYS(end_pfn - start_pfn), 178 &memblock.memory, 0); 179 } 180 } 181 182 #ifdef CONFIG_MMU 183 unsigned long va_pa_offset; 184 EXPORT_SYMBOL(va_pa_offset); 185 unsigned long pfn_base; 186 EXPORT_SYMBOL(pfn_base); 187 188 pgd_t swapper_pg_dir[PTRS_PER_PGD] __page_aligned_bss; 189 pgd_t trampoline_pg_dir[PTRS_PER_PGD] __page_aligned_bss; 190 pte_t fixmap_pte[PTRS_PER_PTE] __page_aligned_bss; 191 static bool mmu_enabled; 192 193 #define MAX_EARLY_MAPPING_SIZE SZ_128M 194 195 pgd_t early_pg_dir[PTRS_PER_PGD] __initdata __aligned(PAGE_SIZE); 196 197 void __set_fixmap(enum fixed_addresses idx, phys_addr_t phys, pgprot_t prot) 198 { 199 unsigned long addr = __fix_to_virt(idx); 200 pte_t *ptep; 201 202 BUG_ON(idx <= FIX_HOLE || idx >= __end_of_fixed_addresses); 203 204 ptep = &fixmap_pte[pte_index(addr)]; 205 206 if (pgprot_val(prot)) { 207 set_pte(ptep, pfn_pte(phys >> PAGE_SHIFT, prot)); 208 } else { 209 pte_clear(&init_mm, addr, ptep); 210 local_flush_tlb_page(addr); 211 } 212 } 213 214 static pte_t *__init get_pte_virt(phys_addr_t pa) 215 { 216 if (mmu_enabled) { 217 clear_fixmap(FIX_PTE); 218 return (pte_t *)set_fixmap_offset(FIX_PTE, pa); 219 } else { 220 return (pte_t *)((uintptr_t)pa); 221 } 222 } 223 224 static phys_addr_t __init alloc_pte(uintptr_t va) 225 { 226 /* 227 * We only create PMD or PGD early mappings so we 228 * should never reach here with MMU disabled. 229 */ 230 BUG_ON(!mmu_enabled); 231 232 return memblock_phys_alloc(PAGE_SIZE, PAGE_SIZE); 233 } 234 235 static void __init create_pte_mapping(pte_t *ptep, 236 uintptr_t va, phys_addr_t pa, 237 phys_addr_t sz, pgprot_t prot) 238 { 239 uintptr_t pte_index = pte_index(va); 240 241 BUG_ON(sz != PAGE_SIZE); 242 243 if (pte_none(ptep[pte_index])) 244 ptep[pte_index] = pfn_pte(PFN_DOWN(pa), prot); 245 } 246 247 #ifndef __PAGETABLE_PMD_FOLDED 248 249 pmd_t trampoline_pmd[PTRS_PER_PMD] __page_aligned_bss; 250 pmd_t fixmap_pmd[PTRS_PER_PMD] __page_aligned_bss; 251 252 #if MAX_EARLY_MAPPING_SIZE < PGDIR_SIZE 253 #define NUM_EARLY_PMDS 1UL 254 #else 255 #define NUM_EARLY_PMDS (1UL + MAX_EARLY_MAPPING_SIZE / PGDIR_SIZE) 256 #endif 257 pmd_t early_pmd[PTRS_PER_PMD * NUM_EARLY_PMDS] __initdata __aligned(PAGE_SIZE); 258 259 static pmd_t *__init get_pmd_virt(phys_addr_t pa) 260 { 261 if (mmu_enabled) { 262 clear_fixmap(FIX_PMD); 263 return (pmd_t *)set_fixmap_offset(FIX_PMD, pa); 264 } else { 265 return (pmd_t *)((uintptr_t)pa); 266 } 267 } 268 269 static phys_addr_t __init alloc_pmd(uintptr_t va) 270 { 271 uintptr_t pmd_num; 272 273 if (mmu_enabled) 274 return memblock_phys_alloc(PAGE_SIZE, PAGE_SIZE); 275 276 pmd_num = (va - PAGE_OFFSET) >> PGDIR_SHIFT; 277 BUG_ON(pmd_num >= NUM_EARLY_PMDS); 278 return (uintptr_t)&early_pmd[pmd_num * PTRS_PER_PMD]; 279 } 280 281 static void __init create_pmd_mapping(pmd_t *pmdp, 282 uintptr_t va, phys_addr_t pa, 283 phys_addr_t sz, pgprot_t prot) 284 { 285 pte_t *ptep; 286 phys_addr_t pte_phys; 287 uintptr_t pmd_index = pmd_index(va); 288 289 if (sz == PMD_SIZE) { 290 if (pmd_none(pmdp[pmd_index])) 291 pmdp[pmd_index] = pfn_pmd(PFN_DOWN(pa), prot); 292 return; 293 } 294 295 if (pmd_none(pmdp[pmd_index])) { 296 pte_phys = alloc_pte(va); 297 pmdp[pmd_index] = pfn_pmd(PFN_DOWN(pte_phys), PAGE_TABLE); 298 ptep = get_pte_virt(pte_phys); 299 memset(ptep, 0, PAGE_SIZE); 300 } else { 301 pte_phys = PFN_PHYS(_pmd_pfn(pmdp[pmd_index])); 302 ptep = get_pte_virt(pte_phys); 303 } 304 305 create_pte_mapping(ptep, va, pa, sz, prot); 306 } 307 308 #define pgd_next_t pmd_t 309 #define alloc_pgd_next(__va) alloc_pmd(__va) 310 #define get_pgd_next_virt(__pa) get_pmd_virt(__pa) 311 #define create_pgd_next_mapping(__nextp, __va, __pa, __sz, __prot) \ 312 create_pmd_mapping(__nextp, __va, __pa, __sz, __prot) 313 #define fixmap_pgd_next fixmap_pmd 314 #else 315 #define pgd_next_t pte_t 316 #define alloc_pgd_next(__va) alloc_pte(__va) 317 #define get_pgd_next_virt(__pa) get_pte_virt(__pa) 318 #define create_pgd_next_mapping(__nextp, __va, __pa, __sz, __prot) \ 319 create_pte_mapping(__nextp, __va, __pa, __sz, __prot) 320 #define fixmap_pgd_next fixmap_pte 321 #endif 322 323 static void __init create_pgd_mapping(pgd_t *pgdp, 324 uintptr_t va, phys_addr_t pa, 325 phys_addr_t sz, pgprot_t prot) 326 { 327 pgd_next_t *nextp; 328 phys_addr_t next_phys; 329 uintptr_t pgd_index = pgd_index(va); 330 331 if (sz == PGDIR_SIZE) { 332 if (pgd_val(pgdp[pgd_index]) == 0) 333 pgdp[pgd_index] = pfn_pgd(PFN_DOWN(pa), prot); 334 return; 335 } 336 337 if (pgd_val(pgdp[pgd_index]) == 0) { 338 next_phys = alloc_pgd_next(va); 339 pgdp[pgd_index] = pfn_pgd(PFN_DOWN(next_phys), PAGE_TABLE); 340 nextp = get_pgd_next_virt(next_phys); 341 memset(nextp, 0, PAGE_SIZE); 342 } else { 343 next_phys = PFN_PHYS(_pgd_pfn(pgdp[pgd_index])); 344 nextp = get_pgd_next_virt(next_phys); 345 } 346 347 create_pgd_next_mapping(nextp, va, pa, sz, prot); 348 } 349 350 static uintptr_t __init best_map_size(phys_addr_t base, phys_addr_t size) 351 { 352 /* Upgrade to PMD_SIZE mappings whenever possible */ 353 if ((base & (PMD_SIZE - 1)) || (size & (PMD_SIZE - 1))) 354 return PAGE_SIZE; 355 356 return PMD_SIZE; 357 } 358 359 /* 360 * setup_vm() is called from head.S with MMU-off. 361 * 362 * Following requirements should be honoured for setup_vm() to work 363 * correctly: 364 * 1) It should use PC-relative addressing for accessing kernel symbols. 365 * To achieve this we always use GCC cmodel=medany. 366 * 2) The compiler instrumentation for FTRACE will not work for setup_vm() 367 * so disable compiler instrumentation when FTRACE is enabled. 368 * 369 * Currently, the above requirements are honoured by using custom CFLAGS 370 * for init.o in mm/Makefile. 371 */ 372 373 #ifndef __riscv_cmodel_medany 374 #error "setup_vm() is called from head.S before relocate so it should not use absolute addressing." 375 #endif 376 377 asmlinkage void __init setup_vm(uintptr_t dtb_pa) 378 { 379 uintptr_t va, end_va; 380 uintptr_t load_pa = (uintptr_t)(&_start); 381 uintptr_t load_sz = (uintptr_t)(&_end) - load_pa; 382 uintptr_t map_size = best_map_size(load_pa, MAX_EARLY_MAPPING_SIZE); 383 384 va_pa_offset = PAGE_OFFSET - load_pa; 385 pfn_base = PFN_DOWN(load_pa); 386 387 /* 388 * Enforce boot alignment requirements of RV32 and 389 * RV64 by only allowing PMD or PGD mappings. 390 */ 391 BUG_ON(map_size == PAGE_SIZE); 392 393 /* Sanity check alignment and size */ 394 BUG_ON((PAGE_OFFSET % PGDIR_SIZE) != 0); 395 BUG_ON((load_pa % map_size) != 0); 396 BUG_ON(load_sz > MAX_EARLY_MAPPING_SIZE); 397 398 /* Setup early PGD for fixmap */ 399 create_pgd_mapping(early_pg_dir, FIXADDR_START, 400 (uintptr_t)fixmap_pgd_next, PGDIR_SIZE, PAGE_TABLE); 401 402 #ifndef __PAGETABLE_PMD_FOLDED 403 /* Setup fixmap PMD */ 404 create_pmd_mapping(fixmap_pmd, FIXADDR_START, 405 (uintptr_t)fixmap_pte, PMD_SIZE, PAGE_TABLE); 406 /* Setup trampoline PGD and PMD */ 407 create_pgd_mapping(trampoline_pg_dir, PAGE_OFFSET, 408 (uintptr_t)trampoline_pmd, PGDIR_SIZE, PAGE_TABLE); 409 create_pmd_mapping(trampoline_pmd, PAGE_OFFSET, 410 load_pa, PMD_SIZE, PAGE_KERNEL_EXEC); 411 #else 412 /* Setup trampoline PGD */ 413 create_pgd_mapping(trampoline_pg_dir, PAGE_OFFSET, 414 load_pa, PGDIR_SIZE, PAGE_KERNEL_EXEC); 415 #endif 416 417 /* 418 * Setup early PGD covering entire kernel which will allows 419 * us to reach paging_init(). We map all memory banks later 420 * in setup_vm_final() below. 421 */ 422 end_va = PAGE_OFFSET + load_sz; 423 for (va = PAGE_OFFSET; va < end_va; va += map_size) 424 create_pgd_mapping(early_pg_dir, va, 425 load_pa + (va - PAGE_OFFSET), 426 map_size, PAGE_KERNEL_EXEC); 427 428 /* Create fixed mapping for early FDT parsing */ 429 end_va = __fix_to_virt(FIX_FDT) + FIX_FDT_SIZE; 430 for (va = __fix_to_virt(FIX_FDT); va < end_va; va += PAGE_SIZE) 431 create_pte_mapping(fixmap_pte, va, 432 dtb_pa + (va - __fix_to_virt(FIX_FDT)), 433 PAGE_SIZE, PAGE_KERNEL); 434 435 /* Save pointer to DTB for early FDT parsing */ 436 dtb_early_va = (void *)fix_to_virt(FIX_FDT) + (dtb_pa & ~PAGE_MASK); 437 /* Save physical address for memblock reservation */ 438 dtb_early_pa = dtb_pa; 439 } 440 441 static void __init setup_vm_final(void) 442 { 443 uintptr_t va, map_size; 444 phys_addr_t pa, start, end; 445 struct memblock_region *reg; 446 447 /* Set mmu_enabled flag */ 448 mmu_enabled = true; 449 450 /* Setup swapper PGD for fixmap */ 451 create_pgd_mapping(swapper_pg_dir, FIXADDR_START, 452 __pa_symbol(fixmap_pgd_next), 453 PGDIR_SIZE, PAGE_TABLE); 454 455 /* Map all memory banks */ 456 for_each_memblock(memory, reg) { 457 start = reg->base; 458 end = start + reg->size; 459 460 if (start >= end) 461 break; 462 if (memblock_is_nomap(reg)) 463 continue; 464 if (start <= __pa(PAGE_OFFSET) && 465 __pa(PAGE_OFFSET) < end) 466 start = __pa(PAGE_OFFSET); 467 468 map_size = best_map_size(start, end - start); 469 for (pa = start; pa < end; pa += map_size) { 470 va = (uintptr_t)__va(pa); 471 create_pgd_mapping(swapper_pg_dir, va, pa, 472 map_size, PAGE_KERNEL_EXEC); 473 } 474 } 475 476 /* Clear fixmap PTE and PMD mappings */ 477 clear_fixmap(FIX_PTE); 478 clear_fixmap(FIX_PMD); 479 480 /* Move to swapper page table */ 481 csr_write(CSR_SATP, PFN_DOWN(__pa_symbol(swapper_pg_dir)) | SATP_MODE); 482 local_flush_tlb_all(); 483 } 484 485 void free_initmem(void) 486 { 487 unsigned long init_begin = (unsigned long)__init_begin; 488 unsigned long init_end = (unsigned long)__init_end; 489 490 /* Make the region as non-execuatble. */ 491 set_memory_nx(init_begin, (init_end - init_begin) >> PAGE_SHIFT); 492 free_initmem_default(POISON_FREE_INITMEM); 493 } 494 495 #else 496 asmlinkage void __init setup_vm(uintptr_t dtb_pa) 497 { 498 #ifdef CONFIG_BUILTIN_DTB 499 dtb_early_va = soc_lookup_builtin_dtb(); 500 if (!dtb_early_va) { 501 /* Fallback to first available DTS */ 502 dtb_early_va = (void *) __dtb_start; 503 } 504 #else 505 dtb_early_va = (void *)dtb_pa; 506 #endif 507 } 508 509 static inline void setup_vm_final(void) 510 { 511 } 512 #endif /* CONFIG_MMU */ 513 514 #ifdef CONFIG_STRICT_KERNEL_RWX 515 void mark_rodata_ro(void) 516 { 517 unsigned long text_start = (unsigned long)_text; 518 unsigned long text_end = (unsigned long)_etext; 519 unsigned long rodata_start = (unsigned long)__start_rodata; 520 unsigned long data_start = (unsigned long)_data; 521 unsigned long max_low = (unsigned long)(__va(PFN_PHYS(max_low_pfn))); 522 523 set_memory_ro(text_start, (text_end - text_start) >> PAGE_SHIFT); 524 set_memory_ro(rodata_start, (data_start - rodata_start) >> PAGE_SHIFT); 525 set_memory_nx(rodata_start, (data_start - rodata_start) >> PAGE_SHIFT); 526 set_memory_nx(data_start, (max_low - data_start) >> PAGE_SHIFT); 527 528 debug_checkwx(); 529 } 530 #endif 531 532 void __init paging_init(void) 533 { 534 setup_vm_final(); 535 memblocks_present(); 536 sparse_init(); 537 setup_zero_page(); 538 zone_sizes_init(); 539 } 540 541 #ifdef CONFIG_SPARSEMEM_VMEMMAP 542 int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node, 543 struct vmem_altmap *altmap) 544 { 545 return vmemmap_populate_basepages(start, end, node); 546 } 547 #endif 548