1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2012 Regents of the University of California 4 * Copyright (C) 2019 Western Digital Corporation or its affiliates. 5 * Copyright (C) 2020 FORTH-ICS/CARV 6 * Nick Kossifidis <mick@ics.forth.gr> 7 */ 8 9 #include <linux/init.h> 10 #include <linux/mm.h> 11 #include <linux/memblock.h> 12 #include <linux/initrd.h> 13 #include <linux/swap.h> 14 #include <linux/swiotlb.h> 15 #include <linux/sizes.h> 16 #include <linux/of_fdt.h> 17 #include <linux/of_reserved_mem.h> 18 #include <linux/libfdt.h> 19 #include <linux/set_memory.h> 20 #include <linux/dma-map-ops.h> 21 #include <linux/crash_dump.h> 22 #include <linux/hugetlb.h> 23 24 #include <asm/fixmap.h> 25 #include <asm/tlbflush.h> 26 #include <asm/sections.h> 27 #include <asm/soc.h> 28 #include <asm/io.h> 29 #include <asm/ptdump.h> 30 #include <asm/numa.h> 31 32 #include "../kernel/head.h" 33 34 struct kernel_mapping kernel_map __ro_after_init; 35 EXPORT_SYMBOL(kernel_map); 36 #ifdef CONFIG_XIP_KERNEL 37 #define kernel_map (*(struct kernel_mapping *)XIP_FIXUP(&kernel_map)) 38 #endif 39 40 #ifdef CONFIG_64BIT 41 u64 satp_mode __ro_after_init = !IS_ENABLED(CONFIG_XIP_KERNEL) ? SATP_MODE_57 : SATP_MODE_39; 42 #else 43 u64 satp_mode __ro_after_init = SATP_MODE_32; 44 #endif 45 EXPORT_SYMBOL(satp_mode); 46 47 bool pgtable_l4_enabled = IS_ENABLED(CONFIG_64BIT) && !IS_ENABLED(CONFIG_XIP_KERNEL); 48 bool pgtable_l5_enabled = IS_ENABLED(CONFIG_64BIT) && !IS_ENABLED(CONFIG_XIP_KERNEL); 49 EXPORT_SYMBOL(pgtable_l4_enabled); 50 EXPORT_SYMBOL(pgtable_l5_enabled); 51 52 phys_addr_t phys_ram_base __ro_after_init; 53 EXPORT_SYMBOL(phys_ram_base); 54 55 unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)] 56 __page_aligned_bss; 57 EXPORT_SYMBOL(empty_zero_page); 58 59 extern char _start[]; 60 #define DTB_EARLY_BASE_VA PGDIR_SIZE 61 void *_dtb_early_va __initdata; 62 uintptr_t _dtb_early_pa __initdata; 63 64 static phys_addr_t dma32_phys_limit __initdata; 65 66 static void __init zone_sizes_init(void) 67 { 68 unsigned long max_zone_pfns[MAX_NR_ZONES] = { 0, }; 69 70 #ifdef CONFIG_ZONE_DMA32 71 max_zone_pfns[ZONE_DMA32] = PFN_DOWN(dma32_phys_limit); 72 #endif 73 max_zone_pfns[ZONE_NORMAL] = max_low_pfn; 74 75 free_area_init(max_zone_pfns); 76 } 77 78 #if defined(CONFIG_MMU) && defined(CONFIG_DEBUG_VM) 79 static inline void print_mlk(char *name, unsigned long b, unsigned long t) 80 { 81 pr_notice("%12s : 0x%08lx - 0x%08lx (%4ld kB)\n", name, b, t, 82 (((t) - (b)) >> 10)); 83 } 84 85 static inline void print_mlm(char *name, unsigned long b, unsigned long t) 86 { 87 pr_notice("%12s : 0x%08lx - 0x%08lx (%4ld MB)\n", name, b, t, 88 (((t) - (b)) >> 20)); 89 } 90 91 static void __init print_vm_layout(void) 92 { 93 pr_notice("Virtual kernel memory layout:\n"); 94 print_mlk("fixmap", (unsigned long)FIXADDR_START, 95 (unsigned long)FIXADDR_TOP); 96 print_mlm("pci io", (unsigned long)PCI_IO_START, 97 (unsigned long)PCI_IO_END); 98 print_mlm("vmemmap", (unsigned long)VMEMMAP_START, 99 (unsigned long)VMEMMAP_END); 100 print_mlm("vmalloc", (unsigned long)VMALLOC_START, 101 (unsigned long)VMALLOC_END); 102 print_mlm("lowmem", (unsigned long)PAGE_OFFSET, 103 (unsigned long)high_memory); 104 if (IS_ENABLED(CONFIG_64BIT)) { 105 #ifdef CONFIG_KASAN 106 print_mlm("kasan", KASAN_SHADOW_START, KASAN_SHADOW_END); 107 #endif 108 109 print_mlm("kernel", (unsigned long)KERNEL_LINK_ADDR, 110 (unsigned long)ADDRESS_SPACE_END); 111 } 112 } 113 #else 114 static void print_vm_layout(void) { } 115 #endif /* CONFIG_DEBUG_VM */ 116 117 void __init mem_init(void) 118 { 119 #ifdef CONFIG_FLATMEM 120 BUG_ON(!mem_map); 121 #endif /* CONFIG_FLATMEM */ 122 123 swiotlb_init(max_pfn > PFN_DOWN(dma32_phys_limit), SWIOTLB_VERBOSE); 124 memblock_free_all(); 125 126 print_vm_layout(); 127 } 128 129 /* Limit the memory size via mem. */ 130 static phys_addr_t memory_limit; 131 132 static int __init early_mem(char *p) 133 { 134 u64 size; 135 136 if (!p) 137 return 1; 138 139 size = memparse(p, &p) & PAGE_MASK; 140 memory_limit = min_t(u64, size, memory_limit); 141 142 pr_notice("Memory limited to %lldMB\n", (u64)memory_limit >> 20); 143 144 return 0; 145 } 146 early_param("mem", early_mem); 147 148 static void __init setup_bootmem(void) 149 { 150 phys_addr_t vmlinux_end = __pa_symbol(&_end); 151 phys_addr_t max_mapped_addr; 152 phys_addr_t phys_ram_end, vmlinux_start; 153 154 if (IS_ENABLED(CONFIG_XIP_KERNEL)) 155 vmlinux_start = __pa_symbol(&_sdata); 156 else 157 vmlinux_start = __pa_symbol(&_start); 158 159 memblock_enforce_memory_limit(memory_limit); 160 161 /* 162 * Make sure we align the reservation on PMD_SIZE since we will 163 * map the kernel in the linear mapping as read-only: we do not want 164 * any allocation to happen between _end and the next pmd aligned page. 165 */ 166 if (IS_ENABLED(CONFIG_64BIT) && IS_ENABLED(CONFIG_STRICT_KERNEL_RWX)) 167 vmlinux_end = (vmlinux_end + PMD_SIZE - 1) & PMD_MASK; 168 /* 169 * Reserve from the start of the kernel to the end of the kernel 170 */ 171 memblock_reserve(vmlinux_start, vmlinux_end - vmlinux_start); 172 173 phys_ram_end = memblock_end_of_DRAM(); 174 if (!IS_ENABLED(CONFIG_XIP_KERNEL)) 175 phys_ram_base = memblock_start_of_DRAM(); 176 /* 177 * memblock allocator is not aware of the fact that last 4K bytes of 178 * the addressable memory can not be mapped because of IS_ERR_VALUE 179 * macro. Make sure that last 4k bytes are not usable by memblock 180 * if end of dram is equal to maximum addressable memory. For 64-bit 181 * kernel, this problem can't happen here as the end of the virtual 182 * address space is occupied by the kernel mapping then this check must 183 * be done as soon as the kernel mapping base address is determined. 184 */ 185 if (!IS_ENABLED(CONFIG_64BIT)) { 186 max_mapped_addr = __pa(~(ulong)0); 187 if (max_mapped_addr == (phys_ram_end - 1)) 188 memblock_set_current_limit(max_mapped_addr - 4096); 189 } 190 191 min_low_pfn = PFN_UP(phys_ram_base); 192 max_low_pfn = max_pfn = PFN_DOWN(phys_ram_end); 193 high_memory = (void *)(__va(PFN_PHYS(max_low_pfn))); 194 195 dma32_phys_limit = min(4UL * SZ_1G, (unsigned long)PFN_PHYS(max_low_pfn)); 196 set_max_mapnr(max_low_pfn - ARCH_PFN_OFFSET); 197 198 reserve_initrd_mem(); 199 /* 200 * If DTB is built in, no need to reserve its memblock. 201 * Otherwise, do reserve it but avoid using 202 * early_init_fdt_reserve_self() since __pa() does 203 * not work for DTB pointers that are fixmap addresses 204 */ 205 if (!IS_ENABLED(CONFIG_BUILTIN_DTB)) { 206 /* 207 * In case the DTB is not located in a memory region we won't 208 * be able to locate it later on via the linear mapping and 209 * get a segfault when accessing it via __va(dtb_early_pa). 210 * To avoid this situation copy DTB to a memory region. 211 * Note that memblock_phys_alloc will also reserve DTB region. 212 */ 213 if (!memblock_is_memory(dtb_early_pa)) { 214 size_t fdt_size = fdt_totalsize(dtb_early_va); 215 phys_addr_t new_dtb_early_pa = memblock_phys_alloc(fdt_size, PAGE_SIZE); 216 void *new_dtb_early_va = early_memremap(new_dtb_early_pa, fdt_size); 217 218 memcpy(new_dtb_early_va, dtb_early_va, fdt_size); 219 early_memunmap(new_dtb_early_va, fdt_size); 220 _dtb_early_pa = new_dtb_early_pa; 221 } else 222 memblock_reserve(dtb_early_pa, fdt_totalsize(dtb_early_va)); 223 } 224 225 early_init_fdt_scan_reserved_mem(); 226 dma_contiguous_reserve(dma32_phys_limit); 227 if (IS_ENABLED(CONFIG_64BIT)) 228 hugetlb_cma_reserve(PUD_SHIFT - PAGE_SHIFT); 229 memblock_allow_resize(); 230 } 231 232 #ifdef CONFIG_MMU 233 struct pt_alloc_ops pt_ops __initdata; 234 235 unsigned long riscv_pfn_base __ro_after_init; 236 EXPORT_SYMBOL(riscv_pfn_base); 237 238 pgd_t swapper_pg_dir[PTRS_PER_PGD] __page_aligned_bss; 239 pgd_t trampoline_pg_dir[PTRS_PER_PGD] __page_aligned_bss; 240 static pte_t fixmap_pte[PTRS_PER_PTE] __page_aligned_bss; 241 242 pgd_t early_pg_dir[PTRS_PER_PGD] __initdata __aligned(PAGE_SIZE); 243 static p4d_t __maybe_unused early_dtb_p4d[PTRS_PER_P4D] __initdata __aligned(PAGE_SIZE); 244 static pud_t __maybe_unused early_dtb_pud[PTRS_PER_PUD] __initdata __aligned(PAGE_SIZE); 245 static pmd_t __maybe_unused early_dtb_pmd[PTRS_PER_PMD] __initdata __aligned(PAGE_SIZE); 246 247 #ifdef CONFIG_XIP_KERNEL 248 #define pt_ops (*(struct pt_alloc_ops *)XIP_FIXUP(&pt_ops)) 249 #define riscv_pfn_base (*(unsigned long *)XIP_FIXUP(&riscv_pfn_base)) 250 #define trampoline_pg_dir ((pgd_t *)XIP_FIXUP(trampoline_pg_dir)) 251 #define fixmap_pte ((pte_t *)XIP_FIXUP(fixmap_pte)) 252 #define early_pg_dir ((pgd_t *)XIP_FIXUP(early_pg_dir)) 253 #endif /* CONFIG_XIP_KERNEL */ 254 255 void __set_fixmap(enum fixed_addresses idx, phys_addr_t phys, pgprot_t prot) 256 { 257 unsigned long addr = __fix_to_virt(idx); 258 pte_t *ptep; 259 260 BUG_ON(idx <= FIX_HOLE || idx >= __end_of_fixed_addresses); 261 262 ptep = &fixmap_pte[pte_index(addr)]; 263 264 if (pgprot_val(prot)) 265 set_pte(ptep, pfn_pte(phys >> PAGE_SHIFT, prot)); 266 else 267 pte_clear(&init_mm, addr, ptep); 268 local_flush_tlb_page(addr); 269 } 270 271 static inline pte_t *__init get_pte_virt_early(phys_addr_t pa) 272 { 273 return (pte_t *)((uintptr_t)pa); 274 } 275 276 static inline pte_t *__init get_pte_virt_fixmap(phys_addr_t pa) 277 { 278 clear_fixmap(FIX_PTE); 279 return (pte_t *)set_fixmap_offset(FIX_PTE, pa); 280 } 281 282 static inline pte_t *__init get_pte_virt_late(phys_addr_t pa) 283 { 284 return (pte_t *) __va(pa); 285 } 286 287 static inline phys_addr_t __init alloc_pte_early(uintptr_t va) 288 { 289 /* 290 * We only create PMD or PGD early mappings so we 291 * should never reach here with MMU disabled. 292 */ 293 BUG(); 294 } 295 296 static inline phys_addr_t __init alloc_pte_fixmap(uintptr_t va) 297 { 298 return memblock_phys_alloc(PAGE_SIZE, PAGE_SIZE); 299 } 300 301 static phys_addr_t __init alloc_pte_late(uintptr_t va) 302 { 303 unsigned long vaddr; 304 305 vaddr = __get_free_page(GFP_KERNEL); 306 BUG_ON(!vaddr || !pgtable_pte_page_ctor(virt_to_page(vaddr))); 307 308 return __pa(vaddr); 309 } 310 311 static void __init create_pte_mapping(pte_t *ptep, 312 uintptr_t va, phys_addr_t pa, 313 phys_addr_t sz, pgprot_t prot) 314 { 315 uintptr_t pte_idx = pte_index(va); 316 317 BUG_ON(sz != PAGE_SIZE); 318 319 if (pte_none(ptep[pte_idx])) 320 ptep[pte_idx] = pfn_pte(PFN_DOWN(pa), prot); 321 } 322 323 #ifndef __PAGETABLE_PMD_FOLDED 324 325 static pmd_t trampoline_pmd[PTRS_PER_PMD] __page_aligned_bss; 326 static pmd_t fixmap_pmd[PTRS_PER_PMD] __page_aligned_bss; 327 static pmd_t early_pmd[PTRS_PER_PMD] __initdata __aligned(PAGE_SIZE); 328 329 #ifdef CONFIG_XIP_KERNEL 330 #define trampoline_pmd ((pmd_t *)XIP_FIXUP(trampoline_pmd)) 331 #define fixmap_pmd ((pmd_t *)XIP_FIXUP(fixmap_pmd)) 332 #define early_pmd ((pmd_t *)XIP_FIXUP(early_pmd)) 333 #endif /* CONFIG_XIP_KERNEL */ 334 335 static p4d_t trampoline_p4d[PTRS_PER_P4D] __page_aligned_bss; 336 static p4d_t fixmap_p4d[PTRS_PER_P4D] __page_aligned_bss; 337 static p4d_t early_p4d[PTRS_PER_P4D] __initdata __aligned(PAGE_SIZE); 338 339 #ifdef CONFIG_XIP_KERNEL 340 #define trampoline_p4d ((p4d_t *)XIP_FIXUP(trampoline_p4d)) 341 #define fixmap_p4d ((p4d_t *)XIP_FIXUP(fixmap_p4d)) 342 #define early_p4d ((p4d_t *)XIP_FIXUP(early_p4d)) 343 #endif /* CONFIG_XIP_KERNEL */ 344 345 static pud_t trampoline_pud[PTRS_PER_PUD] __page_aligned_bss; 346 static pud_t fixmap_pud[PTRS_PER_PUD] __page_aligned_bss; 347 static pud_t early_pud[PTRS_PER_PUD] __initdata __aligned(PAGE_SIZE); 348 349 #ifdef CONFIG_XIP_KERNEL 350 #define trampoline_pud ((pud_t *)XIP_FIXUP(trampoline_pud)) 351 #define fixmap_pud ((pud_t *)XIP_FIXUP(fixmap_pud)) 352 #define early_pud ((pud_t *)XIP_FIXUP(early_pud)) 353 #endif /* CONFIG_XIP_KERNEL */ 354 355 static pmd_t *__init get_pmd_virt_early(phys_addr_t pa) 356 { 357 /* Before MMU is enabled */ 358 return (pmd_t *)((uintptr_t)pa); 359 } 360 361 static pmd_t *__init get_pmd_virt_fixmap(phys_addr_t pa) 362 { 363 clear_fixmap(FIX_PMD); 364 return (pmd_t *)set_fixmap_offset(FIX_PMD, pa); 365 } 366 367 static pmd_t *__init get_pmd_virt_late(phys_addr_t pa) 368 { 369 return (pmd_t *) __va(pa); 370 } 371 372 static phys_addr_t __init alloc_pmd_early(uintptr_t va) 373 { 374 BUG_ON((va - kernel_map.virt_addr) >> PUD_SHIFT); 375 376 return (uintptr_t)early_pmd; 377 } 378 379 static phys_addr_t __init alloc_pmd_fixmap(uintptr_t va) 380 { 381 return memblock_phys_alloc(PAGE_SIZE, PAGE_SIZE); 382 } 383 384 static phys_addr_t __init alloc_pmd_late(uintptr_t va) 385 { 386 unsigned long vaddr; 387 388 vaddr = __get_free_page(GFP_KERNEL); 389 BUG_ON(!vaddr || !pgtable_pmd_page_ctor(virt_to_page(vaddr))); 390 391 return __pa(vaddr); 392 } 393 394 static void __init create_pmd_mapping(pmd_t *pmdp, 395 uintptr_t va, phys_addr_t pa, 396 phys_addr_t sz, pgprot_t prot) 397 { 398 pte_t *ptep; 399 phys_addr_t pte_phys; 400 uintptr_t pmd_idx = pmd_index(va); 401 402 if (sz == PMD_SIZE) { 403 if (pmd_none(pmdp[pmd_idx])) 404 pmdp[pmd_idx] = pfn_pmd(PFN_DOWN(pa), prot); 405 return; 406 } 407 408 if (pmd_none(pmdp[pmd_idx])) { 409 pte_phys = pt_ops.alloc_pte(va); 410 pmdp[pmd_idx] = pfn_pmd(PFN_DOWN(pte_phys), PAGE_TABLE); 411 ptep = pt_ops.get_pte_virt(pte_phys); 412 memset(ptep, 0, PAGE_SIZE); 413 } else { 414 pte_phys = PFN_PHYS(_pmd_pfn(pmdp[pmd_idx])); 415 ptep = pt_ops.get_pte_virt(pte_phys); 416 } 417 418 create_pte_mapping(ptep, va, pa, sz, prot); 419 } 420 421 static pud_t *__init get_pud_virt_early(phys_addr_t pa) 422 { 423 return (pud_t *)((uintptr_t)pa); 424 } 425 426 static pud_t *__init get_pud_virt_fixmap(phys_addr_t pa) 427 { 428 clear_fixmap(FIX_PUD); 429 return (pud_t *)set_fixmap_offset(FIX_PUD, pa); 430 } 431 432 static pud_t *__init get_pud_virt_late(phys_addr_t pa) 433 { 434 return (pud_t *)__va(pa); 435 } 436 437 static phys_addr_t __init alloc_pud_early(uintptr_t va) 438 { 439 /* Only one PUD is available for early mapping */ 440 BUG_ON((va - kernel_map.virt_addr) >> PGDIR_SHIFT); 441 442 return (uintptr_t)early_pud; 443 } 444 445 static phys_addr_t __init alloc_pud_fixmap(uintptr_t va) 446 { 447 return memblock_phys_alloc(PAGE_SIZE, PAGE_SIZE); 448 } 449 450 static phys_addr_t alloc_pud_late(uintptr_t va) 451 { 452 unsigned long vaddr; 453 454 vaddr = __get_free_page(GFP_KERNEL); 455 BUG_ON(!vaddr); 456 return __pa(vaddr); 457 } 458 459 static p4d_t *__init get_p4d_virt_early(phys_addr_t pa) 460 { 461 return (p4d_t *)((uintptr_t)pa); 462 } 463 464 static p4d_t *__init get_p4d_virt_fixmap(phys_addr_t pa) 465 { 466 clear_fixmap(FIX_P4D); 467 return (p4d_t *)set_fixmap_offset(FIX_P4D, pa); 468 } 469 470 static p4d_t *__init get_p4d_virt_late(phys_addr_t pa) 471 { 472 return (p4d_t *)__va(pa); 473 } 474 475 static phys_addr_t __init alloc_p4d_early(uintptr_t va) 476 { 477 /* Only one P4D is available for early mapping */ 478 BUG_ON((va - kernel_map.virt_addr) >> PGDIR_SHIFT); 479 480 return (uintptr_t)early_p4d; 481 } 482 483 static phys_addr_t __init alloc_p4d_fixmap(uintptr_t va) 484 { 485 return memblock_phys_alloc(PAGE_SIZE, PAGE_SIZE); 486 } 487 488 static phys_addr_t alloc_p4d_late(uintptr_t va) 489 { 490 unsigned long vaddr; 491 492 vaddr = __get_free_page(GFP_KERNEL); 493 BUG_ON(!vaddr); 494 return __pa(vaddr); 495 } 496 497 static void __init create_pud_mapping(pud_t *pudp, 498 uintptr_t va, phys_addr_t pa, 499 phys_addr_t sz, pgprot_t prot) 500 { 501 pmd_t *nextp; 502 phys_addr_t next_phys; 503 uintptr_t pud_index = pud_index(va); 504 505 if (sz == PUD_SIZE) { 506 if (pud_val(pudp[pud_index]) == 0) 507 pudp[pud_index] = pfn_pud(PFN_DOWN(pa), prot); 508 return; 509 } 510 511 if (pud_val(pudp[pud_index]) == 0) { 512 next_phys = pt_ops.alloc_pmd(va); 513 pudp[pud_index] = pfn_pud(PFN_DOWN(next_phys), PAGE_TABLE); 514 nextp = pt_ops.get_pmd_virt(next_phys); 515 memset(nextp, 0, PAGE_SIZE); 516 } else { 517 next_phys = PFN_PHYS(_pud_pfn(pudp[pud_index])); 518 nextp = pt_ops.get_pmd_virt(next_phys); 519 } 520 521 create_pmd_mapping(nextp, va, pa, sz, prot); 522 } 523 524 static void __init create_p4d_mapping(p4d_t *p4dp, 525 uintptr_t va, phys_addr_t pa, 526 phys_addr_t sz, pgprot_t prot) 527 { 528 pud_t *nextp; 529 phys_addr_t next_phys; 530 uintptr_t p4d_index = p4d_index(va); 531 532 if (sz == P4D_SIZE) { 533 if (p4d_val(p4dp[p4d_index]) == 0) 534 p4dp[p4d_index] = pfn_p4d(PFN_DOWN(pa), prot); 535 return; 536 } 537 538 if (p4d_val(p4dp[p4d_index]) == 0) { 539 next_phys = pt_ops.alloc_pud(va); 540 p4dp[p4d_index] = pfn_p4d(PFN_DOWN(next_phys), PAGE_TABLE); 541 nextp = pt_ops.get_pud_virt(next_phys); 542 memset(nextp, 0, PAGE_SIZE); 543 } else { 544 next_phys = PFN_PHYS(_p4d_pfn(p4dp[p4d_index])); 545 nextp = pt_ops.get_pud_virt(next_phys); 546 } 547 548 create_pud_mapping(nextp, va, pa, sz, prot); 549 } 550 551 #define pgd_next_t p4d_t 552 #define alloc_pgd_next(__va) (pgtable_l5_enabled ? \ 553 pt_ops.alloc_p4d(__va) : (pgtable_l4_enabled ? \ 554 pt_ops.alloc_pud(__va) : pt_ops.alloc_pmd(__va))) 555 #define get_pgd_next_virt(__pa) (pgtable_l5_enabled ? \ 556 pt_ops.get_p4d_virt(__pa) : (pgd_next_t *)(pgtable_l4_enabled ? \ 557 pt_ops.get_pud_virt(__pa) : (pud_t *)pt_ops.get_pmd_virt(__pa))) 558 #define create_pgd_next_mapping(__nextp, __va, __pa, __sz, __prot) \ 559 (pgtable_l5_enabled ? \ 560 create_p4d_mapping(__nextp, __va, __pa, __sz, __prot) : \ 561 (pgtable_l4_enabled ? \ 562 create_pud_mapping((pud_t *)__nextp, __va, __pa, __sz, __prot) : \ 563 create_pmd_mapping((pmd_t *)__nextp, __va, __pa, __sz, __prot))) 564 #define fixmap_pgd_next (pgtable_l5_enabled ? \ 565 (uintptr_t)fixmap_p4d : (pgtable_l4_enabled ? \ 566 (uintptr_t)fixmap_pud : (uintptr_t)fixmap_pmd)) 567 #define trampoline_pgd_next (pgtable_l5_enabled ? \ 568 (uintptr_t)trampoline_p4d : (pgtable_l4_enabled ? \ 569 (uintptr_t)trampoline_pud : (uintptr_t)trampoline_pmd)) 570 #define early_dtb_pgd_next (pgtable_l5_enabled ? \ 571 (uintptr_t)early_dtb_p4d : (pgtable_l4_enabled ? \ 572 (uintptr_t)early_dtb_pud : (uintptr_t)early_dtb_pmd)) 573 #else 574 #define pgd_next_t pte_t 575 #define alloc_pgd_next(__va) pt_ops.alloc_pte(__va) 576 #define get_pgd_next_virt(__pa) pt_ops.get_pte_virt(__pa) 577 #define create_pgd_next_mapping(__nextp, __va, __pa, __sz, __prot) \ 578 create_pte_mapping(__nextp, __va, __pa, __sz, __prot) 579 #define fixmap_pgd_next ((uintptr_t)fixmap_pte) 580 #define early_dtb_pgd_next ((uintptr_t)early_dtb_pmd) 581 #define create_p4d_mapping(__pmdp, __va, __pa, __sz, __prot) do {} while(0) 582 #define create_pud_mapping(__pmdp, __va, __pa, __sz, __prot) do {} while(0) 583 #define create_pmd_mapping(__pmdp, __va, __pa, __sz, __prot) do {} while(0) 584 #endif /* __PAGETABLE_PMD_FOLDED */ 585 586 void __init create_pgd_mapping(pgd_t *pgdp, 587 uintptr_t va, phys_addr_t pa, 588 phys_addr_t sz, pgprot_t prot) 589 { 590 pgd_next_t *nextp; 591 phys_addr_t next_phys; 592 uintptr_t pgd_idx = pgd_index(va); 593 594 if (sz == PGDIR_SIZE) { 595 if (pgd_val(pgdp[pgd_idx]) == 0) 596 pgdp[pgd_idx] = pfn_pgd(PFN_DOWN(pa), prot); 597 return; 598 } 599 600 if (pgd_val(pgdp[pgd_idx]) == 0) { 601 next_phys = alloc_pgd_next(va); 602 pgdp[pgd_idx] = pfn_pgd(PFN_DOWN(next_phys), PAGE_TABLE); 603 nextp = get_pgd_next_virt(next_phys); 604 memset(nextp, 0, PAGE_SIZE); 605 } else { 606 next_phys = PFN_PHYS(_pgd_pfn(pgdp[pgd_idx])); 607 nextp = get_pgd_next_virt(next_phys); 608 } 609 610 create_pgd_next_mapping(nextp, va, pa, sz, prot); 611 } 612 613 static uintptr_t __init best_map_size(phys_addr_t base, phys_addr_t size) 614 { 615 /* Upgrade to PMD_SIZE mappings whenever possible */ 616 if ((base & (PMD_SIZE - 1)) || (size & (PMD_SIZE - 1))) 617 return PAGE_SIZE; 618 619 return PMD_SIZE; 620 } 621 622 #ifdef CONFIG_XIP_KERNEL 623 #define phys_ram_base (*(phys_addr_t *)XIP_FIXUP(&phys_ram_base)) 624 extern char _xiprom[], _exiprom[], __data_loc; 625 626 /* called from head.S with MMU off */ 627 asmlinkage void __init __copy_data(void) 628 { 629 void *from = (void *)(&__data_loc); 630 void *to = (void *)CONFIG_PHYS_RAM_BASE; 631 size_t sz = (size_t)((uintptr_t)(&_end) - (uintptr_t)(&_sdata)); 632 633 memcpy(to, from, sz); 634 } 635 #endif 636 637 #ifdef CONFIG_STRICT_KERNEL_RWX 638 static __init pgprot_t pgprot_from_va(uintptr_t va) 639 { 640 if (is_va_kernel_text(va)) 641 return PAGE_KERNEL_READ_EXEC; 642 643 /* 644 * In 64-bit kernel, the kernel mapping is outside the linear mapping so 645 * we must protect its linear mapping alias from being executed and 646 * written. 647 * And rodata section is marked readonly in mark_rodata_ro. 648 */ 649 if (IS_ENABLED(CONFIG_64BIT) && is_va_kernel_lm_alias_text(va)) 650 return PAGE_KERNEL_READ; 651 652 return PAGE_KERNEL; 653 } 654 655 void mark_rodata_ro(void) 656 { 657 set_kernel_memory(__start_rodata, _data, set_memory_ro); 658 if (IS_ENABLED(CONFIG_64BIT)) 659 set_kernel_memory(lm_alias(__start_rodata), lm_alias(_data), 660 set_memory_ro); 661 662 debug_checkwx(); 663 } 664 #else 665 static __init pgprot_t pgprot_from_va(uintptr_t va) 666 { 667 if (IS_ENABLED(CONFIG_64BIT) && !is_kernel_mapping(va)) 668 return PAGE_KERNEL; 669 670 return PAGE_KERNEL_EXEC; 671 } 672 #endif /* CONFIG_STRICT_KERNEL_RWX */ 673 674 #if defined(CONFIG_64BIT) && !defined(CONFIG_XIP_KERNEL) 675 static void __init disable_pgtable_l5(void) 676 { 677 pgtable_l5_enabled = false; 678 kernel_map.page_offset = PAGE_OFFSET_L4; 679 satp_mode = SATP_MODE_48; 680 } 681 682 static void __init disable_pgtable_l4(void) 683 { 684 pgtable_l4_enabled = false; 685 kernel_map.page_offset = PAGE_OFFSET_L3; 686 satp_mode = SATP_MODE_39; 687 } 688 689 /* 690 * There is a simple way to determine if 4-level is supported by the 691 * underlying hardware: establish 1:1 mapping in 4-level page table mode 692 * then read SATP to see if the configuration was taken into account 693 * meaning sv48 is supported. 694 */ 695 static __init void set_satp_mode(void) 696 { 697 u64 identity_satp, hw_satp; 698 uintptr_t set_satp_mode_pmd = ((unsigned long)set_satp_mode) & PMD_MASK; 699 bool check_l4 = false; 700 701 create_p4d_mapping(early_p4d, 702 set_satp_mode_pmd, (uintptr_t)early_pud, 703 P4D_SIZE, PAGE_TABLE); 704 create_pud_mapping(early_pud, 705 set_satp_mode_pmd, (uintptr_t)early_pmd, 706 PUD_SIZE, PAGE_TABLE); 707 /* Handle the case where set_satp_mode straddles 2 PMDs */ 708 create_pmd_mapping(early_pmd, 709 set_satp_mode_pmd, set_satp_mode_pmd, 710 PMD_SIZE, PAGE_KERNEL_EXEC); 711 create_pmd_mapping(early_pmd, 712 set_satp_mode_pmd + PMD_SIZE, 713 set_satp_mode_pmd + PMD_SIZE, 714 PMD_SIZE, PAGE_KERNEL_EXEC); 715 retry: 716 create_pgd_mapping(early_pg_dir, 717 set_satp_mode_pmd, 718 check_l4 ? (uintptr_t)early_pud : (uintptr_t)early_p4d, 719 PGDIR_SIZE, PAGE_TABLE); 720 721 identity_satp = PFN_DOWN((uintptr_t)&early_pg_dir) | satp_mode; 722 723 local_flush_tlb_all(); 724 csr_write(CSR_SATP, identity_satp); 725 hw_satp = csr_swap(CSR_SATP, 0ULL); 726 local_flush_tlb_all(); 727 728 if (hw_satp != identity_satp) { 729 if (!check_l4) { 730 disable_pgtable_l5(); 731 check_l4 = true; 732 memset(early_pg_dir, 0, PAGE_SIZE); 733 goto retry; 734 } 735 disable_pgtable_l4(); 736 } 737 738 memset(early_pg_dir, 0, PAGE_SIZE); 739 memset(early_p4d, 0, PAGE_SIZE); 740 memset(early_pud, 0, PAGE_SIZE); 741 memset(early_pmd, 0, PAGE_SIZE); 742 } 743 #endif 744 745 /* 746 * setup_vm() is called from head.S with MMU-off. 747 * 748 * Following requirements should be honoured for setup_vm() to work 749 * correctly: 750 * 1) It should use PC-relative addressing for accessing kernel symbols. 751 * To achieve this we always use GCC cmodel=medany. 752 * 2) The compiler instrumentation for FTRACE will not work for setup_vm() 753 * so disable compiler instrumentation when FTRACE is enabled. 754 * 755 * Currently, the above requirements are honoured by using custom CFLAGS 756 * for init.o in mm/Makefile. 757 */ 758 759 #ifndef __riscv_cmodel_medany 760 #error "setup_vm() is called from head.S before relocate so it should not use absolute addressing." 761 #endif 762 763 #ifdef CONFIG_XIP_KERNEL 764 static void __init create_kernel_page_table(pgd_t *pgdir, 765 __always_unused bool early) 766 { 767 uintptr_t va, end_va; 768 769 /* Map the flash resident part */ 770 end_va = kernel_map.virt_addr + kernel_map.xiprom_sz; 771 for (va = kernel_map.virt_addr; va < end_va; va += PMD_SIZE) 772 create_pgd_mapping(pgdir, va, 773 kernel_map.xiprom + (va - kernel_map.virt_addr), 774 PMD_SIZE, PAGE_KERNEL_EXEC); 775 776 /* Map the data in RAM */ 777 end_va = kernel_map.virt_addr + XIP_OFFSET + kernel_map.size; 778 for (va = kernel_map.virt_addr + XIP_OFFSET; va < end_va; va += PMD_SIZE) 779 create_pgd_mapping(pgdir, va, 780 kernel_map.phys_addr + (va - (kernel_map.virt_addr + XIP_OFFSET)), 781 PMD_SIZE, PAGE_KERNEL); 782 } 783 #else 784 static void __init create_kernel_page_table(pgd_t *pgdir, bool early) 785 { 786 uintptr_t va, end_va; 787 788 end_va = kernel_map.virt_addr + kernel_map.size; 789 for (va = kernel_map.virt_addr; va < end_va; va += PMD_SIZE) 790 create_pgd_mapping(pgdir, va, 791 kernel_map.phys_addr + (va - kernel_map.virt_addr), 792 PMD_SIZE, 793 early ? 794 PAGE_KERNEL_EXEC : pgprot_from_va(va)); 795 } 796 #endif 797 798 /* 799 * Setup a 4MB mapping that encompasses the device tree: for 64-bit kernel, 800 * this means 2 PMD entries whereas for 32-bit kernel, this is only 1 PGDIR 801 * entry. 802 */ 803 static void __init create_fdt_early_page_table(pgd_t *pgdir, uintptr_t dtb_pa) 804 { 805 #ifndef CONFIG_BUILTIN_DTB 806 uintptr_t pa = dtb_pa & ~(PMD_SIZE - 1); 807 808 create_pgd_mapping(early_pg_dir, DTB_EARLY_BASE_VA, 809 IS_ENABLED(CONFIG_64BIT) ? early_dtb_pgd_next : pa, 810 PGDIR_SIZE, 811 IS_ENABLED(CONFIG_64BIT) ? PAGE_TABLE : PAGE_KERNEL); 812 813 if (pgtable_l5_enabled) 814 create_p4d_mapping(early_dtb_p4d, DTB_EARLY_BASE_VA, 815 (uintptr_t)early_dtb_pud, P4D_SIZE, PAGE_TABLE); 816 817 if (pgtable_l4_enabled) 818 create_pud_mapping(early_dtb_pud, DTB_EARLY_BASE_VA, 819 (uintptr_t)early_dtb_pmd, PUD_SIZE, PAGE_TABLE); 820 821 if (IS_ENABLED(CONFIG_64BIT)) { 822 create_pmd_mapping(early_dtb_pmd, DTB_EARLY_BASE_VA, 823 pa, PMD_SIZE, PAGE_KERNEL); 824 create_pmd_mapping(early_dtb_pmd, DTB_EARLY_BASE_VA + PMD_SIZE, 825 pa + PMD_SIZE, PMD_SIZE, PAGE_KERNEL); 826 } 827 828 dtb_early_va = (void *)DTB_EARLY_BASE_VA + (dtb_pa & (PMD_SIZE - 1)); 829 #else 830 /* 831 * For 64-bit kernel, __va can't be used since it would return a linear 832 * mapping address whereas dtb_early_va will be used before 833 * setup_vm_final installs the linear mapping. For 32-bit kernel, as the 834 * kernel is mapped in the linear mapping, that makes no difference. 835 */ 836 dtb_early_va = kernel_mapping_pa_to_va(XIP_FIXUP(dtb_pa)); 837 #endif 838 839 dtb_early_pa = dtb_pa; 840 } 841 842 /* 843 * MMU is not enabled, the page tables are allocated directly using 844 * early_pmd/pud/p4d and the address returned is the physical one. 845 */ 846 void __init pt_ops_set_early(void) 847 { 848 pt_ops.alloc_pte = alloc_pte_early; 849 pt_ops.get_pte_virt = get_pte_virt_early; 850 #ifndef __PAGETABLE_PMD_FOLDED 851 pt_ops.alloc_pmd = alloc_pmd_early; 852 pt_ops.get_pmd_virt = get_pmd_virt_early; 853 pt_ops.alloc_pud = alloc_pud_early; 854 pt_ops.get_pud_virt = get_pud_virt_early; 855 pt_ops.alloc_p4d = alloc_p4d_early; 856 pt_ops.get_p4d_virt = get_p4d_virt_early; 857 #endif 858 } 859 860 /* 861 * MMU is enabled but page table setup is not complete yet. 862 * fixmap page table alloc functions must be used as a means to temporarily 863 * map the allocated physical pages since the linear mapping does not exist yet. 864 * 865 * Note that this is called with MMU disabled, hence kernel_mapping_pa_to_va, 866 * but it will be used as described above. 867 */ 868 void __init pt_ops_set_fixmap(void) 869 { 870 pt_ops.alloc_pte = kernel_mapping_pa_to_va((uintptr_t)alloc_pte_fixmap); 871 pt_ops.get_pte_virt = kernel_mapping_pa_to_va((uintptr_t)get_pte_virt_fixmap); 872 #ifndef __PAGETABLE_PMD_FOLDED 873 pt_ops.alloc_pmd = kernel_mapping_pa_to_va((uintptr_t)alloc_pmd_fixmap); 874 pt_ops.get_pmd_virt = kernel_mapping_pa_to_va((uintptr_t)get_pmd_virt_fixmap); 875 pt_ops.alloc_pud = kernel_mapping_pa_to_va((uintptr_t)alloc_pud_fixmap); 876 pt_ops.get_pud_virt = kernel_mapping_pa_to_va((uintptr_t)get_pud_virt_fixmap); 877 pt_ops.alloc_p4d = kernel_mapping_pa_to_va((uintptr_t)alloc_p4d_fixmap); 878 pt_ops.get_p4d_virt = kernel_mapping_pa_to_va((uintptr_t)get_p4d_virt_fixmap); 879 #endif 880 } 881 882 /* 883 * MMU is enabled and page table setup is complete, so from now, we can use 884 * generic page allocation functions to setup page table. 885 */ 886 void __init pt_ops_set_late(void) 887 { 888 pt_ops.alloc_pte = alloc_pte_late; 889 pt_ops.get_pte_virt = get_pte_virt_late; 890 #ifndef __PAGETABLE_PMD_FOLDED 891 pt_ops.alloc_pmd = alloc_pmd_late; 892 pt_ops.get_pmd_virt = get_pmd_virt_late; 893 pt_ops.alloc_pud = alloc_pud_late; 894 pt_ops.get_pud_virt = get_pud_virt_late; 895 pt_ops.alloc_p4d = alloc_p4d_late; 896 pt_ops.get_p4d_virt = get_p4d_virt_late; 897 #endif 898 } 899 900 asmlinkage void __init setup_vm(uintptr_t dtb_pa) 901 { 902 pmd_t __maybe_unused fix_bmap_spmd, fix_bmap_epmd; 903 904 kernel_map.virt_addr = KERNEL_LINK_ADDR; 905 kernel_map.page_offset = _AC(CONFIG_PAGE_OFFSET, UL); 906 907 #ifdef CONFIG_XIP_KERNEL 908 kernel_map.xiprom = (uintptr_t)CONFIG_XIP_PHYS_ADDR; 909 kernel_map.xiprom_sz = (uintptr_t)(&_exiprom) - (uintptr_t)(&_xiprom); 910 911 phys_ram_base = CONFIG_PHYS_RAM_BASE; 912 kernel_map.phys_addr = (uintptr_t)CONFIG_PHYS_RAM_BASE; 913 kernel_map.size = (uintptr_t)(&_end) - (uintptr_t)(&_sdata); 914 915 kernel_map.va_kernel_xip_pa_offset = kernel_map.virt_addr - kernel_map.xiprom; 916 #else 917 kernel_map.phys_addr = (uintptr_t)(&_start); 918 kernel_map.size = (uintptr_t)(&_end) - kernel_map.phys_addr; 919 #endif 920 921 #if defined(CONFIG_64BIT) && !defined(CONFIG_XIP_KERNEL) 922 set_satp_mode(); 923 #endif 924 925 kernel_map.va_pa_offset = PAGE_OFFSET - kernel_map.phys_addr; 926 kernel_map.va_kernel_pa_offset = kernel_map.virt_addr - kernel_map.phys_addr; 927 928 riscv_pfn_base = PFN_DOWN(kernel_map.phys_addr); 929 930 /* 931 * The default maximal physical memory size is KERN_VIRT_SIZE for 32-bit 932 * kernel, whereas for 64-bit kernel, the end of the virtual address 933 * space is occupied by the modules/BPF/kernel mappings which reduces 934 * the available size of the linear mapping. 935 */ 936 memory_limit = KERN_VIRT_SIZE - (IS_ENABLED(CONFIG_64BIT) ? SZ_4G : 0); 937 938 /* Sanity check alignment and size */ 939 BUG_ON((PAGE_OFFSET % PGDIR_SIZE) != 0); 940 BUG_ON((kernel_map.phys_addr % PMD_SIZE) != 0); 941 942 #ifdef CONFIG_64BIT 943 /* 944 * The last 4K bytes of the addressable memory can not be mapped because 945 * of IS_ERR_VALUE macro. 946 */ 947 BUG_ON((kernel_map.virt_addr + kernel_map.size) > ADDRESS_SPACE_END - SZ_4K); 948 #endif 949 950 apply_early_boot_alternatives(); 951 pt_ops_set_early(); 952 953 /* Setup early PGD for fixmap */ 954 create_pgd_mapping(early_pg_dir, FIXADDR_START, 955 fixmap_pgd_next, PGDIR_SIZE, PAGE_TABLE); 956 957 #ifndef __PAGETABLE_PMD_FOLDED 958 /* Setup fixmap P4D and PUD */ 959 if (pgtable_l5_enabled) 960 create_p4d_mapping(fixmap_p4d, FIXADDR_START, 961 (uintptr_t)fixmap_pud, P4D_SIZE, PAGE_TABLE); 962 /* Setup fixmap PUD and PMD */ 963 if (pgtable_l4_enabled) 964 create_pud_mapping(fixmap_pud, FIXADDR_START, 965 (uintptr_t)fixmap_pmd, PUD_SIZE, PAGE_TABLE); 966 create_pmd_mapping(fixmap_pmd, FIXADDR_START, 967 (uintptr_t)fixmap_pte, PMD_SIZE, PAGE_TABLE); 968 /* Setup trampoline PGD and PMD */ 969 create_pgd_mapping(trampoline_pg_dir, kernel_map.virt_addr, 970 trampoline_pgd_next, PGDIR_SIZE, PAGE_TABLE); 971 if (pgtable_l5_enabled) 972 create_p4d_mapping(trampoline_p4d, kernel_map.virt_addr, 973 (uintptr_t)trampoline_pud, P4D_SIZE, PAGE_TABLE); 974 if (pgtable_l4_enabled) 975 create_pud_mapping(trampoline_pud, kernel_map.virt_addr, 976 (uintptr_t)trampoline_pmd, PUD_SIZE, PAGE_TABLE); 977 #ifdef CONFIG_XIP_KERNEL 978 create_pmd_mapping(trampoline_pmd, kernel_map.virt_addr, 979 kernel_map.xiprom, PMD_SIZE, PAGE_KERNEL_EXEC); 980 #else 981 create_pmd_mapping(trampoline_pmd, kernel_map.virt_addr, 982 kernel_map.phys_addr, PMD_SIZE, PAGE_KERNEL_EXEC); 983 #endif 984 #else 985 /* Setup trampoline PGD */ 986 create_pgd_mapping(trampoline_pg_dir, kernel_map.virt_addr, 987 kernel_map.phys_addr, PGDIR_SIZE, PAGE_KERNEL_EXEC); 988 #endif 989 990 /* 991 * Setup early PGD covering entire kernel which will allow 992 * us to reach paging_init(). We map all memory banks later 993 * in setup_vm_final() below. 994 */ 995 create_kernel_page_table(early_pg_dir, true); 996 997 /* Setup early mapping for FDT early scan */ 998 create_fdt_early_page_table(early_pg_dir, dtb_pa); 999 1000 /* 1001 * Bootime fixmap only can handle PMD_SIZE mapping. Thus, boot-ioremap 1002 * range can not span multiple pmds. 1003 */ 1004 BUG_ON((__fix_to_virt(FIX_BTMAP_BEGIN) >> PMD_SHIFT) 1005 != (__fix_to_virt(FIX_BTMAP_END) >> PMD_SHIFT)); 1006 1007 #ifndef __PAGETABLE_PMD_FOLDED 1008 /* 1009 * Early ioremap fixmap is already created as it lies within first 2MB 1010 * of fixmap region. We always map PMD_SIZE. Thus, both FIX_BTMAP_END 1011 * FIX_BTMAP_BEGIN should lie in the same pmd. Verify that and warn 1012 * the user if not. 1013 */ 1014 fix_bmap_spmd = fixmap_pmd[pmd_index(__fix_to_virt(FIX_BTMAP_BEGIN))]; 1015 fix_bmap_epmd = fixmap_pmd[pmd_index(__fix_to_virt(FIX_BTMAP_END))]; 1016 if (pmd_val(fix_bmap_spmd) != pmd_val(fix_bmap_epmd)) { 1017 WARN_ON(1); 1018 pr_warn("fixmap btmap start [%08lx] != end [%08lx]\n", 1019 pmd_val(fix_bmap_spmd), pmd_val(fix_bmap_epmd)); 1020 pr_warn("fix_to_virt(FIX_BTMAP_BEGIN): %08lx\n", 1021 fix_to_virt(FIX_BTMAP_BEGIN)); 1022 pr_warn("fix_to_virt(FIX_BTMAP_END): %08lx\n", 1023 fix_to_virt(FIX_BTMAP_END)); 1024 1025 pr_warn("FIX_BTMAP_END: %d\n", FIX_BTMAP_END); 1026 pr_warn("FIX_BTMAP_BEGIN: %d\n", FIX_BTMAP_BEGIN); 1027 } 1028 #endif 1029 1030 pt_ops_set_fixmap(); 1031 } 1032 1033 static void __init setup_vm_final(void) 1034 { 1035 uintptr_t va, map_size; 1036 phys_addr_t pa, start, end; 1037 u64 i; 1038 1039 /* Setup swapper PGD for fixmap */ 1040 create_pgd_mapping(swapper_pg_dir, FIXADDR_START, 1041 __pa_symbol(fixmap_pgd_next), 1042 PGDIR_SIZE, PAGE_TABLE); 1043 1044 /* Map all memory banks in the linear mapping */ 1045 for_each_mem_range(i, &start, &end) { 1046 if (start >= end) 1047 break; 1048 if (start <= __pa(PAGE_OFFSET) && 1049 __pa(PAGE_OFFSET) < end) 1050 start = __pa(PAGE_OFFSET); 1051 if (end >= __pa(PAGE_OFFSET) + memory_limit) 1052 end = __pa(PAGE_OFFSET) + memory_limit; 1053 1054 map_size = best_map_size(start, end - start); 1055 for (pa = start; pa < end; pa += map_size) { 1056 va = (uintptr_t)__va(pa); 1057 1058 create_pgd_mapping(swapper_pg_dir, va, pa, map_size, 1059 pgprot_from_va(va)); 1060 } 1061 } 1062 1063 /* Map the kernel */ 1064 if (IS_ENABLED(CONFIG_64BIT)) 1065 create_kernel_page_table(swapper_pg_dir, false); 1066 1067 #ifdef CONFIG_KASAN 1068 kasan_swapper_init(); 1069 #endif 1070 1071 /* Clear fixmap PTE and PMD mappings */ 1072 clear_fixmap(FIX_PTE); 1073 clear_fixmap(FIX_PMD); 1074 clear_fixmap(FIX_PUD); 1075 clear_fixmap(FIX_P4D); 1076 1077 /* Move to swapper page table */ 1078 csr_write(CSR_SATP, PFN_DOWN(__pa_symbol(swapper_pg_dir)) | satp_mode); 1079 local_flush_tlb_all(); 1080 1081 pt_ops_set_late(); 1082 } 1083 #else 1084 asmlinkage void __init setup_vm(uintptr_t dtb_pa) 1085 { 1086 dtb_early_va = (void *)dtb_pa; 1087 dtb_early_pa = dtb_pa; 1088 } 1089 1090 static inline void setup_vm_final(void) 1091 { 1092 } 1093 #endif /* CONFIG_MMU */ 1094 1095 /* 1096 * reserve_crashkernel() - reserves memory for crash kernel 1097 * 1098 * This function reserves memory area given in "crashkernel=" kernel command 1099 * line parameter. The memory reserved is used by dump capture kernel when 1100 * primary kernel is crashing. 1101 */ 1102 static void __init reserve_crashkernel(void) 1103 { 1104 unsigned long long crash_base = 0; 1105 unsigned long long crash_size = 0; 1106 unsigned long search_start = memblock_start_of_DRAM(); 1107 unsigned long search_end = memblock_end_of_DRAM(); 1108 1109 int ret = 0; 1110 1111 if (!IS_ENABLED(CONFIG_KEXEC_CORE)) 1112 return; 1113 /* 1114 * Don't reserve a region for a crash kernel on a crash kernel 1115 * since it doesn't make much sense and we have limited memory 1116 * resources. 1117 */ 1118 if (is_kdump_kernel()) { 1119 pr_info("crashkernel: ignoring reservation request\n"); 1120 return; 1121 } 1122 1123 ret = parse_crashkernel(boot_command_line, memblock_phys_mem_size(), 1124 &crash_size, &crash_base); 1125 if (ret || !crash_size) 1126 return; 1127 1128 crash_size = PAGE_ALIGN(crash_size); 1129 1130 if (crash_base) { 1131 search_start = crash_base; 1132 search_end = crash_base + crash_size; 1133 } 1134 1135 /* 1136 * Current riscv boot protocol requires 2MB alignment for 1137 * RV64 and 4MB alignment for RV32 (hugepage size) 1138 * 1139 * Try to alloc from 32bit addressible physical memory so that 1140 * swiotlb can work on the crash kernel. 1141 */ 1142 crash_base = memblock_phys_alloc_range(crash_size, PMD_SIZE, 1143 search_start, 1144 min(search_end, (unsigned long) SZ_4G)); 1145 if (crash_base == 0) { 1146 /* Try again without restricting region to 32bit addressible memory */ 1147 crash_base = memblock_phys_alloc_range(crash_size, PMD_SIZE, 1148 search_start, search_end); 1149 if (crash_base == 0) { 1150 pr_warn("crashkernel: couldn't allocate %lldKB\n", 1151 crash_size >> 10); 1152 return; 1153 } 1154 } 1155 1156 pr_info("crashkernel: reserved 0x%016llx - 0x%016llx (%lld MB)\n", 1157 crash_base, crash_base + crash_size, crash_size >> 20); 1158 1159 crashk_res.start = crash_base; 1160 crashk_res.end = crash_base + crash_size - 1; 1161 } 1162 1163 void __init paging_init(void) 1164 { 1165 setup_bootmem(); 1166 setup_vm_final(); 1167 } 1168 1169 void __init misc_mem_init(void) 1170 { 1171 early_memtest(min_low_pfn << PAGE_SHIFT, max_low_pfn << PAGE_SHIFT); 1172 arch_numa_init(); 1173 sparse_init(); 1174 zone_sizes_init(); 1175 reserve_crashkernel(); 1176 memblock_dump_all(); 1177 } 1178 1179 #ifdef CONFIG_SPARSEMEM_VMEMMAP 1180 int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node, 1181 struct vmem_altmap *altmap) 1182 { 1183 return vmemmap_populate_basepages(start, end, node, NULL); 1184 } 1185 #endif 1186