1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2012 Regents of the University of California 4 * Copyright (C) 2019 Western Digital Corporation or its affiliates. 5 * Copyright (C) 2020 FORTH-ICS/CARV 6 * Nick Kossifidis <mick@ics.forth.gr> 7 */ 8 9 #include <linux/init.h> 10 #include <linux/mm.h> 11 #include <linux/memblock.h> 12 #include <linux/initrd.h> 13 #include <linux/swap.h> 14 #include <linux/swiotlb.h> 15 #include <linux/sizes.h> 16 #include <linux/of_fdt.h> 17 #include <linux/of_reserved_mem.h> 18 #include <linux/libfdt.h> 19 #include <linux/set_memory.h> 20 #include <linux/dma-map-ops.h> 21 #include <linux/crash_dump.h> 22 #include <linux/hugetlb.h> 23 24 #include <asm/fixmap.h> 25 #include <asm/tlbflush.h> 26 #include <asm/sections.h> 27 #include <asm/soc.h> 28 #include <asm/io.h> 29 #include <asm/ptdump.h> 30 #include <asm/numa.h> 31 32 #include "../kernel/head.h" 33 34 struct kernel_mapping kernel_map __ro_after_init; 35 EXPORT_SYMBOL(kernel_map); 36 #ifdef CONFIG_XIP_KERNEL 37 #define kernel_map (*(struct kernel_mapping *)XIP_FIXUP(&kernel_map)) 38 #endif 39 40 #ifdef CONFIG_64BIT 41 u64 satp_mode __ro_after_init = !IS_ENABLED(CONFIG_XIP_KERNEL) ? SATP_MODE_57 : SATP_MODE_39; 42 #else 43 u64 satp_mode __ro_after_init = SATP_MODE_32; 44 #endif 45 EXPORT_SYMBOL(satp_mode); 46 47 bool pgtable_l4_enabled = IS_ENABLED(CONFIG_64BIT) && !IS_ENABLED(CONFIG_XIP_KERNEL); 48 bool pgtable_l5_enabled = IS_ENABLED(CONFIG_64BIT) && !IS_ENABLED(CONFIG_XIP_KERNEL); 49 EXPORT_SYMBOL(pgtable_l4_enabled); 50 EXPORT_SYMBOL(pgtable_l5_enabled); 51 52 phys_addr_t phys_ram_base __ro_after_init; 53 EXPORT_SYMBOL(phys_ram_base); 54 55 unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)] 56 __page_aligned_bss; 57 EXPORT_SYMBOL(empty_zero_page); 58 59 extern char _start[]; 60 void *_dtb_early_va __initdata; 61 uintptr_t _dtb_early_pa __initdata; 62 63 static phys_addr_t dma32_phys_limit __initdata; 64 65 static void __init zone_sizes_init(void) 66 { 67 unsigned long max_zone_pfns[MAX_NR_ZONES] = { 0, }; 68 69 #ifdef CONFIG_ZONE_DMA32 70 max_zone_pfns[ZONE_DMA32] = PFN_DOWN(dma32_phys_limit); 71 #endif 72 max_zone_pfns[ZONE_NORMAL] = max_low_pfn; 73 74 free_area_init(max_zone_pfns); 75 } 76 77 #if defined(CONFIG_MMU) && defined(CONFIG_DEBUG_VM) 78 79 #define LOG2_SZ_1K ilog2(SZ_1K) 80 #define LOG2_SZ_1M ilog2(SZ_1M) 81 #define LOG2_SZ_1G ilog2(SZ_1G) 82 #define LOG2_SZ_1T ilog2(SZ_1T) 83 84 static inline void print_mlk(char *name, unsigned long b, unsigned long t) 85 { 86 pr_notice("%12s : 0x%08lx - 0x%08lx (%4ld kB)\n", name, b, t, 87 (((t) - (b)) >> LOG2_SZ_1K)); 88 } 89 90 static inline void print_mlm(char *name, unsigned long b, unsigned long t) 91 { 92 pr_notice("%12s : 0x%08lx - 0x%08lx (%4ld MB)\n", name, b, t, 93 (((t) - (b)) >> LOG2_SZ_1M)); 94 } 95 96 static inline void print_mlg(char *name, unsigned long b, unsigned long t) 97 { 98 pr_notice("%12s : 0x%08lx - 0x%08lx (%4ld GB)\n", name, b, t, 99 (((t) - (b)) >> LOG2_SZ_1G)); 100 } 101 102 #ifdef CONFIG_64BIT 103 static inline void print_mlt(char *name, unsigned long b, unsigned long t) 104 { 105 pr_notice("%12s : 0x%08lx - 0x%08lx (%4ld TB)\n", name, b, t, 106 (((t) - (b)) >> LOG2_SZ_1T)); 107 } 108 #else 109 #define print_mlt(n, b, t) do {} while (0) 110 #endif 111 112 static inline void print_ml(char *name, unsigned long b, unsigned long t) 113 { 114 unsigned long diff = t - b; 115 116 if (IS_ENABLED(CONFIG_64BIT) && (diff >> LOG2_SZ_1T) >= 10) 117 print_mlt(name, b, t); 118 else if ((diff >> LOG2_SZ_1G) >= 10) 119 print_mlg(name, b, t); 120 else if ((diff >> LOG2_SZ_1M) >= 10) 121 print_mlm(name, b, t); 122 else 123 print_mlk(name, b, t); 124 } 125 126 static void __init print_vm_layout(void) 127 { 128 pr_notice("Virtual kernel memory layout:\n"); 129 print_ml("fixmap", (unsigned long)FIXADDR_START, 130 (unsigned long)FIXADDR_TOP); 131 print_ml("pci io", (unsigned long)PCI_IO_START, 132 (unsigned long)PCI_IO_END); 133 print_ml("vmemmap", (unsigned long)VMEMMAP_START, 134 (unsigned long)VMEMMAP_END); 135 print_ml("vmalloc", (unsigned long)VMALLOC_START, 136 (unsigned long)VMALLOC_END); 137 #ifdef CONFIG_64BIT 138 print_ml("modules", (unsigned long)MODULES_VADDR, 139 (unsigned long)MODULES_END); 140 #endif 141 print_ml("lowmem", (unsigned long)PAGE_OFFSET, 142 (unsigned long)high_memory); 143 if (IS_ENABLED(CONFIG_64BIT)) { 144 #ifdef CONFIG_KASAN 145 print_ml("kasan", KASAN_SHADOW_START, KASAN_SHADOW_END); 146 #endif 147 148 print_ml("kernel", (unsigned long)KERNEL_LINK_ADDR, 149 (unsigned long)ADDRESS_SPACE_END); 150 } 151 } 152 #else 153 static void print_vm_layout(void) { } 154 #endif /* CONFIG_DEBUG_VM */ 155 156 void __init mem_init(void) 157 { 158 #ifdef CONFIG_FLATMEM 159 BUG_ON(!mem_map); 160 #endif /* CONFIG_FLATMEM */ 161 162 swiotlb_init(max_pfn > PFN_DOWN(dma32_phys_limit), SWIOTLB_VERBOSE); 163 memblock_free_all(); 164 165 print_vm_layout(); 166 } 167 168 /* Limit the memory size via mem. */ 169 static phys_addr_t memory_limit; 170 171 static int __init early_mem(char *p) 172 { 173 u64 size; 174 175 if (!p) 176 return 1; 177 178 size = memparse(p, &p) & PAGE_MASK; 179 memory_limit = min_t(u64, size, memory_limit); 180 181 pr_notice("Memory limited to %lldMB\n", (u64)memory_limit >> 20); 182 183 return 0; 184 } 185 early_param("mem", early_mem); 186 187 static void __init setup_bootmem(void) 188 { 189 phys_addr_t vmlinux_end = __pa_symbol(&_end); 190 phys_addr_t max_mapped_addr; 191 phys_addr_t phys_ram_end, vmlinux_start; 192 193 if (IS_ENABLED(CONFIG_XIP_KERNEL)) 194 vmlinux_start = __pa_symbol(&_sdata); 195 else 196 vmlinux_start = __pa_symbol(&_start); 197 198 memblock_enforce_memory_limit(memory_limit); 199 200 /* 201 * Make sure we align the reservation on PMD_SIZE since we will 202 * map the kernel in the linear mapping as read-only: we do not want 203 * any allocation to happen between _end and the next pmd aligned page. 204 */ 205 if (IS_ENABLED(CONFIG_64BIT) && IS_ENABLED(CONFIG_STRICT_KERNEL_RWX)) 206 vmlinux_end = (vmlinux_end + PMD_SIZE - 1) & PMD_MASK; 207 /* 208 * Reserve from the start of the kernel to the end of the kernel 209 */ 210 memblock_reserve(vmlinux_start, vmlinux_end - vmlinux_start); 211 212 phys_ram_end = memblock_end_of_DRAM(); 213 if (!IS_ENABLED(CONFIG_XIP_KERNEL)) 214 phys_ram_base = memblock_start_of_DRAM(); 215 /* 216 * memblock allocator is not aware of the fact that last 4K bytes of 217 * the addressable memory can not be mapped because of IS_ERR_VALUE 218 * macro. Make sure that last 4k bytes are not usable by memblock 219 * if end of dram is equal to maximum addressable memory. For 64-bit 220 * kernel, this problem can't happen here as the end of the virtual 221 * address space is occupied by the kernel mapping then this check must 222 * be done as soon as the kernel mapping base address is determined. 223 */ 224 if (!IS_ENABLED(CONFIG_64BIT)) { 225 max_mapped_addr = __pa(~(ulong)0); 226 if (max_mapped_addr == (phys_ram_end - 1)) 227 memblock_set_current_limit(max_mapped_addr - 4096); 228 } 229 230 min_low_pfn = PFN_UP(phys_ram_base); 231 max_low_pfn = max_pfn = PFN_DOWN(phys_ram_end); 232 high_memory = (void *)(__va(PFN_PHYS(max_low_pfn))); 233 234 dma32_phys_limit = min(4UL * SZ_1G, (unsigned long)PFN_PHYS(max_low_pfn)); 235 set_max_mapnr(max_low_pfn - ARCH_PFN_OFFSET); 236 237 reserve_initrd_mem(); 238 239 /* 240 * No allocation should be done before reserving the memory as defined 241 * in the device tree, otherwise the allocation could end up in a 242 * reserved region. 243 */ 244 early_init_fdt_scan_reserved_mem(); 245 246 /* 247 * If DTB is built in, no need to reserve its memblock. 248 * Otherwise, do reserve it but avoid using 249 * early_init_fdt_reserve_self() since __pa() does 250 * not work for DTB pointers that are fixmap addresses 251 */ 252 if (!IS_ENABLED(CONFIG_BUILTIN_DTB)) 253 memblock_reserve(dtb_early_pa, fdt_totalsize(dtb_early_va)); 254 255 dma_contiguous_reserve(dma32_phys_limit); 256 if (IS_ENABLED(CONFIG_64BIT)) 257 hugetlb_cma_reserve(PUD_SHIFT - PAGE_SHIFT); 258 memblock_allow_resize(); 259 } 260 261 #ifdef CONFIG_MMU 262 struct pt_alloc_ops pt_ops __initdata; 263 264 unsigned long riscv_pfn_base __ro_after_init; 265 EXPORT_SYMBOL(riscv_pfn_base); 266 267 pgd_t swapper_pg_dir[PTRS_PER_PGD] __page_aligned_bss; 268 pgd_t trampoline_pg_dir[PTRS_PER_PGD] __page_aligned_bss; 269 static pte_t fixmap_pte[PTRS_PER_PTE] __page_aligned_bss; 270 271 pgd_t early_pg_dir[PTRS_PER_PGD] __initdata __aligned(PAGE_SIZE); 272 273 #ifdef CONFIG_XIP_KERNEL 274 #define pt_ops (*(struct pt_alloc_ops *)XIP_FIXUP(&pt_ops)) 275 #define riscv_pfn_base (*(unsigned long *)XIP_FIXUP(&riscv_pfn_base)) 276 #define trampoline_pg_dir ((pgd_t *)XIP_FIXUP(trampoline_pg_dir)) 277 #define fixmap_pte ((pte_t *)XIP_FIXUP(fixmap_pte)) 278 #define early_pg_dir ((pgd_t *)XIP_FIXUP(early_pg_dir)) 279 #endif /* CONFIG_XIP_KERNEL */ 280 281 static const pgprot_t protection_map[16] = { 282 [VM_NONE] = PAGE_NONE, 283 [VM_READ] = PAGE_READ, 284 [VM_WRITE] = PAGE_COPY, 285 [VM_WRITE | VM_READ] = PAGE_COPY, 286 [VM_EXEC] = PAGE_EXEC, 287 [VM_EXEC | VM_READ] = PAGE_READ_EXEC, 288 [VM_EXEC | VM_WRITE] = PAGE_COPY_EXEC, 289 [VM_EXEC | VM_WRITE | VM_READ] = PAGE_COPY_READ_EXEC, 290 [VM_SHARED] = PAGE_NONE, 291 [VM_SHARED | VM_READ] = PAGE_READ, 292 [VM_SHARED | VM_WRITE] = PAGE_SHARED, 293 [VM_SHARED | VM_WRITE | VM_READ] = PAGE_SHARED, 294 [VM_SHARED | VM_EXEC] = PAGE_EXEC, 295 [VM_SHARED | VM_EXEC | VM_READ] = PAGE_READ_EXEC, 296 [VM_SHARED | VM_EXEC | VM_WRITE] = PAGE_SHARED_EXEC, 297 [VM_SHARED | VM_EXEC | VM_WRITE | VM_READ] = PAGE_SHARED_EXEC 298 }; 299 DECLARE_VM_GET_PAGE_PROT 300 301 void __set_fixmap(enum fixed_addresses idx, phys_addr_t phys, pgprot_t prot) 302 { 303 unsigned long addr = __fix_to_virt(idx); 304 pte_t *ptep; 305 306 BUG_ON(idx <= FIX_HOLE || idx >= __end_of_fixed_addresses); 307 308 ptep = &fixmap_pte[pte_index(addr)]; 309 310 if (pgprot_val(prot)) 311 set_pte(ptep, pfn_pte(phys >> PAGE_SHIFT, prot)); 312 else 313 pte_clear(&init_mm, addr, ptep); 314 local_flush_tlb_page(addr); 315 } 316 317 static inline pte_t *__init get_pte_virt_early(phys_addr_t pa) 318 { 319 return (pte_t *)((uintptr_t)pa); 320 } 321 322 static inline pte_t *__init get_pte_virt_fixmap(phys_addr_t pa) 323 { 324 clear_fixmap(FIX_PTE); 325 return (pte_t *)set_fixmap_offset(FIX_PTE, pa); 326 } 327 328 static inline pte_t *__init get_pte_virt_late(phys_addr_t pa) 329 { 330 return (pte_t *) __va(pa); 331 } 332 333 static inline phys_addr_t __init alloc_pte_early(uintptr_t va) 334 { 335 /* 336 * We only create PMD or PGD early mappings so we 337 * should never reach here with MMU disabled. 338 */ 339 BUG(); 340 } 341 342 static inline phys_addr_t __init alloc_pte_fixmap(uintptr_t va) 343 { 344 return memblock_phys_alloc(PAGE_SIZE, PAGE_SIZE); 345 } 346 347 static phys_addr_t __init alloc_pte_late(uintptr_t va) 348 { 349 unsigned long vaddr; 350 351 vaddr = __get_free_page(GFP_KERNEL); 352 BUG_ON(!vaddr || !pgtable_pte_page_ctor(virt_to_page(vaddr))); 353 354 return __pa(vaddr); 355 } 356 357 static void __init create_pte_mapping(pte_t *ptep, 358 uintptr_t va, phys_addr_t pa, 359 phys_addr_t sz, pgprot_t prot) 360 { 361 uintptr_t pte_idx = pte_index(va); 362 363 BUG_ON(sz != PAGE_SIZE); 364 365 if (pte_none(ptep[pte_idx])) 366 ptep[pte_idx] = pfn_pte(PFN_DOWN(pa), prot); 367 } 368 369 #ifndef __PAGETABLE_PMD_FOLDED 370 371 static pmd_t trampoline_pmd[PTRS_PER_PMD] __page_aligned_bss; 372 static pmd_t fixmap_pmd[PTRS_PER_PMD] __page_aligned_bss; 373 static pmd_t early_pmd[PTRS_PER_PMD] __initdata __aligned(PAGE_SIZE); 374 375 #ifdef CONFIG_XIP_KERNEL 376 #define trampoline_pmd ((pmd_t *)XIP_FIXUP(trampoline_pmd)) 377 #define fixmap_pmd ((pmd_t *)XIP_FIXUP(fixmap_pmd)) 378 #define early_pmd ((pmd_t *)XIP_FIXUP(early_pmd)) 379 #endif /* CONFIG_XIP_KERNEL */ 380 381 static p4d_t trampoline_p4d[PTRS_PER_P4D] __page_aligned_bss; 382 static p4d_t fixmap_p4d[PTRS_PER_P4D] __page_aligned_bss; 383 static p4d_t early_p4d[PTRS_PER_P4D] __initdata __aligned(PAGE_SIZE); 384 385 #ifdef CONFIG_XIP_KERNEL 386 #define trampoline_p4d ((p4d_t *)XIP_FIXUP(trampoline_p4d)) 387 #define fixmap_p4d ((p4d_t *)XIP_FIXUP(fixmap_p4d)) 388 #define early_p4d ((p4d_t *)XIP_FIXUP(early_p4d)) 389 #endif /* CONFIG_XIP_KERNEL */ 390 391 static pud_t trampoline_pud[PTRS_PER_PUD] __page_aligned_bss; 392 static pud_t fixmap_pud[PTRS_PER_PUD] __page_aligned_bss; 393 static pud_t early_pud[PTRS_PER_PUD] __initdata __aligned(PAGE_SIZE); 394 395 #ifdef CONFIG_XIP_KERNEL 396 #define trampoline_pud ((pud_t *)XIP_FIXUP(trampoline_pud)) 397 #define fixmap_pud ((pud_t *)XIP_FIXUP(fixmap_pud)) 398 #define early_pud ((pud_t *)XIP_FIXUP(early_pud)) 399 #endif /* CONFIG_XIP_KERNEL */ 400 401 static pmd_t *__init get_pmd_virt_early(phys_addr_t pa) 402 { 403 /* Before MMU is enabled */ 404 return (pmd_t *)((uintptr_t)pa); 405 } 406 407 static pmd_t *__init get_pmd_virt_fixmap(phys_addr_t pa) 408 { 409 clear_fixmap(FIX_PMD); 410 return (pmd_t *)set_fixmap_offset(FIX_PMD, pa); 411 } 412 413 static pmd_t *__init get_pmd_virt_late(phys_addr_t pa) 414 { 415 return (pmd_t *) __va(pa); 416 } 417 418 static phys_addr_t __init alloc_pmd_early(uintptr_t va) 419 { 420 BUG_ON((va - kernel_map.virt_addr) >> PUD_SHIFT); 421 422 return (uintptr_t)early_pmd; 423 } 424 425 static phys_addr_t __init alloc_pmd_fixmap(uintptr_t va) 426 { 427 return memblock_phys_alloc(PAGE_SIZE, PAGE_SIZE); 428 } 429 430 static phys_addr_t __init alloc_pmd_late(uintptr_t va) 431 { 432 unsigned long vaddr; 433 434 vaddr = __get_free_page(GFP_KERNEL); 435 BUG_ON(!vaddr || !pgtable_pmd_page_ctor(virt_to_page(vaddr))); 436 437 return __pa(vaddr); 438 } 439 440 static void __init create_pmd_mapping(pmd_t *pmdp, 441 uintptr_t va, phys_addr_t pa, 442 phys_addr_t sz, pgprot_t prot) 443 { 444 pte_t *ptep; 445 phys_addr_t pte_phys; 446 uintptr_t pmd_idx = pmd_index(va); 447 448 if (sz == PMD_SIZE) { 449 if (pmd_none(pmdp[pmd_idx])) 450 pmdp[pmd_idx] = pfn_pmd(PFN_DOWN(pa), prot); 451 return; 452 } 453 454 if (pmd_none(pmdp[pmd_idx])) { 455 pte_phys = pt_ops.alloc_pte(va); 456 pmdp[pmd_idx] = pfn_pmd(PFN_DOWN(pte_phys), PAGE_TABLE); 457 ptep = pt_ops.get_pte_virt(pte_phys); 458 memset(ptep, 0, PAGE_SIZE); 459 } else { 460 pte_phys = PFN_PHYS(_pmd_pfn(pmdp[pmd_idx])); 461 ptep = pt_ops.get_pte_virt(pte_phys); 462 } 463 464 create_pte_mapping(ptep, va, pa, sz, prot); 465 } 466 467 static pud_t *__init get_pud_virt_early(phys_addr_t pa) 468 { 469 return (pud_t *)((uintptr_t)pa); 470 } 471 472 static pud_t *__init get_pud_virt_fixmap(phys_addr_t pa) 473 { 474 clear_fixmap(FIX_PUD); 475 return (pud_t *)set_fixmap_offset(FIX_PUD, pa); 476 } 477 478 static pud_t *__init get_pud_virt_late(phys_addr_t pa) 479 { 480 return (pud_t *)__va(pa); 481 } 482 483 static phys_addr_t __init alloc_pud_early(uintptr_t va) 484 { 485 /* Only one PUD is available for early mapping */ 486 BUG_ON((va - kernel_map.virt_addr) >> PGDIR_SHIFT); 487 488 return (uintptr_t)early_pud; 489 } 490 491 static phys_addr_t __init alloc_pud_fixmap(uintptr_t va) 492 { 493 return memblock_phys_alloc(PAGE_SIZE, PAGE_SIZE); 494 } 495 496 static phys_addr_t alloc_pud_late(uintptr_t va) 497 { 498 unsigned long vaddr; 499 500 vaddr = __get_free_page(GFP_KERNEL); 501 BUG_ON(!vaddr); 502 return __pa(vaddr); 503 } 504 505 static p4d_t *__init get_p4d_virt_early(phys_addr_t pa) 506 { 507 return (p4d_t *)((uintptr_t)pa); 508 } 509 510 static p4d_t *__init get_p4d_virt_fixmap(phys_addr_t pa) 511 { 512 clear_fixmap(FIX_P4D); 513 return (p4d_t *)set_fixmap_offset(FIX_P4D, pa); 514 } 515 516 static p4d_t *__init get_p4d_virt_late(phys_addr_t pa) 517 { 518 return (p4d_t *)__va(pa); 519 } 520 521 static phys_addr_t __init alloc_p4d_early(uintptr_t va) 522 { 523 /* Only one P4D is available for early mapping */ 524 BUG_ON((va - kernel_map.virt_addr) >> PGDIR_SHIFT); 525 526 return (uintptr_t)early_p4d; 527 } 528 529 static phys_addr_t __init alloc_p4d_fixmap(uintptr_t va) 530 { 531 return memblock_phys_alloc(PAGE_SIZE, PAGE_SIZE); 532 } 533 534 static phys_addr_t alloc_p4d_late(uintptr_t va) 535 { 536 unsigned long vaddr; 537 538 vaddr = __get_free_page(GFP_KERNEL); 539 BUG_ON(!vaddr); 540 return __pa(vaddr); 541 } 542 543 static void __init create_pud_mapping(pud_t *pudp, 544 uintptr_t va, phys_addr_t pa, 545 phys_addr_t sz, pgprot_t prot) 546 { 547 pmd_t *nextp; 548 phys_addr_t next_phys; 549 uintptr_t pud_index = pud_index(va); 550 551 if (sz == PUD_SIZE) { 552 if (pud_val(pudp[pud_index]) == 0) 553 pudp[pud_index] = pfn_pud(PFN_DOWN(pa), prot); 554 return; 555 } 556 557 if (pud_val(pudp[pud_index]) == 0) { 558 next_phys = pt_ops.alloc_pmd(va); 559 pudp[pud_index] = pfn_pud(PFN_DOWN(next_phys), PAGE_TABLE); 560 nextp = pt_ops.get_pmd_virt(next_phys); 561 memset(nextp, 0, PAGE_SIZE); 562 } else { 563 next_phys = PFN_PHYS(_pud_pfn(pudp[pud_index])); 564 nextp = pt_ops.get_pmd_virt(next_phys); 565 } 566 567 create_pmd_mapping(nextp, va, pa, sz, prot); 568 } 569 570 static void __init create_p4d_mapping(p4d_t *p4dp, 571 uintptr_t va, phys_addr_t pa, 572 phys_addr_t sz, pgprot_t prot) 573 { 574 pud_t *nextp; 575 phys_addr_t next_phys; 576 uintptr_t p4d_index = p4d_index(va); 577 578 if (sz == P4D_SIZE) { 579 if (p4d_val(p4dp[p4d_index]) == 0) 580 p4dp[p4d_index] = pfn_p4d(PFN_DOWN(pa), prot); 581 return; 582 } 583 584 if (p4d_val(p4dp[p4d_index]) == 0) { 585 next_phys = pt_ops.alloc_pud(va); 586 p4dp[p4d_index] = pfn_p4d(PFN_DOWN(next_phys), PAGE_TABLE); 587 nextp = pt_ops.get_pud_virt(next_phys); 588 memset(nextp, 0, PAGE_SIZE); 589 } else { 590 next_phys = PFN_PHYS(_p4d_pfn(p4dp[p4d_index])); 591 nextp = pt_ops.get_pud_virt(next_phys); 592 } 593 594 create_pud_mapping(nextp, va, pa, sz, prot); 595 } 596 597 #define pgd_next_t p4d_t 598 #define alloc_pgd_next(__va) (pgtable_l5_enabled ? \ 599 pt_ops.alloc_p4d(__va) : (pgtable_l4_enabled ? \ 600 pt_ops.alloc_pud(__va) : pt_ops.alloc_pmd(__va))) 601 #define get_pgd_next_virt(__pa) (pgtable_l5_enabled ? \ 602 pt_ops.get_p4d_virt(__pa) : (pgd_next_t *)(pgtable_l4_enabled ? \ 603 pt_ops.get_pud_virt(__pa) : (pud_t *)pt_ops.get_pmd_virt(__pa))) 604 #define create_pgd_next_mapping(__nextp, __va, __pa, __sz, __prot) \ 605 (pgtable_l5_enabled ? \ 606 create_p4d_mapping(__nextp, __va, __pa, __sz, __prot) : \ 607 (pgtable_l4_enabled ? \ 608 create_pud_mapping((pud_t *)__nextp, __va, __pa, __sz, __prot) : \ 609 create_pmd_mapping((pmd_t *)__nextp, __va, __pa, __sz, __prot))) 610 #define fixmap_pgd_next (pgtable_l5_enabled ? \ 611 (uintptr_t)fixmap_p4d : (pgtable_l4_enabled ? \ 612 (uintptr_t)fixmap_pud : (uintptr_t)fixmap_pmd)) 613 #define trampoline_pgd_next (pgtable_l5_enabled ? \ 614 (uintptr_t)trampoline_p4d : (pgtable_l4_enabled ? \ 615 (uintptr_t)trampoline_pud : (uintptr_t)trampoline_pmd)) 616 #else 617 #define pgd_next_t pte_t 618 #define alloc_pgd_next(__va) pt_ops.alloc_pte(__va) 619 #define get_pgd_next_virt(__pa) pt_ops.get_pte_virt(__pa) 620 #define create_pgd_next_mapping(__nextp, __va, __pa, __sz, __prot) \ 621 create_pte_mapping(__nextp, __va, __pa, __sz, __prot) 622 #define fixmap_pgd_next ((uintptr_t)fixmap_pte) 623 #define create_p4d_mapping(__pmdp, __va, __pa, __sz, __prot) do {} while(0) 624 #define create_pud_mapping(__pmdp, __va, __pa, __sz, __prot) do {} while(0) 625 #define create_pmd_mapping(__pmdp, __va, __pa, __sz, __prot) do {} while(0) 626 #endif /* __PAGETABLE_PMD_FOLDED */ 627 628 void __init create_pgd_mapping(pgd_t *pgdp, 629 uintptr_t va, phys_addr_t pa, 630 phys_addr_t sz, pgprot_t prot) 631 { 632 pgd_next_t *nextp; 633 phys_addr_t next_phys; 634 uintptr_t pgd_idx = pgd_index(va); 635 636 if (sz == PGDIR_SIZE) { 637 if (pgd_val(pgdp[pgd_idx]) == 0) 638 pgdp[pgd_idx] = pfn_pgd(PFN_DOWN(pa), prot); 639 return; 640 } 641 642 if (pgd_val(pgdp[pgd_idx]) == 0) { 643 next_phys = alloc_pgd_next(va); 644 pgdp[pgd_idx] = pfn_pgd(PFN_DOWN(next_phys), PAGE_TABLE); 645 nextp = get_pgd_next_virt(next_phys); 646 memset(nextp, 0, PAGE_SIZE); 647 } else { 648 next_phys = PFN_PHYS(_pgd_pfn(pgdp[pgd_idx])); 649 nextp = get_pgd_next_virt(next_phys); 650 } 651 652 create_pgd_next_mapping(nextp, va, pa, sz, prot); 653 } 654 655 static uintptr_t __init best_map_size(phys_addr_t base, phys_addr_t size) 656 { 657 /* Upgrade to PMD_SIZE mappings whenever possible */ 658 base &= PMD_SIZE - 1; 659 if (!base && size >= PMD_SIZE) 660 return PMD_SIZE; 661 662 return PAGE_SIZE; 663 } 664 665 #ifdef CONFIG_XIP_KERNEL 666 #define phys_ram_base (*(phys_addr_t *)XIP_FIXUP(&phys_ram_base)) 667 extern char _xiprom[], _exiprom[], __data_loc; 668 669 /* called from head.S with MMU off */ 670 asmlinkage void __init __copy_data(void) 671 { 672 void *from = (void *)(&__data_loc); 673 void *to = (void *)CONFIG_PHYS_RAM_BASE; 674 size_t sz = (size_t)((uintptr_t)(&_end) - (uintptr_t)(&_sdata)); 675 676 memcpy(to, from, sz); 677 } 678 #endif 679 680 #ifdef CONFIG_STRICT_KERNEL_RWX 681 static __init pgprot_t pgprot_from_va(uintptr_t va) 682 { 683 if (is_va_kernel_text(va)) 684 return PAGE_KERNEL_READ_EXEC; 685 686 /* 687 * In 64-bit kernel, the kernel mapping is outside the linear mapping so 688 * we must protect its linear mapping alias from being executed and 689 * written. 690 * And rodata section is marked readonly in mark_rodata_ro. 691 */ 692 if (IS_ENABLED(CONFIG_64BIT) && is_va_kernel_lm_alias_text(va)) 693 return PAGE_KERNEL_READ; 694 695 return PAGE_KERNEL; 696 } 697 698 void mark_rodata_ro(void) 699 { 700 set_kernel_memory(__start_rodata, _data, set_memory_ro); 701 if (IS_ENABLED(CONFIG_64BIT)) 702 set_kernel_memory(lm_alias(__start_rodata), lm_alias(_data), 703 set_memory_ro); 704 705 debug_checkwx(); 706 } 707 #else 708 static __init pgprot_t pgprot_from_va(uintptr_t va) 709 { 710 if (IS_ENABLED(CONFIG_64BIT) && !is_kernel_mapping(va)) 711 return PAGE_KERNEL; 712 713 return PAGE_KERNEL_EXEC; 714 } 715 #endif /* CONFIG_STRICT_KERNEL_RWX */ 716 717 #if defined(CONFIG_64BIT) && !defined(CONFIG_XIP_KERNEL) 718 static void __init disable_pgtable_l5(void) 719 { 720 pgtable_l5_enabled = false; 721 kernel_map.page_offset = PAGE_OFFSET_L4; 722 satp_mode = SATP_MODE_48; 723 } 724 725 static void __init disable_pgtable_l4(void) 726 { 727 pgtable_l4_enabled = false; 728 kernel_map.page_offset = PAGE_OFFSET_L3; 729 satp_mode = SATP_MODE_39; 730 } 731 732 /* 733 * There is a simple way to determine if 4-level is supported by the 734 * underlying hardware: establish 1:1 mapping in 4-level page table mode 735 * then read SATP to see if the configuration was taken into account 736 * meaning sv48 is supported. 737 */ 738 static __init void set_satp_mode(void) 739 { 740 u64 identity_satp, hw_satp; 741 uintptr_t set_satp_mode_pmd = ((unsigned long)set_satp_mode) & PMD_MASK; 742 bool check_l4 = false; 743 744 create_p4d_mapping(early_p4d, 745 set_satp_mode_pmd, (uintptr_t)early_pud, 746 P4D_SIZE, PAGE_TABLE); 747 create_pud_mapping(early_pud, 748 set_satp_mode_pmd, (uintptr_t)early_pmd, 749 PUD_SIZE, PAGE_TABLE); 750 /* Handle the case where set_satp_mode straddles 2 PMDs */ 751 create_pmd_mapping(early_pmd, 752 set_satp_mode_pmd, set_satp_mode_pmd, 753 PMD_SIZE, PAGE_KERNEL_EXEC); 754 create_pmd_mapping(early_pmd, 755 set_satp_mode_pmd + PMD_SIZE, 756 set_satp_mode_pmd + PMD_SIZE, 757 PMD_SIZE, PAGE_KERNEL_EXEC); 758 retry: 759 create_pgd_mapping(early_pg_dir, 760 set_satp_mode_pmd, 761 check_l4 ? (uintptr_t)early_pud : (uintptr_t)early_p4d, 762 PGDIR_SIZE, PAGE_TABLE); 763 764 identity_satp = PFN_DOWN((uintptr_t)&early_pg_dir) | satp_mode; 765 766 local_flush_tlb_all(); 767 csr_write(CSR_SATP, identity_satp); 768 hw_satp = csr_swap(CSR_SATP, 0ULL); 769 local_flush_tlb_all(); 770 771 if (hw_satp != identity_satp) { 772 if (!check_l4) { 773 disable_pgtable_l5(); 774 check_l4 = true; 775 memset(early_pg_dir, 0, PAGE_SIZE); 776 goto retry; 777 } 778 disable_pgtable_l4(); 779 } 780 781 memset(early_pg_dir, 0, PAGE_SIZE); 782 memset(early_p4d, 0, PAGE_SIZE); 783 memset(early_pud, 0, PAGE_SIZE); 784 memset(early_pmd, 0, PAGE_SIZE); 785 } 786 #endif 787 788 /* 789 * setup_vm() is called from head.S with MMU-off. 790 * 791 * Following requirements should be honoured for setup_vm() to work 792 * correctly: 793 * 1) It should use PC-relative addressing for accessing kernel symbols. 794 * To achieve this we always use GCC cmodel=medany. 795 * 2) The compiler instrumentation for FTRACE will not work for setup_vm() 796 * so disable compiler instrumentation when FTRACE is enabled. 797 * 798 * Currently, the above requirements are honoured by using custom CFLAGS 799 * for init.o in mm/Makefile. 800 */ 801 802 #ifndef __riscv_cmodel_medany 803 #error "setup_vm() is called from head.S before relocate so it should not use absolute addressing." 804 #endif 805 806 #ifdef CONFIG_XIP_KERNEL 807 static void __init create_kernel_page_table(pgd_t *pgdir, 808 __always_unused bool early) 809 { 810 uintptr_t va, end_va; 811 812 /* Map the flash resident part */ 813 end_va = kernel_map.virt_addr + kernel_map.xiprom_sz; 814 for (va = kernel_map.virt_addr; va < end_va; va += PMD_SIZE) 815 create_pgd_mapping(pgdir, va, 816 kernel_map.xiprom + (va - kernel_map.virt_addr), 817 PMD_SIZE, PAGE_KERNEL_EXEC); 818 819 /* Map the data in RAM */ 820 end_va = kernel_map.virt_addr + XIP_OFFSET + kernel_map.size; 821 for (va = kernel_map.virt_addr + XIP_OFFSET; va < end_va; va += PMD_SIZE) 822 create_pgd_mapping(pgdir, va, 823 kernel_map.phys_addr + (va - (kernel_map.virt_addr + XIP_OFFSET)), 824 PMD_SIZE, PAGE_KERNEL); 825 } 826 #else 827 static void __init create_kernel_page_table(pgd_t *pgdir, bool early) 828 { 829 uintptr_t va, end_va; 830 831 end_va = kernel_map.virt_addr + kernel_map.size; 832 for (va = kernel_map.virt_addr; va < end_va; va += PMD_SIZE) 833 create_pgd_mapping(pgdir, va, 834 kernel_map.phys_addr + (va - kernel_map.virt_addr), 835 PMD_SIZE, 836 early ? 837 PAGE_KERNEL_EXEC : pgprot_from_va(va)); 838 } 839 #endif 840 841 /* 842 * Setup a 4MB mapping that encompasses the device tree: for 64-bit kernel, 843 * this means 2 PMD entries whereas for 32-bit kernel, this is only 1 PGDIR 844 * entry. 845 */ 846 static void __init create_fdt_early_page_table(pgd_t *pgdir, 847 uintptr_t fix_fdt_va, 848 uintptr_t dtb_pa) 849 { 850 uintptr_t pa = dtb_pa & ~(PMD_SIZE - 1); 851 852 #ifndef CONFIG_BUILTIN_DTB 853 /* Make sure the fdt fixmap address is always aligned on PMD size */ 854 BUILD_BUG_ON(FIX_FDT % (PMD_SIZE / PAGE_SIZE)); 855 856 /* In 32-bit only, the fdt lies in its own PGD */ 857 if (!IS_ENABLED(CONFIG_64BIT)) { 858 create_pgd_mapping(early_pg_dir, fix_fdt_va, 859 pa, MAX_FDT_SIZE, PAGE_KERNEL); 860 } else { 861 create_pmd_mapping(fixmap_pmd, fix_fdt_va, 862 pa, PMD_SIZE, PAGE_KERNEL); 863 create_pmd_mapping(fixmap_pmd, fix_fdt_va + PMD_SIZE, 864 pa + PMD_SIZE, PMD_SIZE, PAGE_KERNEL); 865 } 866 867 dtb_early_va = (void *)fix_fdt_va + (dtb_pa & (PMD_SIZE - 1)); 868 #else 869 /* 870 * For 64-bit kernel, __va can't be used since it would return a linear 871 * mapping address whereas dtb_early_va will be used before 872 * setup_vm_final installs the linear mapping. For 32-bit kernel, as the 873 * kernel is mapped in the linear mapping, that makes no difference. 874 */ 875 dtb_early_va = kernel_mapping_pa_to_va(XIP_FIXUP(dtb_pa)); 876 #endif 877 878 dtb_early_pa = dtb_pa; 879 } 880 881 /* 882 * MMU is not enabled, the page tables are allocated directly using 883 * early_pmd/pud/p4d and the address returned is the physical one. 884 */ 885 static void __init pt_ops_set_early(void) 886 { 887 pt_ops.alloc_pte = alloc_pte_early; 888 pt_ops.get_pte_virt = get_pte_virt_early; 889 #ifndef __PAGETABLE_PMD_FOLDED 890 pt_ops.alloc_pmd = alloc_pmd_early; 891 pt_ops.get_pmd_virt = get_pmd_virt_early; 892 pt_ops.alloc_pud = alloc_pud_early; 893 pt_ops.get_pud_virt = get_pud_virt_early; 894 pt_ops.alloc_p4d = alloc_p4d_early; 895 pt_ops.get_p4d_virt = get_p4d_virt_early; 896 #endif 897 } 898 899 /* 900 * MMU is enabled but page table setup is not complete yet. 901 * fixmap page table alloc functions must be used as a means to temporarily 902 * map the allocated physical pages since the linear mapping does not exist yet. 903 * 904 * Note that this is called with MMU disabled, hence kernel_mapping_pa_to_va, 905 * but it will be used as described above. 906 */ 907 static void __init pt_ops_set_fixmap(void) 908 { 909 pt_ops.alloc_pte = kernel_mapping_pa_to_va(alloc_pte_fixmap); 910 pt_ops.get_pte_virt = kernel_mapping_pa_to_va(get_pte_virt_fixmap); 911 #ifndef __PAGETABLE_PMD_FOLDED 912 pt_ops.alloc_pmd = kernel_mapping_pa_to_va(alloc_pmd_fixmap); 913 pt_ops.get_pmd_virt = kernel_mapping_pa_to_va(get_pmd_virt_fixmap); 914 pt_ops.alloc_pud = kernel_mapping_pa_to_va(alloc_pud_fixmap); 915 pt_ops.get_pud_virt = kernel_mapping_pa_to_va(get_pud_virt_fixmap); 916 pt_ops.alloc_p4d = kernel_mapping_pa_to_va(alloc_p4d_fixmap); 917 pt_ops.get_p4d_virt = kernel_mapping_pa_to_va(get_p4d_virt_fixmap); 918 #endif 919 } 920 921 /* 922 * MMU is enabled and page table setup is complete, so from now, we can use 923 * generic page allocation functions to setup page table. 924 */ 925 static void __init pt_ops_set_late(void) 926 { 927 pt_ops.alloc_pte = alloc_pte_late; 928 pt_ops.get_pte_virt = get_pte_virt_late; 929 #ifndef __PAGETABLE_PMD_FOLDED 930 pt_ops.alloc_pmd = alloc_pmd_late; 931 pt_ops.get_pmd_virt = get_pmd_virt_late; 932 pt_ops.alloc_pud = alloc_pud_late; 933 pt_ops.get_pud_virt = get_pud_virt_late; 934 pt_ops.alloc_p4d = alloc_p4d_late; 935 pt_ops.get_p4d_virt = get_p4d_virt_late; 936 #endif 937 } 938 939 asmlinkage void __init setup_vm(uintptr_t dtb_pa) 940 { 941 pmd_t __maybe_unused fix_bmap_spmd, fix_bmap_epmd; 942 943 kernel_map.virt_addr = KERNEL_LINK_ADDR; 944 kernel_map.page_offset = _AC(CONFIG_PAGE_OFFSET, UL); 945 946 #ifdef CONFIG_XIP_KERNEL 947 kernel_map.xiprom = (uintptr_t)CONFIG_XIP_PHYS_ADDR; 948 kernel_map.xiprom_sz = (uintptr_t)(&_exiprom) - (uintptr_t)(&_xiprom); 949 950 phys_ram_base = CONFIG_PHYS_RAM_BASE; 951 kernel_map.phys_addr = (uintptr_t)CONFIG_PHYS_RAM_BASE; 952 kernel_map.size = (uintptr_t)(&_end) - (uintptr_t)(&_sdata); 953 954 kernel_map.va_kernel_xip_pa_offset = kernel_map.virt_addr - kernel_map.xiprom; 955 #else 956 kernel_map.phys_addr = (uintptr_t)(&_start); 957 kernel_map.size = (uintptr_t)(&_end) - kernel_map.phys_addr; 958 #endif 959 960 #if defined(CONFIG_64BIT) && !defined(CONFIG_XIP_KERNEL) 961 set_satp_mode(); 962 #endif 963 964 kernel_map.va_pa_offset = PAGE_OFFSET - kernel_map.phys_addr; 965 kernel_map.va_kernel_pa_offset = kernel_map.virt_addr - kernel_map.phys_addr; 966 967 riscv_pfn_base = PFN_DOWN(kernel_map.phys_addr); 968 969 /* 970 * The default maximal physical memory size is KERN_VIRT_SIZE for 32-bit 971 * kernel, whereas for 64-bit kernel, the end of the virtual address 972 * space is occupied by the modules/BPF/kernel mappings which reduces 973 * the available size of the linear mapping. 974 */ 975 memory_limit = KERN_VIRT_SIZE - (IS_ENABLED(CONFIG_64BIT) ? SZ_4G : 0); 976 977 /* Sanity check alignment and size */ 978 BUG_ON((PAGE_OFFSET % PGDIR_SIZE) != 0); 979 BUG_ON((kernel_map.phys_addr % PMD_SIZE) != 0); 980 981 #ifdef CONFIG_64BIT 982 /* 983 * The last 4K bytes of the addressable memory can not be mapped because 984 * of IS_ERR_VALUE macro. 985 */ 986 BUG_ON((kernel_map.virt_addr + kernel_map.size) > ADDRESS_SPACE_END - SZ_4K); 987 #endif 988 989 apply_early_boot_alternatives(); 990 pt_ops_set_early(); 991 992 /* Setup early PGD for fixmap */ 993 create_pgd_mapping(early_pg_dir, FIXADDR_START, 994 fixmap_pgd_next, PGDIR_SIZE, PAGE_TABLE); 995 996 #ifndef __PAGETABLE_PMD_FOLDED 997 /* Setup fixmap P4D and PUD */ 998 if (pgtable_l5_enabled) 999 create_p4d_mapping(fixmap_p4d, FIXADDR_START, 1000 (uintptr_t)fixmap_pud, P4D_SIZE, PAGE_TABLE); 1001 /* Setup fixmap PUD and PMD */ 1002 if (pgtable_l4_enabled) 1003 create_pud_mapping(fixmap_pud, FIXADDR_START, 1004 (uintptr_t)fixmap_pmd, PUD_SIZE, PAGE_TABLE); 1005 create_pmd_mapping(fixmap_pmd, FIXADDR_START, 1006 (uintptr_t)fixmap_pte, PMD_SIZE, PAGE_TABLE); 1007 /* Setup trampoline PGD and PMD */ 1008 create_pgd_mapping(trampoline_pg_dir, kernel_map.virt_addr, 1009 trampoline_pgd_next, PGDIR_SIZE, PAGE_TABLE); 1010 if (pgtable_l5_enabled) 1011 create_p4d_mapping(trampoline_p4d, kernel_map.virt_addr, 1012 (uintptr_t)trampoline_pud, P4D_SIZE, PAGE_TABLE); 1013 if (pgtable_l4_enabled) 1014 create_pud_mapping(trampoline_pud, kernel_map.virt_addr, 1015 (uintptr_t)trampoline_pmd, PUD_SIZE, PAGE_TABLE); 1016 #ifdef CONFIG_XIP_KERNEL 1017 create_pmd_mapping(trampoline_pmd, kernel_map.virt_addr, 1018 kernel_map.xiprom, PMD_SIZE, PAGE_KERNEL_EXEC); 1019 #else 1020 create_pmd_mapping(trampoline_pmd, kernel_map.virt_addr, 1021 kernel_map.phys_addr, PMD_SIZE, PAGE_KERNEL_EXEC); 1022 #endif 1023 #else 1024 /* Setup trampoline PGD */ 1025 create_pgd_mapping(trampoline_pg_dir, kernel_map.virt_addr, 1026 kernel_map.phys_addr, PGDIR_SIZE, PAGE_KERNEL_EXEC); 1027 #endif 1028 1029 /* 1030 * Setup early PGD covering entire kernel which will allow 1031 * us to reach paging_init(). We map all memory banks later 1032 * in setup_vm_final() below. 1033 */ 1034 create_kernel_page_table(early_pg_dir, true); 1035 1036 /* Setup early mapping for FDT early scan */ 1037 create_fdt_early_page_table(early_pg_dir, 1038 __fix_to_virt(FIX_FDT), dtb_pa); 1039 1040 /* 1041 * Bootime fixmap only can handle PMD_SIZE mapping. Thus, boot-ioremap 1042 * range can not span multiple pmds. 1043 */ 1044 BUG_ON((__fix_to_virt(FIX_BTMAP_BEGIN) >> PMD_SHIFT) 1045 != (__fix_to_virt(FIX_BTMAP_END) >> PMD_SHIFT)); 1046 1047 #ifndef __PAGETABLE_PMD_FOLDED 1048 /* 1049 * Early ioremap fixmap is already created as it lies within first 2MB 1050 * of fixmap region. We always map PMD_SIZE. Thus, both FIX_BTMAP_END 1051 * FIX_BTMAP_BEGIN should lie in the same pmd. Verify that and warn 1052 * the user if not. 1053 */ 1054 fix_bmap_spmd = fixmap_pmd[pmd_index(__fix_to_virt(FIX_BTMAP_BEGIN))]; 1055 fix_bmap_epmd = fixmap_pmd[pmd_index(__fix_to_virt(FIX_BTMAP_END))]; 1056 if (pmd_val(fix_bmap_spmd) != pmd_val(fix_bmap_epmd)) { 1057 WARN_ON(1); 1058 pr_warn("fixmap btmap start [%08lx] != end [%08lx]\n", 1059 pmd_val(fix_bmap_spmd), pmd_val(fix_bmap_epmd)); 1060 pr_warn("fix_to_virt(FIX_BTMAP_BEGIN): %08lx\n", 1061 fix_to_virt(FIX_BTMAP_BEGIN)); 1062 pr_warn("fix_to_virt(FIX_BTMAP_END): %08lx\n", 1063 fix_to_virt(FIX_BTMAP_END)); 1064 1065 pr_warn("FIX_BTMAP_END: %d\n", FIX_BTMAP_END); 1066 pr_warn("FIX_BTMAP_BEGIN: %d\n", FIX_BTMAP_BEGIN); 1067 } 1068 #endif 1069 1070 pt_ops_set_fixmap(); 1071 } 1072 1073 static void __init setup_vm_final(void) 1074 { 1075 uintptr_t va, map_size; 1076 phys_addr_t pa, start, end; 1077 u64 i; 1078 1079 /* Setup swapper PGD for fixmap */ 1080 #if !defined(CONFIG_64BIT) 1081 /* 1082 * In 32-bit, the device tree lies in a pgd entry, so it must be copied 1083 * directly in swapper_pg_dir in addition to the pgd entry that points 1084 * to fixmap_pte. 1085 */ 1086 unsigned long idx = pgd_index(__fix_to_virt(FIX_FDT)); 1087 1088 set_pgd(&swapper_pg_dir[idx], early_pg_dir[idx]); 1089 #endif 1090 create_pgd_mapping(swapper_pg_dir, FIXADDR_START, 1091 __pa_symbol(fixmap_pgd_next), 1092 PGDIR_SIZE, PAGE_TABLE); 1093 1094 /* Map all memory banks in the linear mapping */ 1095 for_each_mem_range(i, &start, &end) { 1096 if (start >= end) 1097 break; 1098 if (start <= __pa(PAGE_OFFSET) && 1099 __pa(PAGE_OFFSET) < end) 1100 start = __pa(PAGE_OFFSET); 1101 if (end >= __pa(PAGE_OFFSET) + memory_limit) 1102 end = __pa(PAGE_OFFSET) + memory_limit; 1103 1104 for (pa = start; pa < end; pa += map_size) { 1105 va = (uintptr_t)__va(pa); 1106 map_size = best_map_size(pa, end - pa); 1107 1108 create_pgd_mapping(swapper_pg_dir, va, pa, map_size, 1109 pgprot_from_va(va)); 1110 } 1111 } 1112 1113 /* Map the kernel */ 1114 if (IS_ENABLED(CONFIG_64BIT)) 1115 create_kernel_page_table(swapper_pg_dir, false); 1116 1117 #ifdef CONFIG_KASAN 1118 kasan_swapper_init(); 1119 #endif 1120 1121 /* Clear fixmap PTE and PMD mappings */ 1122 clear_fixmap(FIX_PTE); 1123 clear_fixmap(FIX_PMD); 1124 clear_fixmap(FIX_PUD); 1125 clear_fixmap(FIX_P4D); 1126 1127 /* Move to swapper page table */ 1128 csr_write(CSR_SATP, PFN_DOWN(__pa_symbol(swapper_pg_dir)) | satp_mode); 1129 local_flush_tlb_all(); 1130 1131 pt_ops_set_late(); 1132 } 1133 #else 1134 asmlinkage void __init setup_vm(uintptr_t dtb_pa) 1135 { 1136 dtb_early_va = (void *)dtb_pa; 1137 dtb_early_pa = dtb_pa; 1138 } 1139 1140 static inline void setup_vm_final(void) 1141 { 1142 } 1143 #endif /* CONFIG_MMU */ 1144 1145 /* 1146 * reserve_crashkernel() - reserves memory for crash kernel 1147 * 1148 * This function reserves memory area given in "crashkernel=" kernel command 1149 * line parameter. The memory reserved is used by dump capture kernel when 1150 * primary kernel is crashing. 1151 */ 1152 static void __init reserve_crashkernel(void) 1153 { 1154 unsigned long long crash_base = 0; 1155 unsigned long long crash_size = 0; 1156 unsigned long search_start = memblock_start_of_DRAM(); 1157 unsigned long search_end = memblock_end_of_DRAM(); 1158 1159 int ret = 0; 1160 1161 if (!IS_ENABLED(CONFIG_KEXEC_CORE)) 1162 return; 1163 /* 1164 * Don't reserve a region for a crash kernel on a crash kernel 1165 * since it doesn't make much sense and we have limited memory 1166 * resources. 1167 */ 1168 if (is_kdump_kernel()) { 1169 pr_info("crashkernel: ignoring reservation request\n"); 1170 return; 1171 } 1172 1173 ret = parse_crashkernel(boot_command_line, memblock_phys_mem_size(), 1174 &crash_size, &crash_base); 1175 if (ret || !crash_size) 1176 return; 1177 1178 crash_size = PAGE_ALIGN(crash_size); 1179 1180 if (crash_base) { 1181 search_start = crash_base; 1182 search_end = crash_base + crash_size; 1183 } 1184 1185 /* 1186 * Current riscv boot protocol requires 2MB alignment for 1187 * RV64 and 4MB alignment for RV32 (hugepage size) 1188 * 1189 * Try to alloc from 32bit addressible physical memory so that 1190 * swiotlb can work on the crash kernel. 1191 */ 1192 crash_base = memblock_phys_alloc_range(crash_size, PMD_SIZE, 1193 search_start, 1194 min(search_end, (unsigned long) SZ_4G)); 1195 if (crash_base == 0) { 1196 /* Try again without restricting region to 32bit addressible memory */ 1197 crash_base = memblock_phys_alloc_range(crash_size, PMD_SIZE, 1198 search_start, search_end); 1199 if (crash_base == 0) { 1200 pr_warn("crashkernel: couldn't allocate %lldKB\n", 1201 crash_size >> 10); 1202 return; 1203 } 1204 } 1205 1206 pr_info("crashkernel: reserved 0x%016llx - 0x%016llx (%lld MB)\n", 1207 crash_base, crash_base + crash_size, crash_size >> 20); 1208 1209 crashk_res.start = crash_base; 1210 crashk_res.end = crash_base + crash_size - 1; 1211 } 1212 1213 void __init paging_init(void) 1214 { 1215 setup_bootmem(); 1216 setup_vm_final(); 1217 } 1218 1219 void __init misc_mem_init(void) 1220 { 1221 early_memtest(min_low_pfn << PAGE_SHIFT, max_low_pfn << PAGE_SHIFT); 1222 arch_numa_init(); 1223 sparse_init(); 1224 zone_sizes_init(); 1225 reserve_crashkernel(); 1226 memblock_dump_all(); 1227 } 1228 1229 #ifdef CONFIG_SPARSEMEM_VMEMMAP 1230 int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node, 1231 struct vmem_altmap *altmap) 1232 { 1233 return vmemmap_populate_basepages(start, end, node, NULL); 1234 } 1235 #endif 1236