1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2012 Regents of the University of California 4 * Copyright (C) 2019 Western Digital Corporation or its affiliates. 5 * Copyright (C) 2020 FORTH-ICS/CARV 6 * Nick Kossifidis <mick@ics.forth.gr> 7 */ 8 9 #include <linux/init.h> 10 #include <linux/mm.h> 11 #include <linux/memblock.h> 12 #include <linux/initrd.h> 13 #include <linux/swap.h> 14 #include <linux/swiotlb.h> 15 #include <linux/sizes.h> 16 #include <linux/of_fdt.h> 17 #include <linux/of_reserved_mem.h> 18 #include <linux/libfdt.h> 19 #include <linux/set_memory.h> 20 #include <linux/dma-map-ops.h> 21 #include <linux/crash_dump.h> 22 #include <linux/hugetlb.h> 23 24 #include <asm/fixmap.h> 25 #include <asm/tlbflush.h> 26 #include <asm/sections.h> 27 #include <asm/soc.h> 28 #include <asm/io.h> 29 #include <asm/ptdump.h> 30 #include <asm/numa.h> 31 32 #include "../kernel/head.h" 33 34 struct kernel_mapping kernel_map __ro_after_init; 35 EXPORT_SYMBOL(kernel_map); 36 #ifdef CONFIG_XIP_KERNEL 37 #define kernel_map (*(struct kernel_mapping *)XIP_FIXUP(&kernel_map)) 38 #endif 39 40 #ifdef CONFIG_64BIT 41 u64 satp_mode = !IS_ENABLED(CONFIG_XIP_KERNEL) ? SATP_MODE_48 : SATP_MODE_39; 42 #else 43 u64 satp_mode = SATP_MODE_32; 44 #endif 45 EXPORT_SYMBOL(satp_mode); 46 47 bool pgtable_l4_enabled = IS_ENABLED(CONFIG_64BIT) && !IS_ENABLED(CONFIG_XIP_KERNEL); 48 EXPORT_SYMBOL(pgtable_l4_enabled); 49 50 phys_addr_t phys_ram_base __ro_after_init; 51 EXPORT_SYMBOL(phys_ram_base); 52 53 unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)] 54 __page_aligned_bss; 55 EXPORT_SYMBOL(empty_zero_page); 56 57 extern char _start[]; 58 #define DTB_EARLY_BASE_VA PGDIR_SIZE 59 void *_dtb_early_va __initdata; 60 uintptr_t _dtb_early_pa __initdata; 61 62 static phys_addr_t dma32_phys_limit __initdata; 63 64 static void __init zone_sizes_init(void) 65 { 66 unsigned long max_zone_pfns[MAX_NR_ZONES] = { 0, }; 67 68 #ifdef CONFIG_ZONE_DMA32 69 max_zone_pfns[ZONE_DMA32] = PFN_DOWN(dma32_phys_limit); 70 #endif 71 max_zone_pfns[ZONE_NORMAL] = max_low_pfn; 72 73 free_area_init(max_zone_pfns); 74 } 75 76 #if defined(CONFIG_MMU) && defined(CONFIG_DEBUG_VM) 77 static inline void print_mlk(char *name, unsigned long b, unsigned long t) 78 { 79 pr_notice("%12s : 0x%08lx - 0x%08lx (%4ld kB)\n", name, b, t, 80 (((t) - (b)) >> 10)); 81 } 82 83 static inline void print_mlm(char *name, unsigned long b, unsigned long t) 84 { 85 pr_notice("%12s : 0x%08lx - 0x%08lx (%4ld MB)\n", name, b, t, 86 (((t) - (b)) >> 20)); 87 } 88 89 static void __init print_vm_layout(void) 90 { 91 pr_notice("Virtual kernel memory layout:\n"); 92 print_mlk("fixmap", (unsigned long)FIXADDR_START, 93 (unsigned long)FIXADDR_TOP); 94 print_mlm("pci io", (unsigned long)PCI_IO_START, 95 (unsigned long)PCI_IO_END); 96 print_mlm("vmemmap", (unsigned long)VMEMMAP_START, 97 (unsigned long)VMEMMAP_END); 98 print_mlm("vmalloc", (unsigned long)VMALLOC_START, 99 (unsigned long)VMALLOC_END); 100 print_mlm("lowmem", (unsigned long)PAGE_OFFSET, 101 (unsigned long)high_memory); 102 if (IS_ENABLED(CONFIG_64BIT)) { 103 #ifdef CONFIG_KASAN 104 print_mlm("kasan", KASAN_SHADOW_START, KASAN_SHADOW_END); 105 #endif 106 107 print_mlm("kernel", (unsigned long)KERNEL_LINK_ADDR, 108 (unsigned long)ADDRESS_SPACE_END); 109 } 110 } 111 #else 112 static void print_vm_layout(void) { } 113 #endif /* CONFIG_DEBUG_VM */ 114 115 void __init mem_init(void) 116 { 117 #ifdef CONFIG_FLATMEM 118 BUG_ON(!mem_map); 119 #endif /* CONFIG_FLATMEM */ 120 121 #ifdef CONFIG_SWIOTLB 122 if (swiotlb_force == SWIOTLB_FORCE || 123 max_pfn > PFN_DOWN(dma32_phys_limit)) 124 swiotlb_init(1); 125 else 126 swiotlb_force = SWIOTLB_NO_FORCE; 127 #endif 128 memblock_free_all(); 129 130 print_vm_layout(); 131 } 132 133 /* Limit the memory size via mem. */ 134 static phys_addr_t memory_limit; 135 136 static int __init early_mem(char *p) 137 { 138 u64 size; 139 140 if (!p) 141 return 1; 142 143 size = memparse(p, &p) & PAGE_MASK; 144 memory_limit = min_t(u64, size, memory_limit); 145 146 pr_notice("Memory limited to %lldMB\n", (u64)memory_limit >> 20); 147 148 return 0; 149 } 150 early_param("mem", early_mem); 151 152 static void __init setup_bootmem(void) 153 { 154 phys_addr_t vmlinux_end = __pa_symbol(&_end); 155 phys_addr_t max_mapped_addr; 156 phys_addr_t phys_ram_end, vmlinux_start; 157 158 if (IS_ENABLED(CONFIG_XIP_KERNEL)) 159 vmlinux_start = __pa_symbol(&_sdata); 160 else 161 vmlinux_start = __pa_symbol(&_start); 162 163 memblock_enforce_memory_limit(memory_limit); 164 165 /* 166 * Make sure we align the reservation on PMD_SIZE since we will 167 * map the kernel in the linear mapping as read-only: we do not want 168 * any allocation to happen between _end and the next pmd aligned page. 169 */ 170 if (IS_ENABLED(CONFIG_64BIT) && IS_ENABLED(CONFIG_STRICT_KERNEL_RWX)) 171 vmlinux_end = (vmlinux_end + PMD_SIZE - 1) & PMD_MASK; 172 /* 173 * Reserve from the start of the kernel to the end of the kernel 174 */ 175 memblock_reserve(vmlinux_start, vmlinux_end - vmlinux_start); 176 177 phys_ram_end = memblock_end_of_DRAM(); 178 if (!IS_ENABLED(CONFIG_XIP_KERNEL)) 179 phys_ram_base = memblock_start_of_DRAM(); 180 /* 181 * memblock allocator is not aware of the fact that last 4K bytes of 182 * the addressable memory can not be mapped because of IS_ERR_VALUE 183 * macro. Make sure that last 4k bytes are not usable by memblock 184 * if end of dram is equal to maximum addressable memory. For 64-bit 185 * kernel, this problem can't happen here as the end of the virtual 186 * address space is occupied by the kernel mapping then this check must 187 * be done as soon as the kernel mapping base address is determined. 188 */ 189 if (!IS_ENABLED(CONFIG_64BIT)) { 190 max_mapped_addr = __pa(~(ulong)0); 191 if (max_mapped_addr == (phys_ram_end - 1)) 192 memblock_set_current_limit(max_mapped_addr - 4096); 193 } 194 195 min_low_pfn = PFN_UP(phys_ram_base); 196 max_low_pfn = max_pfn = PFN_DOWN(phys_ram_end); 197 high_memory = (void *)(__va(PFN_PHYS(max_low_pfn))); 198 199 dma32_phys_limit = min(4UL * SZ_1G, (unsigned long)PFN_PHYS(max_low_pfn)); 200 set_max_mapnr(max_low_pfn - ARCH_PFN_OFFSET); 201 202 reserve_initrd_mem(); 203 /* 204 * If DTB is built in, no need to reserve its memblock. 205 * Otherwise, do reserve it but avoid using 206 * early_init_fdt_reserve_self() since __pa() does 207 * not work for DTB pointers that are fixmap addresses 208 */ 209 if (!IS_ENABLED(CONFIG_BUILTIN_DTB)) 210 memblock_reserve(dtb_early_pa, fdt_totalsize(dtb_early_va)); 211 212 early_init_fdt_scan_reserved_mem(); 213 dma_contiguous_reserve(dma32_phys_limit); 214 if (IS_ENABLED(CONFIG_64BIT)) 215 hugetlb_cma_reserve(PUD_SHIFT - PAGE_SHIFT); 216 memblock_allow_resize(); 217 } 218 219 #ifdef CONFIG_MMU 220 struct pt_alloc_ops pt_ops __initdata; 221 222 unsigned long riscv_pfn_base __ro_after_init; 223 EXPORT_SYMBOL(riscv_pfn_base); 224 225 pgd_t swapper_pg_dir[PTRS_PER_PGD] __page_aligned_bss; 226 pgd_t trampoline_pg_dir[PTRS_PER_PGD] __page_aligned_bss; 227 static pte_t fixmap_pte[PTRS_PER_PTE] __page_aligned_bss; 228 229 pgd_t early_pg_dir[PTRS_PER_PGD] __initdata __aligned(PAGE_SIZE); 230 static pud_t __maybe_unused early_dtb_pud[PTRS_PER_PUD] __initdata __aligned(PAGE_SIZE); 231 static pmd_t __maybe_unused early_dtb_pmd[PTRS_PER_PMD] __initdata __aligned(PAGE_SIZE); 232 233 #ifdef CONFIG_XIP_KERNEL 234 #define pt_ops (*(struct pt_alloc_ops *)XIP_FIXUP(&pt_ops)) 235 #define riscv_pfn_base (*(unsigned long *)XIP_FIXUP(&riscv_pfn_base)) 236 #define trampoline_pg_dir ((pgd_t *)XIP_FIXUP(trampoline_pg_dir)) 237 #define fixmap_pte ((pte_t *)XIP_FIXUP(fixmap_pte)) 238 #define early_pg_dir ((pgd_t *)XIP_FIXUP(early_pg_dir)) 239 #endif /* CONFIG_XIP_KERNEL */ 240 241 void __set_fixmap(enum fixed_addresses idx, phys_addr_t phys, pgprot_t prot) 242 { 243 unsigned long addr = __fix_to_virt(idx); 244 pte_t *ptep; 245 246 BUG_ON(idx <= FIX_HOLE || idx >= __end_of_fixed_addresses); 247 248 ptep = &fixmap_pte[pte_index(addr)]; 249 250 if (pgprot_val(prot)) 251 set_pte(ptep, pfn_pte(phys >> PAGE_SHIFT, prot)); 252 else 253 pte_clear(&init_mm, addr, ptep); 254 local_flush_tlb_page(addr); 255 } 256 257 static inline pte_t *__init get_pte_virt_early(phys_addr_t pa) 258 { 259 return (pte_t *)((uintptr_t)pa); 260 } 261 262 static inline pte_t *__init get_pte_virt_fixmap(phys_addr_t pa) 263 { 264 clear_fixmap(FIX_PTE); 265 return (pte_t *)set_fixmap_offset(FIX_PTE, pa); 266 } 267 268 static inline pte_t *__init get_pte_virt_late(phys_addr_t pa) 269 { 270 return (pte_t *) __va(pa); 271 } 272 273 static inline phys_addr_t __init alloc_pte_early(uintptr_t va) 274 { 275 /* 276 * We only create PMD or PGD early mappings so we 277 * should never reach here with MMU disabled. 278 */ 279 BUG(); 280 } 281 282 static inline phys_addr_t __init alloc_pte_fixmap(uintptr_t va) 283 { 284 return memblock_phys_alloc(PAGE_SIZE, PAGE_SIZE); 285 } 286 287 static phys_addr_t __init alloc_pte_late(uintptr_t va) 288 { 289 unsigned long vaddr; 290 291 vaddr = __get_free_page(GFP_KERNEL); 292 BUG_ON(!vaddr || !pgtable_pte_page_ctor(virt_to_page(vaddr))); 293 294 return __pa(vaddr); 295 } 296 297 static void __init create_pte_mapping(pte_t *ptep, 298 uintptr_t va, phys_addr_t pa, 299 phys_addr_t sz, pgprot_t prot) 300 { 301 uintptr_t pte_idx = pte_index(va); 302 303 BUG_ON(sz != PAGE_SIZE); 304 305 if (pte_none(ptep[pte_idx])) 306 ptep[pte_idx] = pfn_pte(PFN_DOWN(pa), prot); 307 } 308 309 #ifndef __PAGETABLE_PMD_FOLDED 310 311 static pmd_t trampoline_pmd[PTRS_PER_PMD] __page_aligned_bss; 312 static pmd_t fixmap_pmd[PTRS_PER_PMD] __page_aligned_bss; 313 static pmd_t early_pmd[PTRS_PER_PMD] __initdata __aligned(PAGE_SIZE); 314 315 #ifdef CONFIG_XIP_KERNEL 316 #define trampoline_pmd ((pmd_t *)XIP_FIXUP(trampoline_pmd)) 317 #define fixmap_pmd ((pmd_t *)XIP_FIXUP(fixmap_pmd)) 318 #define early_pmd ((pmd_t *)XIP_FIXUP(early_pmd)) 319 #endif /* CONFIG_XIP_KERNEL */ 320 321 static pud_t trampoline_pud[PTRS_PER_PUD] __page_aligned_bss; 322 static pud_t fixmap_pud[PTRS_PER_PUD] __page_aligned_bss; 323 static pud_t early_pud[PTRS_PER_PUD] __initdata __aligned(PAGE_SIZE); 324 325 #ifdef CONFIG_XIP_KERNEL 326 #define trampoline_pud ((pud_t *)XIP_FIXUP(trampoline_pud)) 327 #define fixmap_pud ((pud_t *)XIP_FIXUP(fixmap_pud)) 328 #define early_pud ((pud_t *)XIP_FIXUP(early_pud)) 329 #endif /* CONFIG_XIP_KERNEL */ 330 331 static pmd_t *__init get_pmd_virt_early(phys_addr_t pa) 332 { 333 /* Before MMU is enabled */ 334 return (pmd_t *)((uintptr_t)pa); 335 } 336 337 static pmd_t *__init get_pmd_virt_fixmap(phys_addr_t pa) 338 { 339 clear_fixmap(FIX_PMD); 340 return (pmd_t *)set_fixmap_offset(FIX_PMD, pa); 341 } 342 343 static pmd_t *__init get_pmd_virt_late(phys_addr_t pa) 344 { 345 return (pmd_t *) __va(pa); 346 } 347 348 static phys_addr_t __init alloc_pmd_early(uintptr_t va) 349 { 350 BUG_ON((va - kernel_map.virt_addr) >> PUD_SHIFT); 351 352 return (uintptr_t)early_pmd; 353 } 354 355 static phys_addr_t __init alloc_pmd_fixmap(uintptr_t va) 356 { 357 return memblock_phys_alloc(PAGE_SIZE, PAGE_SIZE); 358 } 359 360 static phys_addr_t __init alloc_pmd_late(uintptr_t va) 361 { 362 unsigned long vaddr; 363 364 vaddr = __get_free_page(GFP_KERNEL); 365 BUG_ON(!vaddr || !pgtable_pmd_page_ctor(virt_to_page(vaddr))); 366 367 return __pa(vaddr); 368 } 369 370 static void __init create_pmd_mapping(pmd_t *pmdp, 371 uintptr_t va, phys_addr_t pa, 372 phys_addr_t sz, pgprot_t prot) 373 { 374 pte_t *ptep; 375 phys_addr_t pte_phys; 376 uintptr_t pmd_idx = pmd_index(va); 377 378 if (sz == PMD_SIZE) { 379 if (pmd_none(pmdp[pmd_idx])) 380 pmdp[pmd_idx] = pfn_pmd(PFN_DOWN(pa), prot); 381 return; 382 } 383 384 if (pmd_none(pmdp[pmd_idx])) { 385 pte_phys = pt_ops.alloc_pte(va); 386 pmdp[pmd_idx] = pfn_pmd(PFN_DOWN(pte_phys), PAGE_TABLE); 387 ptep = pt_ops.get_pte_virt(pte_phys); 388 memset(ptep, 0, PAGE_SIZE); 389 } else { 390 pte_phys = PFN_PHYS(_pmd_pfn(pmdp[pmd_idx])); 391 ptep = pt_ops.get_pte_virt(pte_phys); 392 } 393 394 create_pte_mapping(ptep, va, pa, sz, prot); 395 } 396 397 static pud_t *__init get_pud_virt_early(phys_addr_t pa) 398 { 399 return (pud_t *)((uintptr_t)pa); 400 } 401 402 static pud_t *__init get_pud_virt_fixmap(phys_addr_t pa) 403 { 404 clear_fixmap(FIX_PUD); 405 return (pud_t *)set_fixmap_offset(FIX_PUD, pa); 406 } 407 408 static pud_t *__init get_pud_virt_late(phys_addr_t pa) 409 { 410 return (pud_t *)__va(pa); 411 } 412 413 static phys_addr_t __init alloc_pud_early(uintptr_t va) 414 { 415 /* Only one PUD is available for early mapping */ 416 BUG_ON((va - kernel_map.virt_addr) >> PGDIR_SHIFT); 417 418 return (uintptr_t)early_pud; 419 } 420 421 static phys_addr_t __init alloc_pud_fixmap(uintptr_t va) 422 { 423 return memblock_phys_alloc(PAGE_SIZE, PAGE_SIZE); 424 } 425 426 static phys_addr_t alloc_pud_late(uintptr_t va) 427 { 428 unsigned long vaddr; 429 430 vaddr = __get_free_page(GFP_KERNEL); 431 BUG_ON(!vaddr); 432 return __pa(vaddr); 433 } 434 435 static void __init create_pud_mapping(pud_t *pudp, 436 uintptr_t va, phys_addr_t pa, 437 phys_addr_t sz, pgprot_t prot) 438 { 439 pmd_t *nextp; 440 phys_addr_t next_phys; 441 uintptr_t pud_index = pud_index(va); 442 443 if (sz == PUD_SIZE) { 444 if (pud_val(pudp[pud_index]) == 0) 445 pudp[pud_index] = pfn_pud(PFN_DOWN(pa), prot); 446 return; 447 } 448 449 if (pud_val(pudp[pud_index]) == 0) { 450 next_phys = pt_ops.alloc_pmd(va); 451 pudp[pud_index] = pfn_pud(PFN_DOWN(next_phys), PAGE_TABLE); 452 nextp = pt_ops.get_pmd_virt(next_phys); 453 memset(nextp, 0, PAGE_SIZE); 454 } else { 455 next_phys = PFN_PHYS(_pud_pfn(pudp[pud_index])); 456 nextp = pt_ops.get_pmd_virt(next_phys); 457 } 458 459 create_pmd_mapping(nextp, va, pa, sz, prot); 460 } 461 462 #define pgd_next_t pud_t 463 #define alloc_pgd_next(__va) (pgtable_l4_enabled ? \ 464 pt_ops.alloc_pud(__va) : pt_ops.alloc_pmd(__va)) 465 #define get_pgd_next_virt(__pa) (pgtable_l4_enabled ? \ 466 pt_ops.get_pud_virt(__pa) : (pgd_next_t *)pt_ops.get_pmd_virt(__pa)) 467 #define create_pgd_next_mapping(__nextp, __va, __pa, __sz, __prot) \ 468 (pgtable_l4_enabled ? \ 469 create_pud_mapping(__nextp, __va, __pa, __sz, __prot) : \ 470 create_pmd_mapping((pmd_t *)__nextp, __va, __pa, __sz, __prot)) 471 #define fixmap_pgd_next (pgtable_l4_enabled ? \ 472 (uintptr_t)fixmap_pud : (uintptr_t)fixmap_pmd) 473 #define trampoline_pgd_next (pgtable_l4_enabled ? \ 474 (uintptr_t)trampoline_pud : (uintptr_t)trampoline_pmd) 475 #define early_dtb_pgd_next (pgtable_l4_enabled ? \ 476 (uintptr_t)early_dtb_pud : (uintptr_t)early_dtb_pmd) 477 #else 478 #define pgd_next_t pte_t 479 #define alloc_pgd_next(__va) pt_ops.alloc_pte(__va) 480 #define get_pgd_next_virt(__pa) pt_ops.get_pte_virt(__pa) 481 #define create_pgd_next_mapping(__nextp, __va, __pa, __sz, __prot) \ 482 create_pte_mapping(__nextp, __va, __pa, __sz, __prot) 483 #define fixmap_pgd_next ((uintptr_t)fixmap_pte) 484 #define early_dtb_pgd_next ((uintptr_t)early_dtb_pmd) 485 #define create_pud_mapping(__pmdp, __va, __pa, __sz, __prot) 486 #define create_pmd_mapping(__pmdp, __va, __pa, __sz, __prot) 487 #endif /* __PAGETABLE_PMD_FOLDED */ 488 489 void __init create_pgd_mapping(pgd_t *pgdp, 490 uintptr_t va, phys_addr_t pa, 491 phys_addr_t sz, pgprot_t prot) 492 { 493 pgd_next_t *nextp; 494 phys_addr_t next_phys; 495 uintptr_t pgd_idx = pgd_index(va); 496 497 if (sz == PGDIR_SIZE) { 498 if (pgd_val(pgdp[pgd_idx]) == 0) 499 pgdp[pgd_idx] = pfn_pgd(PFN_DOWN(pa), prot); 500 return; 501 } 502 503 if (pgd_val(pgdp[pgd_idx]) == 0) { 504 next_phys = alloc_pgd_next(va); 505 pgdp[pgd_idx] = pfn_pgd(PFN_DOWN(next_phys), PAGE_TABLE); 506 nextp = get_pgd_next_virt(next_phys); 507 memset(nextp, 0, PAGE_SIZE); 508 } else { 509 next_phys = PFN_PHYS(_pgd_pfn(pgdp[pgd_idx])); 510 nextp = get_pgd_next_virt(next_phys); 511 } 512 513 create_pgd_next_mapping(nextp, va, pa, sz, prot); 514 } 515 516 static uintptr_t __init best_map_size(phys_addr_t base, phys_addr_t size) 517 { 518 /* Upgrade to PMD_SIZE mappings whenever possible */ 519 if ((base & (PMD_SIZE - 1)) || (size & (PMD_SIZE - 1))) 520 return PAGE_SIZE; 521 522 return PMD_SIZE; 523 } 524 525 #ifdef CONFIG_XIP_KERNEL 526 #define phys_ram_base (*(phys_addr_t *)XIP_FIXUP(&phys_ram_base)) 527 extern char _xiprom[], _exiprom[], __data_loc; 528 529 /* called from head.S with MMU off */ 530 asmlinkage void __init __copy_data(void) 531 { 532 void *from = (void *)(&__data_loc); 533 void *to = (void *)CONFIG_PHYS_RAM_BASE; 534 size_t sz = (size_t)((uintptr_t)(&_end) - (uintptr_t)(&_sdata)); 535 536 memcpy(to, from, sz); 537 } 538 #endif 539 540 #ifdef CONFIG_STRICT_KERNEL_RWX 541 static __init pgprot_t pgprot_from_va(uintptr_t va) 542 { 543 if (is_va_kernel_text(va)) 544 return PAGE_KERNEL_READ_EXEC; 545 546 /* 547 * In 64-bit kernel, the kernel mapping is outside the linear mapping so 548 * we must protect its linear mapping alias from being executed and 549 * written. 550 * And rodata section is marked readonly in mark_rodata_ro. 551 */ 552 if (IS_ENABLED(CONFIG_64BIT) && is_va_kernel_lm_alias_text(va)) 553 return PAGE_KERNEL_READ; 554 555 return PAGE_KERNEL; 556 } 557 558 void mark_rodata_ro(void) 559 { 560 set_kernel_memory(__start_rodata, _data, set_memory_ro); 561 if (IS_ENABLED(CONFIG_64BIT)) 562 set_kernel_memory(lm_alias(__start_rodata), lm_alias(_data), 563 set_memory_ro); 564 565 debug_checkwx(); 566 } 567 #else 568 static __init pgprot_t pgprot_from_va(uintptr_t va) 569 { 570 if (IS_ENABLED(CONFIG_64BIT) && !is_kernel_mapping(va)) 571 return PAGE_KERNEL; 572 573 return PAGE_KERNEL_EXEC; 574 } 575 #endif /* CONFIG_STRICT_KERNEL_RWX */ 576 577 #ifdef CONFIG_64BIT 578 static void __init disable_pgtable_l4(void) 579 { 580 pgtable_l4_enabled = false; 581 kernel_map.page_offset = PAGE_OFFSET_L3; 582 satp_mode = SATP_MODE_39; 583 } 584 585 /* 586 * There is a simple way to determine if 4-level is supported by the 587 * underlying hardware: establish 1:1 mapping in 4-level page table mode 588 * then read SATP to see if the configuration was taken into account 589 * meaning sv48 is supported. 590 */ 591 static __init void set_satp_mode(void) 592 { 593 u64 identity_satp, hw_satp; 594 uintptr_t set_satp_mode_pmd; 595 596 set_satp_mode_pmd = ((unsigned long)set_satp_mode) & PMD_MASK; 597 create_pgd_mapping(early_pg_dir, 598 set_satp_mode_pmd, (uintptr_t)early_pud, 599 PGDIR_SIZE, PAGE_TABLE); 600 create_pud_mapping(early_pud, 601 set_satp_mode_pmd, (uintptr_t)early_pmd, 602 PUD_SIZE, PAGE_TABLE); 603 /* Handle the case where set_satp_mode straddles 2 PMDs */ 604 create_pmd_mapping(early_pmd, 605 set_satp_mode_pmd, set_satp_mode_pmd, 606 PMD_SIZE, PAGE_KERNEL_EXEC); 607 create_pmd_mapping(early_pmd, 608 set_satp_mode_pmd + PMD_SIZE, 609 set_satp_mode_pmd + PMD_SIZE, 610 PMD_SIZE, PAGE_KERNEL_EXEC); 611 612 identity_satp = PFN_DOWN((uintptr_t)&early_pg_dir) | satp_mode; 613 614 local_flush_tlb_all(); 615 csr_write(CSR_SATP, identity_satp); 616 hw_satp = csr_swap(CSR_SATP, 0ULL); 617 local_flush_tlb_all(); 618 619 if (hw_satp != identity_satp) 620 disable_pgtable_l4(); 621 622 memset(early_pg_dir, 0, PAGE_SIZE); 623 memset(early_pud, 0, PAGE_SIZE); 624 memset(early_pmd, 0, PAGE_SIZE); 625 } 626 #endif 627 628 /* 629 * setup_vm() is called from head.S with MMU-off. 630 * 631 * Following requirements should be honoured for setup_vm() to work 632 * correctly: 633 * 1) It should use PC-relative addressing for accessing kernel symbols. 634 * To achieve this we always use GCC cmodel=medany. 635 * 2) The compiler instrumentation for FTRACE will not work for setup_vm() 636 * so disable compiler instrumentation when FTRACE is enabled. 637 * 638 * Currently, the above requirements are honoured by using custom CFLAGS 639 * for init.o in mm/Makefile. 640 */ 641 642 #ifndef __riscv_cmodel_medany 643 #error "setup_vm() is called from head.S before relocate so it should not use absolute addressing." 644 #endif 645 646 #ifdef CONFIG_XIP_KERNEL 647 static void __init create_kernel_page_table(pgd_t *pgdir, 648 __always_unused bool early) 649 { 650 uintptr_t va, end_va; 651 652 /* Map the flash resident part */ 653 end_va = kernel_map.virt_addr + kernel_map.xiprom_sz; 654 for (va = kernel_map.virt_addr; va < end_va; va += PMD_SIZE) 655 create_pgd_mapping(pgdir, va, 656 kernel_map.xiprom + (va - kernel_map.virt_addr), 657 PMD_SIZE, PAGE_KERNEL_EXEC); 658 659 /* Map the data in RAM */ 660 end_va = kernel_map.virt_addr + XIP_OFFSET + kernel_map.size; 661 for (va = kernel_map.virt_addr + XIP_OFFSET; va < end_va; va += PMD_SIZE) 662 create_pgd_mapping(pgdir, va, 663 kernel_map.phys_addr + (va - (kernel_map.virt_addr + XIP_OFFSET)), 664 PMD_SIZE, PAGE_KERNEL); 665 } 666 #else 667 static void __init create_kernel_page_table(pgd_t *pgdir, bool early) 668 { 669 uintptr_t va, end_va; 670 671 end_va = kernel_map.virt_addr + kernel_map.size; 672 for (va = kernel_map.virt_addr; va < end_va; va += PMD_SIZE) 673 create_pgd_mapping(pgdir, va, 674 kernel_map.phys_addr + (va - kernel_map.virt_addr), 675 PMD_SIZE, 676 early ? 677 PAGE_KERNEL_EXEC : pgprot_from_va(va)); 678 } 679 #endif 680 681 /* 682 * Setup a 4MB mapping that encompasses the device tree: for 64-bit kernel, 683 * this means 2 PMD entries whereas for 32-bit kernel, this is only 1 PGDIR 684 * entry. 685 */ 686 static void __init create_fdt_early_page_table(pgd_t *pgdir, uintptr_t dtb_pa) 687 { 688 #ifndef CONFIG_BUILTIN_DTB 689 uintptr_t pa = dtb_pa & ~(PMD_SIZE - 1); 690 691 create_pgd_mapping(early_pg_dir, DTB_EARLY_BASE_VA, 692 IS_ENABLED(CONFIG_64BIT) ? early_dtb_pgd_next : pa, 693 PGDIR_SIZE, 694 IS_ENABLED(CONFIG_64BIT) ? PAGE_TABLE : PAGE_KERNEL); 695 696 if (pgtable_l4_enabled) { 697 create_pud_mapping(early_dtb_pud, DTB_EARLY_BASE_VA, 698 (uintptr_t)early_dtb_pmd, PUD_SIZE, PAGE_TABLE); 699 } 700 701 if (IS_ENABLED(CONFIG_64BIT)) { 702 create_pmd_mapping(early_dtb_pmd, DTB_EARLY_BASE_VA, 703 pa, PMD_SIZE, PAGE_KERNEL); 704 create_pmd_mapping(early_dtb_pmd, DTB_EARLY_BASE_VA + PMD_SIZE, 705 pa + PMD_SIZE, PMD_SIZE, PAGE_KERNEL); 706 } 707 708 dtb_early_va = (void *)DTB_EARLY_BASE_VA + (dtb_pa & (PMD_SIZE - 1)); 709 #else 710 /* 711 * For 64-bit kernel, __va can't be used since it would return a linear 712 * mapping address whereas dtb_early_va will be used before 713 * setup_vm_final installs the linear mapping. For 32-bit kernel, as the 714 * kernel is mapped in the linear mapping, that makes no difference. 715 */ 716 dtb_early_va = kernel_mapping_pa_to_va(XIP_FIXUP(dtb_pa)); 717 #endif 718 719 dtb_early_pa = dtb_pa; 720 } 721 722 /* 723 * MMU is not enabled, the page tables are allocated directly using 724 * early_pmd/pud/p4d and the address returned is the physical one. 725 */ 726 void __init pt_ops_set_early(void) 727 { 728 pt_ops.alloc_pte = alloc_pte_early; 729 pt_ops.get_pte_virt = get_pte_virt_early; 730 #ifndef __PAGETABLE_PMD_FOLDED 731 pt_ops.alloc_pmd = alloc_pmd_early; 732 pt_ops.get_pmd_virt = get_pmd_virt_early; 733 pt_ops.alloc_pud = alloc_pud_early; 734 pt_ops.get_pud_virt = get_pud_virt_early; 735 #endif 736 } 737 738 /* 739 * MMU is enabled but page table setup is not complete yet. 740 * fixmap page table alloc functions must be used as a means to temporarily 741 * map the allocated physical pages since the linear mapping does not exist yet. 742 * 743 * Note that this is called with MMU disabled, hence kernel_mapping_pa_to_va, 744 * but it will be used as described above. 745 */ 746 void __init pt_ops_set_fixmap(void) 747 { 748 pt_ops.alloc_pte = kernel_mapping_pa_to_va((uintptr_t)alloc_pte_fixmap); 749 pt_ops.get_pte_virt = kernel_mapping_pa_to_va((uintptr_t)get_pte_virt_fixmap); 750 #ifndef __PAGETABLE_PMD_FOLDED 751 pt_ops.alloc_pmd = kernel_mapping_pa_to_va((uintptr_t)alloc_pmd_fixmap); 752 pt_ops.get_pmd_virt = kernel_mapping_pa_to_va((uintptr_t)get_pmd_virt_fixmap); 753 pt_ops.alloc_pud = kernel_mapping_pa_to_va((uintptr_t)alloc_pud_fixmap); 754 pt_ops.get_pud_virt = kernel_mapping_pa_to_va((uintptr_t)get_pud_virt_fixmap); 755 #endif 756 } 757 758 /* 759 * MMU is enabled and page table setup is complete, so from now, we can use 760 * generic page allocation functions to setup page table. 761 */ 762 void __init pt_ops_set_late(void) 763 { 764 pt_ops.alloc_pte = alloc_pte_late; 765 pt_ops.get_pte_virt = get_pte_virt_late; 766 #ifndef __PAGETABLE_PMD_FOLDED 767 pt_ops.alloc_pmd = alloc_pmd_late; 768 pt_ops.get_pmd_virt = get_pmd_virt_late; 769 pt_ops.alloc_pud = alloc_pud_late; 770 pt_ops.get_pud_virt = get_pud_virt_late; 771 #endif 772 } 773 774 asmlinkage void __init setup_vm(uintptr_t dtb_pa) 775 { 776 pmd_t __maybe_unused fix_bmap_spmd, fix_bmap_epmd; 777 778 kernel_map.virt_addr = KERNEL_LINK_ADDR; 779 kernel_map.page_offset = _AC(CONFIG_PAGE_OFFSET, UL); 780 781 #ifdef CONFIG_XIP_KERNEL 782 kernel_map.xiprom = (uintptr_t)CONFIG_XIP_PHYS_ADDR; 783 kernel_map.xiprom_sz = (uintptr_t)(&_exiprom) - (uintptr_t)(&_xiprom); 784 785 phys_ram_base = CONFIG_PHYS_RAM_BASE; 786 kernel_map.phys_addr = (uintptr_t)CONFIG_PHYS_RAM_BASE; 787 kernel_map.size = (uintptr_t)(&_end) - (uintptr_t)(&_sdata); 788 789 kernel_map.va_kernel_xip_pa_offset = kernel_map.virt_addr - kernel_map.xiprom; 790 #else 791 kernel_map.phys_addr = (uintptr_t)(&_start); 792 kernel_map.size = (uintptr_t)(&_end) - kernel_map.phys_addr; 793 #endif 794 795 #if defined(CONFIG_64BIT) && !defined(CONFIG_XIP_KERNEL) 796 set_satp_mode(); 797 #endif 798 799 kernel_map.va_pa_offset = PAGE_OFFSET - kernel_map.phys_addr; 800 kernel_map.va_kernel_pa_offset = kernel_map.virt_addr - kernel_map.phys_addr; 801 802 riscv_pfn_base = PFN_DOWN(kernel_map.phys_addr); 803 804 /* 805 * The default maximal physical memory size is KERN_VIRT_SIZE for 32-bit 806 * kernel, whereas for 64-bit kernel, the end of the virtual address 807 * space is occupied by the modules/BPF/kernel mappings which reduces 808 * the available size of the linear mapping. 809 */ 810 memory_limit = KERN_VIRT_SIZE - (IS_ENABLED(CONFIG_64BIT) ? SZ_4G : 0); 811 812 /* Sanity check alignment and size */ 813 BUG_ON((PAGE_OFFSET % PGDIR_SIZE) != 0); 814 BUG_ON((kernel_map.phys_addr % PMD_SIZE) != 0); 815 816 #ifdef CONFIG_64BIT 817 /* 818 * The last 4K bytes of the addressable memory can not be mapped because 819 * of IS_ERR_VALUE macro. 820 */ 821 BUG_ON((kernel_map.virt_addr + kernel_map.size) > ADDRESS_SPACE_END - SZ_4K); 822 #endif 823 824 pt_ops_set_early(); 825 826 /* Setup early PGD for fixmap */ 827 create_pgd_mapping(early_pg_dir, FIXADDR_START, 828 fixmap_pgd_next, PGDIR_SIZE, PAGE_TABLE); 829 830 #ifndef __PAGETABLE_PMD_FOLDED 831 /* Setup fixmap PUD and PMD */ 832 if (pgtable_l4_enabled) 833 create_pud_mapping(fixmap_pud, FIXADDR_START, 834 (uintptr_t)fixmap_pmd, PUD_SIZE, PAGE_TABLE); 835 create_pmd_mapping(fixmap_pmd, FIXADDR_START, 836 (uintptr_t)fixmap_pte, PMD_SIZE, PAGE_TABLE); 837 /* Setup trampoline PGD and PMD */ 838 create_pgd_mapping(trampoline_pg_dir, kernel_map.virt_addr, 839 trampoline_pgd_next, PGDIR_SIZE, PAGE_TABLE); 840 if (pgtable_l4_enabled) 841 create_pud_mapping(trampoline_pud, kernel_map.virt_addr, 842 (uintptr_t)trampoline_pmd, PUD_SIZE, PAGE_TABLE); 843 #ifdef CONFIG_XIP_KERNEL 844 create_pmd_mapping(trampoline_pmd, kernel_map.virt_addr, 845 kernel_map.xiprom, PMD_SIZE, PAGE_KERNEL_EXEC); 846 #else 847 create_pmd_mapping(trampoline_pmd, kernel_map.virt_addr, 848 kernel_map.phys_addr, PMD_SIZE, PAGE_KERNEL_EXEC); 849 #endif 850 #else 851 /* Setup trampoline PGD */ 852 create_pgd_mapping(trampoline_pg_dir, kernel_map.virt_addr, 853 kernel_map.phys_addr, PGDIR_SIZE, PAGE_KERNEL_EXEC); 854 #endif 855 856 /* 857 * Setup early PGD covering entire kernel which will allow 858 * us to reach paging_init(). We map all memory banks later 859 * in setup_vm_final() below. 860 */ 861 create_kernel_page_table(early_pg_dir, true); 862 863 /* Setup early mapping for FDT early scan */ 864 create_fdt_early_page_table(early_pg_dir, dtb_pa); 865 866 /* 867 * Bootime fixmap only can handle PMD_SIZE mapping. Thus, boot-ioremap 868 * range can not span multiple pmds. 869 */ 870 BUG_ON((__fix_to_virt(FIX_BTMAP_BEGIN) >> PMD_SHIFT) 871 != (__fix_to_virt(FIX_BTMAP_END) >> PMD_SHIFT)); 872 873 #ifndef __PAGETABLE_PMD_FOLDED 874 /* 875 * Early ioremap fixmap is already created as it lies within first 2MB 876 * of fixmap region. We always map PMD_SIZE. Thus, both FIX_BTMAP_END 877 * FIX_BTMAP_BEGIN should lie in the same pmd. Verify that and warn 878 * the user if not. 879 */ 880 fix_bmap_spmd = fixmap_pmd[pmd_index(__fix_to_virt(FIX_BTMAP_BEGIN))]; 881 fix_bmap_epmd = fixmap_pmd[pmd_index(__fix_to_virt(FIX_BTMAP_END))]; 882 if (pmd_val(fix_bmap_spmd) != pmd_val(fix_bmap_epmd)) { 883 WARN_ON(1); 884 pr_warn("fixmap btmap start [%08lx] != end [%08lx]\n", 885 pmd_val(fix_bmap_spmd), pmd_val(fix_bmap_epmd)); 886 pr_warn("fix_to_virt(FIX_BTMAP_BEGIN): %08lx\n", 887 fix_to_virt(FIX_BTMAP_BEGIN)); 888 pr_warn("fix_to_virt(FIX_BTMAP_END): %08lx\n", 889 fix_to_virt(FIX_BTMAP_END)); 890 891 pr_warn("FIX_BTMAP_END: %d\n", FIX_BTMAP_END); 892 pr_warn("FIX_BTMAP_BEGIN: %d\n", FIX_BTMAP_BEGIN); 893 } 894 #endif 895 896 pt_ops_set_fixmap(); 897 } 898 899 static void __init setup_vm_final(void) 900 { 901 uintptr_t va, map_size; 902 phys_addr_t pa, start, end; 903 u64 i; 904 905 /* Setup swapper PGD for fixmap */ 906 create_pgd_mapping(swapper_pg_dir, FIXADDR_START, 907 __pa_symbol(fixmap_pgd_next), 908 PGDIR_SIZE, PAGE_TABLE); 909 910 /* Map all memory banks in the linear mapping */ 911 for_each_mem_range(i, &start, &end) { 912 if (start >= end) 913 break; 914 if (start <= __pa(PAGE_OFFSET) && 915 __pa(PAGE_OFFSET) < end) 916 start = __pa(PAGE_OFFSET); 917 if (end >= __pa(PAGE_OFFSET) + memory_limit) 918 end = __pa(PAGE_OFFSET) + memory_limit; 919 920 map_size = best_map_size(start, end - start); 921 for (pa = start; pa < end; pa += map_size) { 922 va = (uintptr_t)__va(pa); 923 924 create_pgd_mapping(swapper_pg_dir, va, pa, map_size, 925 pgprot_from_va(va)); 926 } 927 } 928 929 /* Map the kernel */ 930 if (IS_ENABLED(CONFIG_64BIT)) 931 create_kernel_page_table(swapper_pg_dir, false); 932 933 #ifdef CONFIG_KASAN 934 kasan_swapper_init(); 935 #endif 936 937 /* Clear fixmap PTE and PMD mappings */ 938 clear_fixmap(FIX_PTE); 939 clear_fixmap(FIX_PMD); 940 clear_fixmap(FIX_PUD); 941 942 /* Move to swapper page table */ 943 csr_write(CSR_SATP, PFN_DOWN(__pa_symbol(swapper_pg_dir)) | satp_mode); 944 local_flush_tlb_all(); 945 946 pt_ops_set_late(); 947 } 948 #else 949 asmlinkage void __init setup_vm(uintptr_t dtb_pa) 950 { 951 dtb_early_va = (void *)dtb_pa; 952 dtb_early_pa = dtb_pa; 953 } 954 955 static inline void setup_vm_final(void) 956 { 957 } 958 #endif /* CONFIG_MMU */ 959 960 #ifdef CONFIG_KEXEC_CORE 961 /* 962 * reserve_crashkernel() - reserves memory for crash kernel 963 * 964 * This function reserves memory area given in "crashkernel=" kernel command 965 * line parameter. The memory reserved is used by dump capture kernel when 966 * primary kernel is crashing. 967 */ 968 static void __init reserve_crashkernel(void) 969 { 970 unsigned long long crash_base = 0; 971 unsigned long long crash_size = 0; 972 unsigned long search_start = memblock_start_of_DRAM(); 973 unsigned long search_end = memblock_end_of_DRAM(); 974 975 int ret = 0; 976 977 /* 978 * Don't reserve a region for a crash kernel on a crash kernel 979 * since it doesn't make much sense and we have limited memory 980 * resources. 981 */ 982 if (is_kdump_kernel()) { 983 pr_info("crashkernel: ignoring reservation request\n"); 984 return; 985 } 986 987 ret = parse_crashkernel(boot_command_line, memblock_phys_mem_size(), 988 &crash_size, &crash_base); 989 if (ret || !crash_size) 990 return; 991 992 crash_size = PAGE_ALIGN(crash_size); 993 994 if (crash_base) { 995 search_start = crash_base; 996 search_end = crash_base + crash_size; 997 } 998 999 /* 1000 * Current riscv boot protocol requires 2MB alignment for 1001 * RV64 and 4MB alignment for RV32 (hugepage size) 1002 * 1003 * Try to alloc from 32bit addressible physical memory so that 1004 * swiotlb can work on the crash kernel. 1005 */ 1006 crash_base = memblock_phys_alloc_range(crash_size, PMD_SIZE, 1007 search_start, 1008 min(search_end, (unsigned long) SZ_4G)); 1009 if (crash_base == 0) { 1010 /* Try again without restricting region to 32bit addressible memory */ 1011 crash_base = memblock_phys_alloc_range(crash_size, PMD_SIZE, 1012 search_start, search_end); 1013 if (crash_base == 0) { 1014 pr_warn("crashkernel: couldn't allocate %lldKB\n", 1015 crash_size >> 10); 1016 return; 1017 } 1018 } 1019 1020 pr_info("crashkernel: reserved 0x%016llx - 0x%016llx (%lld MB)\n", 1021 crash_base, crash_base + crash_size, crash_size >> 20); 1022 1023 crashk_res.start = crash_base; 1024 crashk_res.end = crash_base + crash_size - 1; 1025 } 1026 #endif /* CONFIG_KEXEC_CORE */ 1027 1028 void __init paging_init(void) 1029 { 1030 setup_bootmem(); 1031 setup_vm_final(); 1032 } 1033 1034 void __init misc_mem_init(void) 1035 { 1036 early_memtest(min_low_pfn << PAGE_SHIFT, max_low_pfn << PAGE_SHIFT); 1037 arch_numa_init(); 1038 sparse_init(); 1039 zone_sizes_init(); 1040 #ifdef CONFIG_KEXEC_CORE 1041 reserve_crashkernel(); 1042 #endif 1043 memblock_dump_all(); 1044 } 1045 1046 #ifdef CONFIG_SPARSEMEM_VMEMMAP 1047 int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node, 1048 struct vmem_altmap *altmap) 1049 { 1050 return vmemmap_populate_basepages(start, end, node, NULL); 1051 } 1052 #endif 1053