xref: /openbmc/linux/arch/riscv/mm/init.c (revision 6c33a6f4)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright (C) 2012 Regents of the University of California
4  * Copyright (C) 2019 Western Digital Corporation or its affiliates.
5  */
6 
7 #include <linux/init.h>
8 #include <linux/mm.h>
9 #include <linux/memblock.h>
10 #include <linux/initrd.h>
11 #include <linux/swap.h>
12 #include <linux/sizes.h>
13 #include <linux/of_fdt.h>
14 #include <linux/libfdt.h>
15 
16 #include <asm/fixmap.h>
17 #include <asm/tlbflush.h>
18 #include <asm/sections.h>
19 #include <asm/pgtable.h>
20 #include <asm/io.h>
21 
22 #include "../kernel/head.h"
23 
24 unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)]
25 							__page_aligned_bss;
26 EXPORT_SYMBOL(empty_zero_page);
27 
28 extern char _start[];
29 void *dtb_early_va;
30 
31 static void __init zone_sizes_init(void)
32 {
33 	unsigned long max_zone_pfns[MAX_NR_ZONES] = { 0, };
34 
35 #ifdef CONFIG_ZONE_DMA32
36 	max_zone_pfns[ZONE_DMA32] = PFN_DOWN(min(4UL * SZ_1G,
37 			(unsigned long) PFN_PHYS(max_low_pfn)));
38 #endif
39 	max_zone_pfns[ZONE_NORMAL] = max_low_pfn;
40 
41 	free_area_init_nodes(max_zone_pfns);
42 }
43 
44 static void setup_zero_page(void)
45 {
46 	memset((void *)empty_zero_page, 0, PAGE_SIZE);
47 }
48 
49 #ifdef CONFIG_DEBUG_VM
50 static inline void print_mlk(char *name, unsigned long b, unsigned long t)
51 {
52 	pr_notice("%12s : 0x%08lx - 0x%08lx   (%4ld kB)\n", name, b, t,
53 		  (((t) - (b)) >> 10));
54 }
55 
56 static inline void print_mlm(char *name, unsigned long b, unsigned long t)
57 {
58 	pr_notice("%12s : 0x%08lx - 0x%08lx   (%4ld MB)\n", name, b, t,
59 		  (((t) - (b)) >> 20));
60 }
61 
62 static void print_vm_layout(void)
63 {
64 	pr_notice("Virtual kernel memory layout:\n");
65 	print_mlk("fixmap", (unsigned long)FIXADDR_START,
66 		  (unsigned long)FIXADDR_TOP);
67 	print_mlm("pci io", (unsigned long)PCI_IO_START,
68 		  (unsigned long)PCI_IO_END);
69 	print_mlm("vmemmap", (unsigned long)VMEMMAP_START,
70 		  (unsigned long)VMEMMAP_END);
71 	print_mlm("vmalloc", (unsigned long)VMALLOC_START,
72 		  (unsigned long)VMALLOC_END);
73 	print_mlm("lowmem", (unsigned long)PAGE_OFFSET,
74 		  (unsigned long)high_memory);
75 }
76 #else
77 static void print_vm_layout(void) { }
78 #endif /* CONFIG_DEBUG_VM */
79 
80 void __init mem_init(void)
81 {
82 #ifdef CONFIG_FLATMEM
83 	BUG_ON(!mem_map);
84 #endif /* CONFIG_FLATMEM */
85 
86 	high_memory = (void *)(__va(PFN_PHYS(max_low_pfn)));
87 	memblock_free_all();
88 
89 	mem_init_print_info(NULL);
90 	print_vm_layout();
91 }
92 
93 #ifdef CONFIG_BLK_DEV_INITRD
94 static void __init setup_initrd(void)
95 {
96 	unsigned long size;
97 
98 	if (initrd_start >= initrd_end) {
99 		pr_info("initrd not found or empty");
100 		goto disable;
101 	}
102 	if (__pa_symbol(initrd_end) > PFN_PHYS(max_low_pfn)) {
103 		pr_err("initrd extends beyond end of memory");
104 		goto disable;
105 	}
106 
107 	size = initrd_end - initrd_start;
108 	memblock_reserve(__pa_symbol(initrd_start), size);
109 	initrd_below_start_ok = 1;
110 
111 	pr_info("Initial ramdisk at: 0x%p (%lu bytes)\n",
112 		(void *)(initrd_start), size);
113 	return;
114 disable:
115 	pr_cont(" - disabling initrd\n");
116 	initrd_start = 0;
117 	initrd_end = 0;
118 }
119 #endif /* CONFIG_BLK_DEV_INITRD */
120 
121 static phys_addr_t dtb_early_pa __initdata;
122 
123 void __init setup_bootmem(void)
124 {
125 	struct memblock_region *reg;
126 	phys_addr_t mem_size = 0;
127 	phys_addr_t vmlinux_end = __pa_symbol(&_end);
128 	phys_addr_t vmlinux_start = __pa_symbol(&_start);
129 
130 	/* Find the memory region containing the kernel */
131 	for_each_memblock(memory, reg) {
132 		phys_addr_t end = reg->base + reg->size;
133 
134 		if (reg->base <= vmlinux_start && vmlinux_end <= end) {
135 			mem_size = min(reg->size, (phys_addr_t)-PAGE_OFFSET);
136 
137 			/*
138 			 * Remove memblock from the end of usable area to the
139 			 * end of region
140 			 */
141 			if (reg->base + mem_size < end)
142 				memblock_remove(reg->base + mem_size,
143 						end - reg->base - mem_size);
144 		}
145 	}
146 	BUG_ON(mem_size == 0);
147 
148 	/* Reserve from the start of the kernel to the end of the kernel */
149 	memblock_reserve(vmlinux_start, vmlinux_end - vmlinux_start);
150 
151 	set_max_mapnr(PFN_DOWN(mem_size));
152 	max_low_pfn = PFN_DOWN(memblock_end_of_DRAM());
153 
154 #ifdef CONFIG_BLK_DEV_INITRD
155 	setup_initrd();
156 #endif /* CONFIG_BLK_DEV_INITRD */
157 
158 	/*
159 	 * Avoid using early_init_fdt_reserve_self() since __pa() does
160 	 * not work for DTB pointers that are fixmap addresses
161 	 */
162 	memblock_reserve(dtb_early_pa, fdt_totalsize(dtb_early_va));
163 
164 	early_init_fdt_scan_reserved_mem();
165 	memblock_allow_resize();
166 	memblock_dump_all();
167 
168 	for_each_memblock(memory, reg) {
169 		unsigned long start_pfn = memblock_region_memory_base_pfn(reg);
170 		unsigned long end_pfn = memblock_region_memory_end_pfn(reg);
171 
172 		memblock_set_node(PFN_PHYS(start_pfn),
173 				  PFN_PHYS(end_pfn - start_pfn),
174 				  &memblock.memory, 0);
175 	}
176 }
177 
178 #ifdef CONFIG_MMU
179 unsigned long va_pa_offset;
180 EXPORT_SYMBOL(va_pa_offset);
181 unsigned long pfn_base;
182 EXPORT_SYMBOL(pfn_base);
183 
184 pgd_t swapper_pg_dir[PTRS_PER_PGD] __page_aligned_bss;
185 pgd_t trampoline_pg_dir[PTRS_PER_PGD] __page_aligned_bss;
186 pte_t fixmap_pte[PTRS_PER_PTE] __page_aligned_bss;
187 static bool mmu_enabled;
188 
189 #define MAX_EARLY_MAPPING_SIZE	SZ_128M
190 
191 pgd_t early_pg_dir[PTRS_PER_PGD] __initdata __aligned(PAGE_SIZE);
192 
193 void __set_fixmap(enum fixed_addresses idx, phys_addr_t phys, pgprot_t prot)
194 {
195 	unsigned long addr = __fix_to_virt(idx);
196 	pte_t *ptep;
197 
198 	BUG_ON(idx <= FIX_HOLE || idx >= __end_of_fixed_addresses);
199 
200 	ptep = &fixmap_pte[pte_index(addr)];
201 
202 	if (pgprot_val(prot)) {
203 		set_pte(ptep, pfn_pte(phys >> PAGE_SHIFT, prot));
204 	} else {
205 		pte_clear(&init_mm, addr, ptep);
206 		local_flush_tlb_page(addr);
207 	}
208 }
209 
210 static pte_t *__init get_pte_virt(phys_addr_t pa)
211 {
212 	if (mmu_enabled) {
213 		clear_fixmap(FIX_PTE);
214 		return (pte_t *)set_fixmap_offset(FIX_PTE, pa);
215 	} else {
216 		return (pte_t *)((uintptr_t)pa);
217 	}
218 }
219 
220 static phys_addr_t __init alloc_pte(uintptr_t va)
221 {
222 	/*
223 	 * We only create PMD or PGD early mappings so we
224 	 * should never reach here with MMU disabled.
225 	 */
226 	BUG_ON(!mmu_enabled);
227 
228 	return memblock_phys_alloc(PAGE_SIZE, PAGE_SIZE);
229 }
230 
231 static void __init create_pte_mapping(pte_t *ptep,
232 				      uintptr_t va, phys_addr_t pa,
233 				      phys_addr_t sz, pgprot_t prot)
234 {
235 	uintptr_t pte_index = pte_index(va);
236 
237 	BUG_ON(sz != PAGE_SIZE);
238 
239 	if (pte_none(ptep[pte_index]))
240 		ptep[pte_index] = pfn_pte(PFN_DOWN(pa), prot);
241 }
242 
243 #ifndef __PAGETABLE_PMD_FOLDED
244 
245 pmd_t trampoline_pmd[PTRS_PER_PMD] __page_aligned_bss;
246 pmd_t fixmap_pmd[PTRS_PER_PMD] __page_aligned_bss;
247 
248 #if MAX_EARLY_MAPPING_SIZE < PGDIR_SIZE
249 #define NUM_EARLY_PMDS		1UL
250 #else
251 #define NUM_EARLY_PMDS		(1UL + MAX_EARLY_MAPPING_SIZE / PGDIR_SIZE)
252 #endif
253 pmd_t early_pmd[PTRS_PER_PMD * NUM_EARLY_PMDS] __initdata __aligned(PAGE_SIZE);
254 
255 static pmd_t *__init get_pmd_virt(phys_addr_t pa)
256 {
257 	if (mmu_enabled) {
258 		clear_fixmap(FIX_PMD);
259 		return (pmd_t *)set_fixmap_offset(FIX_PMD, pa);
260 	} else {
261 		return (pmd_t *)((uintptr_t)pa);
262 	}
263 }
264 
265 static phys_addr_t __init alloc_pmd(uintptr_t va)
266 {
267 	uintptr_t pmd_num;
268 
269 	if (mmu_enabled)
270 		return memblock_phys_alloc(PAGE_SIZE, PAGE_SIZE);
271 
272 	pmd_num = (va - PAGE_OFFSET) >> PGDIR_SHIFT;
273 	BUG_ON(pmd_num >= NUM_EARLY_PMDS);
274 	return (uintptr_t)&early_pmd[pmd_num * PTRS_PER_PMD];
275 }
276 
277 static void __init create_pmd_mapping(pmd_t *pmdp,
278 				      uintptr_t va, phys_addr_t pa,
279 				      phys_addr_t sz, pgprot_t prot)
280 {
281 	pte_t *ptep;
282 	phys_addr_t pte_phys;
283 	uintptr_t pmd_index = pmd_index(va);
284 
285 	if (sz == PMD_SIZE) {
286 		if (pmd_none(pmdp[pmd_index]))
287 			pmdp[pmd_index] = pfn_pmd(PFN_DOWN(pa), prot);
288 		return;
289 	}
290 
291 	if (pmd_none(pmdp[pmd_index])) {
292 		pte_phys = alloc_pte(va);
293 		pmdp[pmd_index] = pfn_pmd(PFN_DOWN(pte_phys), PAGE_TABLE);
294 		ptep = get_pte_virt(pte_phys);
295 		memset(ptep, 0, PAGE_SIZE);
296 	} else {
297 		pte_phys = PFN_PHYS(_pmd_pfn(pmdp[pmd_index]));
298 		ptep = get_pte_virt(pte_phys);
299 	}
300 
301 	create_pte_mapping(ptep, va, pa, sz, prot);
302 }
303 
304 #define pgd_next_t		pmd_t
305 #define alloc_pgd_next(__va)	alloc_pmd(__va)
306 #define get_pgd_next_virt(__pa)	get_pmd_virt(__pa)
307 #define create_pgd_next_mapping(__nextp, __va, __pa, __sz, __prot)	\
308 	create_pmd_mapping(__nextp, __va, __pa, __sz, __prot)
309 #define fixmap_pgd_next		fixmap_pmd
310 #else
311 #define pgd_next_t		pte_t
312 #define alloc_pgd_next(__va)	alloc_pte(__va)
313 #define get_pgd_next_virt(__pa)	get_pte_virt(__pa)
314 #define create_pgd_next_mapping(__nextp, __va, __pa, __sz, __prot)	\
315 	create_pte_mapping(__nextp, __va, __pa, __sz, __prot)
316 #define fixmap_pgd_next		fixmap_pte
317 #endif
318 
319 static void __init create_pgd_mapping(pgd_t *pgdp,
320 				      uintptr_t va, phys_addr_t pa,
321 				      phys_addr_t sz, pgprot_t prot)
322 {
323 	pgd_next_t *nextp;
324 	phys_addr_t next_phys;
325 	uintptr_t pgd_index = pgd_index(va);
326 
327 	if (sz == PGDIR_SIZE) {
328 		if (pgd_val(pgdp[pgd_index]) == 0)
329 			pgdp[pgd_index] = pfn_pgd(PFN_DOWN(pa), prot);
330 		return;
331 	}
332 
333 	if (pgd_val(pgdp[pgd_index]) == 0) {
334 		next_phys = alloc_pgd_next(va);
335 		pgdp[pgd_index] = pfn_pgd(PFN_DOWN(next_phys), PAGE_TABLE);
336 		nextp = get_pgd_next_virt(next_phys);
337 		memset(nextp, 0, PAGE_SIZE);
338 	} else {
339 		next_phys = PFN_PHYS(_pgd_pfn(pgdp[pgd_index]));
340 		nextp = get_pgd_next_virt(next_phys);
341 	}
342 
343 	create_pgd_next_mapping(nextp, va, pa, sz, prot);
344 }
345 
346 static uintptr_t __init best_map_size(phys_addr_t base, phys_addr_t size)
347 {
348 	/* Upgrade to PMD_SIZE mappings whenever possible */
349 	if ((base & (PMD_SIZE - 1)) || (size & (PMD_SIZE - 1)))
350 		return PAGE_SIZE;
351 
352 	return PMD_SIZE;
353 }
354 
355 /*
356  * setup_vm() is called from head.S with MMU-off.
357  *
358  * Following requirements should be honoured for setup_vm() to work
359  * correctly:
360  * 1) It should use PC-relative addressing for accessing kernel symbols.
361  *    To achieve this we always use GCC cmodel=medany.
362  * 2) The compiler instrumentation for FTRACE will not work for setup_vm()
363  *    so disable compiler instrumentation when FTRACE is enabled.
364  *
365  * Currently, the above requirements are honoured by using custom CFLAGS
366  * for init.o in mm/Makefile.
367  */
368 
369 #ifndef __riscv_cmodel_medany
370 #error "setup_vm() is called from head.S before relocate so it should not use absolute addressing."
371 #endif
372 
373 asmlinkage void __init setup_vm(uintptr_t dtb_pa)
374 {
375 	uintptr_t va, end_va;
376 	uintptr_t load_pa = (uintptr_t)(&_start);
377 	uintptr_t load_sz = (uintptr_t)(&_end) - load_pa;
378 	uintptr_t map_size = best_map_size(load_pa, MAX_EARLY_MAPPING_SIZE);
379 
380 	va_pa_offset = PAGE_OFFSET - load_pa;
381 	pfn_base = PFN_DOWN(load_pa);
382 
383 	/*
384 	 * Enforce boot alignment requirements of RV32 and
385 	 * RV64 by only allowing PMD or PGD mappings.
386 	 */
387 	BUG_ON(map_size == PAGE_SIZE);
388 
389 	/* Sanity check alignment and size */
390 	BUG_ON((PAGE_OFFSET % PGDIR_SIZE) != 0);
391 	BUG_ON((load_pa % map_size) != 0);
392 	BUG_ON(load_sz > MAX_EARLY_MAPPING_SIZE);
393 
394 	/* Setup early PGD for fixmap */
395 	create_pgd_mapping(early_pg_dir, FIXADDR_START,
396 			   (uintptr_t)fixmap_pgd_next, PGDIR_SIZE, PAGE_TABLE);
397 
398 #ifndef __PAGETABLE_PMD_FOLDED
399 	/* Setup fixmap PMD */
400 	create_pmd_mapping(fixmap_pmd, FIXADDR_START,
401 			   (uintptr_t)fixmap_pte, PMD_SIZE, PAGE_TABLE);
402 	/* Setup trampoline PGD and PMD */
403 	create_pgd_mapping(trampoline_pg_dir, PAGE_OFFSET,
404 			   (uintptr_t)trampoline_pmd, PGDIR_SIZE, PAGE_TABLE);
405 	create_pmd_mapping(trampoline_pmd, PAGE_OFFSET,
406 			   load_pa, PMD_SIZE, PAGE_KERNEL_EXEC);
407 #else
408 	/* Setup trampoline PGD */
409 	create_pgd_mapping(trampoline_pg_dir, PAGE_OFFSET,
410 			   load_pa, PGDIR_SIZE, PAGE_KERNEL_EXEC);
411 #endif
412 
413 	/*
414 	 * Setup early PGD covering entire kernel which will allows
415 	 * us to reach paging_init(). We map all memory banks later
416 	 * in setup_vm_final() below.
417 	 */
418 	end_va = PAGE_OFFSET + load_sz;
419 	for (va = PAGE_OFFSET; va < end_va; va += map_size)
420 		create_pgd_mapping(early_pg_dir, va,
421 				   load_pa + (va - PAGE_OFFSET),
422 				   map_size, PAGE_KERNEL_EXEC);
423 
424 	/* Create fixed mapping for early FDT parsing */
425 	end_va = __fix_to_virt(FIX_FDT) + FIX_FDT_SIZE;
426 	for (va = __fix_to_virt(FIX_FDT); va < end_va; va += PAGE_SIZE)
427 		create_pte_mapping(fixmap_pte, va,
428 				   dtb_pa + (va - __fix_to_virt(FIX_FDT)),
429 				   PAGE_SIZE, PAGE_KERNEL);
430 
431 	/* Save pointer to DTB for early FDT parsing */
432 	dtb_early_va = (void *)fix_to_virt(FIX_FDT) + (dtb_pa & ~PAGE_MASK);
433 	/* Save physical address for memblock reservation */
434 	dtb_early_pa = dtb_pa;
435 }
436 
437 static void __init setup_vm_final(void)
438 {
439 	uintptr_t va, map_size;
440 	phys_addr_t pa, start, end;
441 	struct memblock_region *reg;
442 
443 	/* Set mmu_enabled flag */
444 	mmu_enabled = true;
445 
446 	/* Setup swapper PGD for fixmap */
447 	create_pgd_mapping(swapper_pg_dir, FIXADDR_START,
448 			   __pa_symbol(fixmap_pgd_next),
449 			   PGDIR_SIZE, PAGE_TABLE);
450 
451 	/* Map all memory banks */
452 	for_each_memblock(memory, reg) {
453 		start = reg->base;
454 		end = start + reg->size;
455 
456 		if (start >= end)
457 			break;
458 		if (memblock_is_nomap(reg))
459 			continue;
460 		if (start <= __pa(PAGE_OFFSET) &&
461 		    __pa(PAGE_OFFSET) < end)
462 			start = __pa(PAGE_OFFSET);
463 
464 		map_size = best_map_size(start, end - start);
465 		for (pa = start; pa < end; pa += map_size) {
466 			va = (uintptr_t)__va(pa);
467 			create_pgd_mapping(swapper_pg_dir, va, pa,
468 					   map_size, PAGE_KERNEL_EXEC);
469 		}
470 	}
471 
472 	/* Clear fixmap PTE and PMD mappings */
473 	clear_fixmap(FIX_PTE);
474 	clear_fixmap(FIX_PMD);
475 
476 	/* Move to swapper page table */
477 	csr_write(CSR_SATP, PFN_DOWN(__pa_symbol(swapper_pg_dir)) | SATP_MODE);
478 	local_flush_tlb_all();
479 }
480 #else
481 asmlinkage void __init setup_vm(uintptr_t dtb_pa)
482 {
483 	dtb_early_va = (void *)dtb_pa;
484 }
485 
486 static inline void setup_vm_final(void)
487 {
488 }
489 #endif /* CONFIG_MMU */
490 
491 void __init paging_init(void)
492 {
493 	setup_vm_final();
494 	memblocks_present();
495 	sparse_init();
496 	setup_zero_page();
497 	zone_sizes_init();
498 }
499 
500 #ifdef CONFIG_SPARSEMEM_VMEMMAP
501 int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node,
502 			       struct vmem_altmap *altmap)
503 {
504 	return vmemmap_populate_basepages(start, end, node);
505 }
506 #endif
507