xref: /openbmc/linux/arch/arm64/mm/mmu.c (revision c819e2cf)
1 /*
2  * Based on arch/arm/mm/mmu.c
3  *
4  * Copyright (C) 1995-2005 Russell King
5  * Copyright (C) 2012 ARM Ltd.
6  *
7  * This program is free software; you can redistribute it and/or modify
8  * it under the terms of the GNU General Public License version 2 as
9  * published by the Free Software Foundation.
10  *
11  * This program is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
14  * GNU General Public License for more details.
15  *
16  * You should have received a copy of the GNU General Public License
17  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
18  */
19 
20 #include <linux/export.h>
21 #include <linux/kernel.h>
22 #include <linux/errno.h>
23 #include <linux/init.h>
24 #include <linux/mman.h>
25 #include <linux/nodemask.h>
26 #include <linux/memblock.h>
27 #include <linux/fs.h>
28 #include <linux/io.h>
29 
30 #include <asm/cputype.h>
31 #include <asm/fixmap.h>
32 #include <asm/sections.h>
33 #include <asm/setup.h>
34 #include <asm/sizes.h>
35 #include <asm/tlb.h>
36 #include <asm/memblock.h>
37 #include <asm/mmu_context.h>
38 
39 #include "mm.h"
40 
41 /*
42  * Empty_zero_page is a special page that is used for zero-initialized data
43  * and COW.
44  */
45 struct page *empty_zero_page;
46 EXPORT_SYMBOL(empty_zero_page);
47 
48 struct cachepolicy {
49 	const char	policy[16];
50 	u64		mair;
51 	u64		tcr;
52 };
53 
54 static struct cachepolicy cache_policies[] __initdata = {
55 	{
56 		.policy		= "uncached",
57 		.mair		= 0x44,			/* inner, outer non-cacheable */
58 		.tcr		= TCR_IRGN_NC | TCR_ORGN_NC,
59 	}, {
60 		.policy		= "writethrough",
61 		.mair		= 0xaa,			/* inner, outer write-through, read-allocate */
62 		.tcr		= TCR_IRGN_WT | TCR_ORGN_WT,
63 	}, {
64 		.policy		= "writeback",
65 		.mair		= 0xee,			/* inner, outer write-back, read-allocate */
66 		.tcr		= TCR_IRGN_WBnWA | TCR_ORGN_WBnWA,
67 	}
68 };
69 
70 /*
71  * These are useful for identifying cache coherency problems by allowing the
72  * cache or the cache and writebuffer to be turned off. It changes the Normal
73  * memory caching attributes in the MAIR_EL1 register.
74  */
75 static int __init early_cachepolicy(char *p)
76 {
77 	int i;
78 	u64 tmp;
79 
80 	for (i = 0; i < ARRAY_SIZE(cache_policies); i++) {
81 		int len = strlen(cache_policies[i].policy);
82 
83 		if (memcmp(p, cache_policies[i].policy, len) == 0)
84 			break;
85 	}
86 	if (i == ARRAY_SIZE(cache_policies)) {
87 		pr_err("ERROR: unknown or unsupported cache policy: %s\n", p);
88 		return 0;
89 	}
90 
91 	flush_cache_all();
92 
93 	/*
94 	 * Modify MT_NORMAL attributes in MAIR_EL1.
95 	 */
96 	asm volatile(
97 	"	mrs	%0, mair_el1\n"
98 	"	bfi	%0, %1, %2, #8\n"
99 	"	msr	mair_el1, %0\n"
100 	"	isb\n"
101 	: "=&r" (tmp)
102 	: "r" (cache_policies[i].mair), "i" (MT_NORMAL * 8));
103 
104 	/*
105 	 * Modify TCR PTW cacheability attributes.
106 	 */
107 	asm volatile(
108 	"	mrs	%0, tcr_el1\n"
109 	"	bic	%0, %0, %2\n"
110 	"	orr	%0, %0, %1\n"
111 	"	msr	tcr_el1, %0\n"
112 	"	isb\n"
113 	: "=&r" (tmp)
114 	: "r" (cache_policies[i].tcr), "r" (TCR_IRGN_MASK | TCR_ORGN_MASK));
115 
116 	flush_cache_all();
117 
118 	return 0;
119 }
120 early_param("cachepolicy", early_cachepolicy);
121 
122 pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn,
123 			      unsigned long size, pgprot_t vma_prot)
124 {
125 	if (!pfn_valid(pfn))
126 		return pgprot_noncached(vma_prot);
127 	else if (file->f_flags & O_SYNC)
128 		return pgprot_writecombine(vma_prot);
129 	return vma_prot;
130 }
131 EXPORT_SYMBOL(phys_mem_access_prot);
132 
133 static void __init *early_alloc(unsigned long sz)
134 {
135 	void *ptr = __va(memblock_alloc(sz, sz));
136 	memset(ptr, 0, sz);
137 	return ptr;
138 }
139 
140 static void __init alloc_init_pte(pmd_t *pmd, unsigned long addr,
141 				  unsigned long end, unsigned long pfn,
142 				  pgprot_t prot)
143 {
144 	pte_t *pte;
145 
146 	if (pmd_none(*pmd)) {
147 		pte = early_alloc(PTRS_PER_PTE * sizeof(pte_t));
148 		__pmd_populate(pmd, __pa(pte), PMD_TYPE_TABLE);
149 	}
150 	BUG_ON(pmd_bad(*pmd));
151 
152 	pte = pte_offset_kernel(pmd, addr);
153 	do {
154 		set_pte(pte, pfn_pte(pfn, prot));
155 		pfn++;
156 	} while (pte++, addr += PAGE_SIZE, addr != end);
157 }
158 
159 static void __init alloc_init_pmd(pud_t *pud, unsigned long addr,
160 				  unsigned long end, phys_addr_t phys,
161 				  int map_io)
162 {
163 	pmd_t *pmd;
164 	unsigned long next;
165 	pmdval_t prot_sect;
166 	pgprot_t prot_pte;
167 
168 	if (map_io) {
169 		prot_sect = PROT_SECT_DEVICE_nGnRE;
170 		prot_pte = __pgprot(PROT_DEVICE_nGnRE);
171 	} else {
172 		prot_sect = PROT_SECT_NORMAL_EXEC;
173 		prot_pte = PAGE_KERNEL_EXEC;
174 	}
175 
176 	/*
177 	 * Check for initial section mappings in the pgd/pud and remove them.
178 	 */
179 	if (pud_none(*pud) || pud_bad(*pud)) {
180 		pmd = early_alloc(PTRS_PER_PMD * sizeof(pmd_t));
181 		pud_populate(&init_mm, pud, pmd);
182 	}
183 
184 	pmd = pmd_offset(pud, addr);
185 	do {
186 		next = pmd_addr_end(addr, end);
187 		/* try section mapping first */
188 		if (((addr | next | phys) & ~SECTION_MASK) == 0) {
189 			pmd_t old_pmd =*pmd;
190 			set_pmd(pmd, __pmd(phys | prot_sect));
191 			/*
192 			 * Check for previous table entries created during
193 			 * boot (__create_page_tables) and flush them.
194 			 */
195 			if (!pmd_none(old_pmd))
196 				flush_tlb_all();
197 		} else {
198 			alloc_init_pte(pmd, addr, next, __phys_to_pfn(phys),
199 				       prot_pte);
200 		}
201 		phys += next - addr;
202 	} while (pmd++, addr = next, addr != end);
203 }
204 
205 static void __init alloc_init_pud(pgd_t *pgd, unsigned long addr,
206 				  unsigned long end, phys_addr_t phys,
207 				  int map_io)
208 {
209 	pud_t *pud;
210 	unsigned long next;
211 
212 	if (pgd_none(*pgd)) {
213 		pud = early_alloc(PTRS_PER_PUD * sizeof(pud_t));
214 		pgd_populate(&init_mm, pgd, pud);
215 	}
216 	BUG_ON(pgd_bad(*pgd));
217 
218 	pud = pud_offset(pgd, addr);
219 	do {
220 		next = pud_addr_end(addr, end);
221 
222 		/*
223 		 * For 4K granule only, attempt to put down a 1GB block
224 		 */
225 		if (!map_io && (PAGE_SHIFT == 12) &&
226 		    ((addr | next | phys) & ~PUD_MASK) == 0) {
227 			pud_t old_pud = *pud;
228 			set_pud(pud, __pud(phys | PROT_SECT_NORMAL_EXEC));
229 
230 			/*
231 			 * If we have an old value for a pud, it will
232 			 * be pointing to a pmd table that we no longer
233 			 * need (from swapper_pg_dir).
234 			 *
235 			 * Look up the old pmd table and free it.
236 			 */
237 			if (!pud_none(old_pud)) {
238 				phys_addr_t table = __pa(pmd_offset(&old_pud, 0));
239 				memblock_free(table, PAGE_SIZE);
240 				flush_tlb_all();
241 			}
242 		} else {
243 			alloc_init_pmd(pud, addr, next, phys, map_io);
244 		}
245 		phys += next - addr;
246 	} while (pud++, addr = next, addr != end);
247 }
248 
249 /*
250  * Create the page directory entries and any necessary page tables for the
251  * mapping specified by 'md'.
252  */
253 static void __init __create_mapping(pgd_t *pgd, phys_addr_t phys,
254 				    unsigned long virt, phys_addr_t size,
255 				    int map_io)
256 {
257 	unsigned long addr, length, end, next;
258 
259 	addr = virt & PAGE_MASK;
260 	length = PAGE_ALIGN(size + (virt & ~PAGE_MASK));
261 
262 	end = addr + length;
263 	do {
264 		next = pgd_addr_end(addr, end);
265 		alloc_init_pud(pgd, addr, next, phys, map_io);
266 		phys += next - addr;
267 	} while (pgd++, addr = next, addr != end);
268 }
269 
270 static void __init create_mapping(phys_addr_t phys, unsigned long virt,
271 				  phys_addr_t size)
272 {
273 	if (virt < VMALLOC_START) {
274 		pr_warn("BUG: not creating mapping for %pa at 0x%016lx - outside kernel range\n",
275 			&phys, virt);
276 		return;
277 	}
278 	__create_mapping(pgd_offset_k(virt & PAGE_MASK), phys, virt, size, 0);
279 }
280 
281 void __init create_id_mapping(phys_addr_t addr, phys_addr_t size, int map_io)
282 {
283 	if ((addr >> PGDIR_SHIFT) >= ARRAY_SIZE(idmap_pg_dir)) {
284 		pr_warn("BUG: not creating id mapping for %pa\n", &addr);
285 		return;
286 	}
287 	__create_mapping(&idmap_pg_dir[pgd_index(addr)],
288 			 addr, addr, size, map_io);
289 }
290 
291 static void __init map_mem(void)
292 {
293 	struct memblock_region *reg;
294 	phys_addr_t limit;
295 
296 	/*
297 	 * Temporarily limit the memblock range. We need to do this as
298 	 * create_mapping requires puds, pmds and ptes to be allocated from
299 	 * memory addressable from the initial direct kernel mapping.
300 	 *
301 	 * The initial direct kernel mapping, located at swapper_pg_dir, gives
302 	 * us PUD_SIZE (4K pages) or PMD_SIZE (64K pages) memory starting from
303 	 * PHYS_OFFSET (which must be aligned to 2MB as per
304 	 * Documentation/arm64/booting.txt).
305 	 */
306 	if (IS_ENABLED(CONFIG_ARM64_64K_PAGES))
307 		limit = PHYS_OFFSET + PMD_SIZE;
308 	else
309 		limit = PHYS_OFFSET + PUD_SIZE;
310 	memblock_set_current_limit(limit);
311 
312 	/* map all the memory banks */
313 	for_each_memblock(memory, reg) {
314 		phys_addr_t start = reg->base;
315 		phys_addr_t end = start + reg->size;
316 
317 		if (start >= end)
318 			break;
319 
320 #ifndef CONFIG_ARM64_64K_PAGES
321 		/*
322 		 * For the first memory bank align the start address and
323 		 * current memblock limit to prevent create_mapping() from
324 		 * allocating pte page tables from unmapped memory.
325 		 * When 64K pages are enabled, the pte page table for the
326 		 * first PGDIR_SIZE is already present in swapper_pg_dir.
327 		 */
328 		if (start < limit)
329 			start = ALIGN(start, PMD_SIZE);
330 		if (end < limit) {
331 			limit = end & PMD_MASK;
332 			memblock_set_current_limit(limit);
333 		}
334 #endif
335 
336 		create_mapping(start, __phys_to_virt(start), end - start);
337 	}
338 
339 	/* Limit no longer required. */
340 	memblock_set_current_limit(MEMBLOCK_ALLOC_ANYWHERE);
341 }
342 
343 /*
344  * paging_init() sets up the page tables, initialises the zone memory
345  * maps and sets up the zero page.
346  */
347 void __init paging_init(void)
348 {
349 	void *zero_page;
350 
351 	map_mem();
352 
353 	/*
354 	 * Finally flush the caches and tlb to ensure that we're in a
355 	 * consistent state.
356 	 */
357 	flush_cache_all();
358 	flush_tlb_all();
359 
360 	/* allocate the zero page. */
361 	zero_page = early_alloc(PAGE_SIZE);
362 
363 	bootmem_init();
364 
365 	empty_zero_page = virt_to_page(zero_page);
366 
367 	/*
368 	 * TTBR0 is only used for the identity mapping at this stage. Make it
369 	 * point to zero page to avoid speculatively fetching new entries.
370 	 */
371 	cpu_set_reserved_ttbr0();
372 	flush_tlb_all();
373 }
374 
375 /*
376  * Enable the identity mapping to allow the MMU disabling.
377  */
378 void setup_mm_for_reboot(void)
379 {
380 	cpu_switch_mm(idmap_pg_dir, &init_mm);
381 	flush_tlb_all();
382 }
383 
384 /*
385  * Check whether a kernel address is valid (derived from arch/x86/).
386  */
387 int kern_addr_valid(unsigned long addr)
388 {
389 	pgd_t *pgd;
390 	pud_t *pud;
391 	pmd_t *pmd;
392 	pte_t *pte;
393 
394 	if ((((long)addr) >> VA_BITS) != -1UL)
395 		return 0;
396 
397 	pgd = pgd_offset_k(addr);
398 	if (pgd_none(*pgd))
399 		return 0;
400 
401 	pud = pud_offset(pgd, addr);
402 	if (pud_none(*pud))
403 		return 0;
404 
405 	if (pud_sect(*pud))
406 		return pfn_valid(pud_pfn(*pud));
407 
408 	pmd = pmd_offset(pud, addr);
409 	if (pmd_none(*pmd))
410 		return 0;
411 
412 	if (pmd_sect(*pmd))
413 		return pfn_valid(pmd_pfn(*pmd));
414 
415 	pte = pte_offset_kernel(pmd, addr);
416 	if (pte_none(*pte))
417 		return 0;
418 
419 	return pfn_valid(pte_pfn(*pte));
420 }
421 #ifdef CONFIG_SPARSEMEM_VMEMMAP
422 #ifdef CONFIG_ARM64_64K_PAGES
423 int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node)
424 {
425 	return vmemmap_populate_basepages(start, end, node);
426 }
427 #else	/* !CONFIG_ARM64_64K_PAGES */
428 int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node)
429 {
430 	unsigned long addr = start;
431 	unsigned long next;
432 	pgd_t *pgd;
433 	pud_t *pud;
434 	pmd_t *pmd;
435 
436 	do {
437 		next = pmd_addr_end(addr, end);
438 
439 		pgd = vmemmap_pgd_populate(addr, node);
440 		if (!pgd)
441 			return -ENOMEM;
442 
443 		pud = vmemmap_pud_populate(pgd, addr, node);
444 		if (!pud)
445 			return -ENOMEM;
446 
447 		pmd = pmd_offset(pud, addr);
448 		if (pmd_none(*pmd)) {
449 			void *p = NULL;
450 
451 			p = vmemmap_alloc_block_buf(PMD_SIZE, node);
452 			if (!p)
453 				return -ENOMEM;
454 
455 			set_pmd(pmd, __pmd(__pa(p) | PROT_SECT_NORMAL));
456 		} else
457 			vmemmap_verify((pte_t *)pmd, node, addr, next);
458 	} while (addr = next, addr != end);
459 
460 	return 0;
461 }
462 #endif	/* CONFIG_ARM64_64K_PAGES */
463 void vmemmap_free(unsigned long start, unsigned long end)
464 {
465 }
466 #endif	/* CONFIG_SPARSEMEM_VMEMMAP */
467 
468 static pte_t bm_pte[PTRS_PER_PTE] __page_aligned_bss;
469 #if CONFIG_ARM64_PGTABLE_LEVELS > 2
470 static pmd_t bm_pmd[PTRS_PER_PMD] __page_aligned_bss;
471 #endif
472 #if CONFIG_ARM64_PGTABLE_LEVELS > 3
473 static pud_t bm_pud[PTRS_PER_PUD] __page_aligned_bss;
474 #endif
475 
476 static inline pud_t * fixmap_pud(unsigned long addr)
477 {
478 	pgd_t *pgd = pgd_offset_k(addr);
479 
480 	BUG_ON(pgd_none(*pgd) || pgd_bad(*pgd));
481 
482 	return pud_offset(pgd, addr);
483 }
484 
485 static inline pmd_t * fixmap_pmd(unsigned long addr)
486 {
487 	pud_t *pud = fixmap_pud(addr);
488 
489 	BUG_ON(pud_none(*pud) || pud_bad(*pud));
490 
491 	return pmd_offset(pud, addr);
492 }
493 
494 static inline pte_t * fixmap_pte(unsigned long addr)
495 {
496 	pmd_t *pmd = fixmap_pmd(addr);
497 
498 	BUG_ON(pmd_none(*pmd) || pmd_bad(*pmd));
499 
500 	return pte_offset_kernel(pmd, addr);
501 }
502 
503 void __init early_fixmap_init(void)
504 {
505 	pgd_t *pgd;
506 	pud_t *pud;
507 	pmd_t *pmd;
508 	unsigned long addr = FIXADDR_START;
509 
510 	pgd = pgd_offset_k(addr);
511 	pgd_populate(&init_mm, pgd, bm_pud);
512 	pud = pud_offset(pgd, addr);
513 	pud_populate(&init_mm, pud, bm_pmd);
514 	pmd = pmd_offset(pud, addr);
515 	pmd_populate_kernel(&init_mm, pmd, bm_pte);
516 
517 	/*
518 	 * The boot-ioremap range spans multiple pmds, for which
519 	 * we are not preparted:
520 	 */
521 	BUILD_BUG_ON((__fix_to_virt(FIX_BTMAP_BEGIN) >> PMD_SHIFT)
522 		     != (__fix_to_virt(FIX_BTMAP_END) >> PMD_SHIFT));
523 
524 	if ((pmd != fixmap_pmd(fix_to_virt(FIX_BTMAP_BEGIN)))
525 	     || pmd != fixmap_pmd(fix_to_virt(FIX_BTMAP_END))) {
526 		WARN_ON(1);
527 		pr_warn("pmd %p != %p, %p\n",
528 			pmd, fixmap_pmd(fix_to_virt(FIX_BTMAP_BEGIN)),
529 			fixmap_pmd(fix_to_virt(FIX_BTMAP_END)));
530 		pr_warn("fix_to_virt(FIX_BTMAP_BEGIN): %08lx\n",
531 			fix_to_virt(FIX_BTMAP_BEGIN));
532 		pr_warn("fix_to_virt(FIX_BTMAP_END):   %08lx\n",
533 			fix_to_virt(FIX_BTMAP_END));
534 
535 		pr_warn("FIX_BTMAP_END:       %d\n", FIX_BTMAP_END);
536 		pr_warn("FIX_BTMAP_BEGIN:     %d\n", FIX_BTMAP_BEGIN);
537 	}
538 }
539 
540 void __set_fixmap(enum fixed_addresses idx,
541 			       phys_addr_t phys, pgprot_t flags)
542 {
543 	unsigned long addr = __fix_to_virt(idx);
544 	pte_t *pte;
545 
546 	if (idx >= __end_of_fixed_addresses) {
547 		BUG();
548 		return;
549 	}
550 
551 	pte = fixmap_pte(addr);
552 
553 	if (pgprot_val(flags)) {
554 		set_pte(pte, pfn_pte(phys >> PAGE_SHIFT, flags));
555 	} else {
556 		pte_clear(&init_mm, addr, pte);
557 		flush_tlb_kernel_range(addr, addr+PAGE_SIZE);
558 	}
559 }
560