xref: /openbmc/linux/arch/arm/mm/init.c (revision c819e2cf)
1 /*
2  *  linux/arch/arm/mm/init.c
3  *
4  *  Copyright (C) 1995-2005 Russell King
5  *
6  * This program is free software; you can redistribute it and/or modify
7  * it under the terms of the GNU General Public License version 2 as
8  * published by the Free Software Foundation.
9  */
10 #include <linux/kernel.h>
11 #include <linux/errno.h>
12 #include <linux/swap.h>
13 #include <linux/init.h>
14 #include <linux/bootmem.h>
15 #include <linux/mman.h>
16 #include <linux/export.h>
17 #include <linux/nodemask.h>
18 #include <linux/initrd.h>
19 #include <linux/of_fdt.h>
20 #include <linux/highmem.h>
21 #include <linux/gfp.h>
22 #include <linux/memblock.h>
23 #include <linux/dma-contiguous.h>
24 #include <linux/sizes.h>
25 
26 #include <asm/cp15.h>
27 #include <asm/mach-types.h>
28 #include <asm/memblock.h>
29 #include <asm/prom.h>
30 #include <asm/sections.h>
31 #include <asm/setup.h>
32 #include <asm/system_info.h>
33 #include <asm/tlb.h>
34 #include <asm/fixmap.h>
35 
36 #include <asm/mach/arch.h>
37 #include <asm/mach/map.h>
38 
39 #include "mm.h"
40 
41 #ifdef CONFIG_CPU_CP15_MMU
42 unsigned long __init __clear_cr(unsigned long mask)
43 {
44 	cr_alignment = cr_alignment & ~mask;
45 	return cr_alignment;
46 }
47 #endif
48 
49 static phys_addr_t phys_initrd_start __initdata = 0;
50 static unsigned long phys_initrd_size __initdata = 0;
51 
52 static int __init early_initrd(char *p)
53 {
54 	phys_addr_t start;
55 	unsigned long size;
56 	char *endp;
57 
58 	start = memparse(p, &endp);
59 	if (*endp == ',') {
60 		size = memparse(endp + 1, NULL);
61 
62 		phys_initrd_start = start;
63 		phys_initrd_size = size;
64 	}
65 	return 0;
66 }
67 early_param("initrd", early_initrd);
68 
69 static int __init parse_tag_initrd(const struct tag *tag)
70 {
71 	pr_warn("ATAG_INITRD is deprecated; "
72 		"please update your bootloader.\n");
73 	phys_initrd_start = __virt_to_phys(tag->u.initrd.start);
74 	phys_initrd_size = tag->u.initrd.size;
75 	return 0;
76 }
77 
78 __tagtable(ATAG_INITRD, parse_tag_initrd);
79 
80 static int __init parse_tag_initrd2(const struct tag *tag)
81 {
82 	phys_initrd_start = tag->u.initrd.start;
83 	phys_initrd_size = tag->u.initrd.size;
84 	return 0;
85 }
86 
87 __tagtable(ATAG_INITRD2, parse_tag_initrd2);
88 
89 /*
90  * This keeps memory configuration data used by a couple memory
91  * initialization functions, as well as show_mem() for the skipping
92  * of holes in the memory map.  It is populated by arm_add_memory().
93  */
94 void show_mem(unsigned int filter)
95 {
96 	int free = 0, total = 0, reserved = 0;
97 	int shared = 0, cached = 0, slab = 0;
98 	struct memblock_region *reg;
99 
100 	printk("Mem-info:\n");
101 	show_free_areas(filter);
102 
103 	for_each_memblock (memory, reg) {
104 		unsigned int pfn1, pfn2;
105 		struct page *page, *end;
106 
107 		pfn1 = memblock_region_memory_base_pfn(reg);
108 		pfn2 = memblock_region_memory_end_pfn(reg);
109 
110 		page = pfn_to_page(pfn1);
111 		end  = pfn_to_page(pfn2 - 1) + 1;
112 
113 		do {
114 			total++;
115 			if (PageReserved(page))
116 				reserved++;
117 			else if (PageSwapCache(page))
118 				cached++;
119 			else if (PageSlab(page))
120 				slab++;
121 			else if (!page_count(page))
122 				free++;
123 			else
124 				shared += page_count(page) - 1;
125 			pfn1++;
126 			page = pfn_to_page(pfn1);
127 		} while (pfn1 < pfn2);
128 	}
129 
130 	printk("%d pages of RAM\n", total);
131 	printk("%d free pages\n", free);
132 	printk("%d reserved pages\n", reserved);
133 	printk("%d slab pages\n", slab);
134 	printk("%d pages shared\n", shared);
135 	printk("%d pages swap cached\n", cached);
136 }
137 
138 static void __init find_limits(unsigned long *min, unsigned long *max_low,
139 			       unsigned long *max_high)
140 {
141 	*max_low = PFN_DOWN(memblock_get_current_limit());
142 	*min = PFN_UP(memblock_start_of_DRAM());
143 	*max_high = PFN_DOWN(memblock_end_of_DRAM());
144 }
145 
146 #ifdef CONFIG_ZONE_DMA
147 
148 phys_addr_t arm_dma_zone_size __read_mostly;
149 EXPORT_SYMBOL(arm_dma_zone_size);
150 
151 /*
152  * The DMA mask corresponding to the maximum bus address allocatable
153  * using GFP_DMA.  The default here places no restriction on DMA
154  * allocations.  This must be the smallest DMA mask in the system,
155  * so a successful GFP_DMA allocation will always satisfy this.
156  */
157 phys_addr_t arm_dma_limit;
158 unsigned long arm_dma_pfn_limit;
159 
160 static void __init arm_adjust_dma_zone(unsigned long *size, unsigned long *hole,
161 	unsigned long dma_size)
162 {
163 	if (size[0] <= dma_size)
164 		return;
165 
166 	size[ZONE_NORMAL] = size[0] - dma_size;
167 	size[ZONE_DMA] = dma_size;
168 	hole[ZONE_NORMAL] = hole[0];
169 	hole[ZONE_DMA] = 0;
170 }
171 #endif
172 
173 void __init setup_dma_zone(const struct machine_desc *mdesc)
174 {
175 #ifdef CONFIG_ZONE_DMA
176 	if (mdesc->dma_zone_size) {
177 		arm_dma_zone_size = mdesc->dma_zone_size;
178 		arm_dma_limit = PHYS_OFFSET + arm_dma_zone_size - 1;
179 	} else
180 		arm_dma_limit = 0xffffffff;
181 	arm_dma_pfn_limit = arm_dma_limit >> PAGE_SHIFT;
182 #endif
183 }
184 
185 static void __init zone_sizes_init(unsigned long min, unsigned long max_low,
186 	unsigned long max_high)
187 {
188 	unsigned long zone_size[MAX_NR_ZONES], zhole_size[MAX_NR_ZONES];
189 	struct memblock_region *reg;
190 
191 	/*
192 	 * initialise the zones.
193 	 */
194 	memset(zone_size, 0, sizeof(zone_size));
195 
196 	/*
197 	 * The memory size has already been determined.  If we need
198 	 * to do anything fancy with the allocation of this memory
199 	 * to the zones, now is the time to do it.
200 	 */
201 	zone_size[0] = max_low - min;
202 #ifdef CONFIG_HIGHMEM
203 	zone_size[ZONE_HIGHMEM] = max_high - max_low;
204 #endif
205 
206 	/*
207 	 * Calculate the size of the holes.
208 	 *  holes = node_size - sum(bank_sizes)
209 	 */
210 	memcpy(zhole_size, zone_size, sizeof(zhole_size));
211 	for_each_memblock(memory, reg) {
212 		unsigned long start = memblock_region_memory_base_pfn(reg);
213 		unsigned long end = memblock_region_memory_end_pfn(reg);
214 
215 		if (start < max_low) {
216 			unsigned long low_end = min(end, max_low);
217 			zhole_size[0] -= low_end - start;
218 		}
219 #ifdef CONFIG_HIGHMEM
220 		if (end > max_low) {
221 			unsigned long high_start = max(start, max_low);
222 			zhole_size[ZONE_HIGHMEM] -= end - high_start;
223 		}
224 #endif
225 	}
226 
227 #ifdef CONFIG_ZONE_DMA
228 	/*
229 	 * Adjust the sizes according to any special requirements for
230 	 * this machine type.
231 	 */
232 	if (arm_dma_zone_size)
233 		arm_adjust_dma_zone(zone_size, zhole_size,
234 			arm_dma_zone_size >> PAGE_SHIFT);
235 #endif
236 
237 	free_area_init_node(0, zone_size, min, zhole_size);
238 }
239 
240 #ifdef CONFIG_HAVE_ARCH_PFN_VALID
241 int pfn_valid(unsigned long pfn)
242 {
243 	return memblock_is_memory(__pfn_to_phys(pfn));
244 }
245 EXPORT_SYMBOL(pfn_valid);
246 #endif
247 
248 #ifndef CONFIG_SPARSEMEM
249 static void __init arm_memory_present(void)
250 {
251 }
252 #else
253 static void __init arm_memory_present(void)
254 {
255 	struct memblock_region *reg;
256 
257 	for_each_memblock(memory, reg)
258 		memory_present(0, memblock_region_memory_base_pfn(reg),
259 			       memblock_region_memory_end_pfn(reg));
260 }
261 #endif
262 
263 static bool arm_memblock_steal_permitted = true;
264 
265 phys_addr_t __init arm_memblock_steal(phys_addr_t size, phys_addr_t align)
266 {
267 	phys_addr_t phys;
268 
269 	BUG_ON(!arm_memblock_steal_permitted);
270 
271 	phys = memblock_alloc_base(size, align, MEMBLOCK_ALLOC_ANYWHERE);
272 	memblock_free(phys, size);
273 	memblock_remove(phys, size);
274 
275 	return phys;
276 }
277 
278 void __init arm_memblock_init(const struct machine_desc *mdesc)
279 {
280 	/* Register the kernel text, kernel data and initrd with memblock. */
281 #ifdef CONFIG_XIP_KERNEL
282 	memblock_reserve(__pa(_sdata), _end - _sdata);
283 #else
284 	memblock_reserve(__pa(_stext), _end - _stext);
285 #endif
286 #ifdef CONFIG_BLK_DEV_INITRD
287 	/* FDT scan will populate initrd_start */
288 	if (initrd_start && !phys_initrd_size) {
289 		phys_initrd_start = __virt_to_phys(initrd_start);
290 		phys_initrd_size = initrd_end - initrd_start;
291 	}
292 	initrd_start = initrd_end = 0;
293 	if (phys_initrd_size &&
294 	    !memblock_is_region_memory(phys_initrd_start, phys_initrd_size)) {
295 		pr_err("INITRD: 0x%08llx+0x%08lx is not a memory region - disabling initrd\n",
296 		       (u64)phys_initrd_start, phys_initrd_size);
297 		phys_initrd_start = phys_initrd_size = 0;
298 	}
299 	if (phys_initrd_size &&
300 	    memblock_is_region_reserved(phys_initrd_start, phys_initrd_size)) {
301 		pr_err("INITRD: 0x%08llx+0x%08lx overlaps in-use memory region - disabling initrd\n",
302 		       (u64)phys_initrd_start, phys_initrd_size);
303 		phys_initrd_start = phys_initrd_size = 0;
304 	}
305 	if (phys_initrd_size) {
306 		memblock_reserve(phys_initrd_start, phys_initrd_size);
307 
308 		/* Now convert initrd to virtual addresses */
309 		initrd_start = __phys_to_virt(phys_initrd_start);
310 		initrd_end = initrd_start + phys_initrd_size;
311 	}
312 #endif
313 
314 	arm_mm_memblock_reserve();
315 
316 	/* reserve any platform specific memblock areas */
317 	if (mdesc->reserve)
318 		mdesc->reserve();
319 
320 	early_init_fdt_scan_reserved_mem();
321 
322 	/*
323 	 * reserve memory for DMA contigouos allocations,
324 	 * must come from DMA area inside low memory
325 	 */
326 	dma_contiguous_reserve(arm_dma_limit);
327 
328 	arm_memblock_steal_permitted = false;
329 	memblock_dump_all();
330 }
331 
332 void __init bootmem_init(void)
333 {
334 	unsigned long min, max_low, max_high;
335 
336 	memblock_allow_resize();
337 	max_low = max_high = 0;
338 
339 	find_limits(&min, &max_low, &max_high);
340 
341 	/*
342 	 * Sparsemem tries to allocate bootmem in memory_present(),
343 	 * so must be done after the fixed reservations
344 	 */
345 	arm_memory_present();
346 
347 	/*
348 	 * sparse_init() needs the bootmem allocator up and running.
349 	 */
350 	sparse_init();
351 
352 	/*
353 	 * Now free the memory - free_area_init_node needs
354 	 * the sparse mem_map arrays initialized by sparse_init()
355 	 * for memmap_init_zone(), otherwise all PFNs are invalid.
356 	 */
357 	zone_sizes_init(min, max_low, max_high);
358 
359 	/*
360 	 * This doesn't seem to be used by the Linux memory manager any
361 	 * more, but is used by ll_rw_block.  If we can get rid of it, we
362 	 * also get rid of some of the stuff above as well.
363 	 */
364 	min_low_pfn = min;
365 	max_low_pfn = max_low;
366 	max_pfn = max_high;
367 }
368 
369 /*
370  * Poison init memory with an undefined instruction (ARM) or a branch to an
371  * undefined instruction (Thumb).
372  */
373 static inline void poison_init_mem(void *s, size_t count)
374 {
375 	u32 *p = (u32 *)s;
376 	for (; count != 0; count -= 4)
377 		*p++ = 0xe7fddef0;
378 }
379 
380 static inline void
381 free_memmap(unsigned long start_pfn, unsigned long end_pfn)
382 {
383 	struct page *start_pg, *end_pg;
384 	phys_addr_t pg, pgend;
385 
386 	/*
387 	 * Convert start_pfn/end_pfn to a struct page pointer.
388 	 */
389 	start_pg = pfn_to_page(start_pfn - 1) + 1;
390 	end_pg = pfn_to_page(end_pfn - 1) + 1;
391 
392 	/*
393 	 * Convert to physical addresses, and
394 	 * round start upwards and end downwards.
395 	 */
396 	pg = PAGE_ALIGN(__pa(start_pg));
397 	pgend = __pa(end_pg) & PAGE_MASK;
398 
399 	/*
400 	 * If there are free pages between these,
401 	 * free the section of the memmap array.
402 	 */
403 	if (pg < pgend)
404 		memblock_free_early(pg, pgend - pg);
405 }
406 
407 /*
408  * The mem_map array can get very big.  Free the unused area of the memory map.
409  */
410 static void __init free_unused_memmap(void)
411 {
412 	unsigned long start, prev_end = 0;
413 	struct memblock_region *reg;
414 
415 	/*
416 	 * This relies on each bank being in address order.
417 	 * The banks are sorted previously in bootmem_init().
418 	 */
419 	for_each_memblock(memory, reg) {
420 		start = memblock_region_memory_base_pfn(reg);
421 
422 #ifdef CONFIG_SPARSEMEM
423 		/*
424 		 * Take care not to free memmap entries that don't exist
425 		 * due to SPARSEMEM sections which aren't present.
426 		 */
427 		start = min(start,
428 				 ALIGN(prev_end, PAGES_PER_SECTION));
429 #else
430 		/*
431 		 * Align down here since the VM subsystem insists that the
432 		 * memmap entries are valid from the bank start aligned to
433 		 * MAX_ORDER_NR_PAGES.
434 		 */
435 		start = round_down(start, MAX_ORDER_NR_PAGES);
436 #endif
437 		/*
438 		 * If we had a previous bank, and there is a space
439 		 * between the current bank and the previous, free it.
440 		 */
441 		if (prev_end && prev_end < start)
442 			free_memmap(prev_end, start);
443 
444 		/*
445 		 * Align up here since the VM subsystem insists that the
446 		 * memmap entries are valid from the bank end aligned to
447 		 * MAX_ORDER_NR_PAGES.
448 		 */
449 		prev_end = ALIGN(memblock_region_memory_end_pfn(reg),
450 				 MAX_ORDER_NR_PAGES);
451 	}
452 
453 #ifdef CONFIG_SPARSEMEM
454 	if (!IS_ALIGNED(prev_end, PAGES_PER_SECTION))
455 		free_memmap(prev_end,
456 			    ALIGN(prev_end, PAGES_PER_SECTION));
457 #endif
458 }
459 
460 #ifdef CONFIG_HIGHMEM
461 static inline void free_area_high(unsigned long pfn, unsigned long end)
462 {
463 	for (; pfn < end; pfn++)
464 		free_highmem_page(pfn_to_page(pfn));
465 }
466 #endif
467 
468 static void __init free_highpages(void)
469 {
470 #ifdef CONFIG_HIGHMEM
471 	unsigned long max_low = max_low_pfn;
472 	struct memblock_region *mem, *res;
473 
474 	/* set highmem page free */
475 	for_each_memblock(memory, mem) {
476 		unsigned long start = memblock_region_memory_base_pfn(mem);
477 		unsigned long end = memblock_region_memory_end_pfn(mem);
478 
479 		/* Ignore complete lowmem entries */
480 		if (end <= max_low)
481 			continue;
482 
483 		/* Truncate partial highmem entries */
484 		if (start < max_low)
485 			start = max_low;
486 
487 		/* Find and exclude any reserved regions */
488 		for_each_memblock(reserved, res) {
489 			unsigned long res_start, res_end;
490 
491 			res_start = memblock_region_reserved_base_pfn(res);
492 			res_end = memblock_region_reserved_end_pfn(res);
493 
494 			if (res_end < start)
495 				continue;
496 			if (res_start < start)
497 				res_start = start;
498 			if (res_start > end)
499 				res_start = end;
500 			if (res_end > end)
501 				res_end = end;
502 			if (res_start != start)
503 				free_area_high(start, res_start);
504 			start = res_end;
505 			if (start == end)
506 				break;
507 		}
508 
509 		/* And now free anything which remains */
510 		if (start < end)
511 			free_area_high(start, end);
512 	}
513 #endif
514 }
515 
516 /*
517  * mem_init() marks the free areas in the mem_map and tells us how much
518  * memory is free.  This is done after various parts of the system have
519  * claimed their memory after the kernel image.
520  */
521 void __init mem_init(void)
522 {
523 #ifdef CONFIG_HAVE_TCM
524 	/* These pointers are filled in on TCM detection */
525 	extern u32 dtcm_end;
526 	extern u32 itcm_end;
527 #endif
528 
529 	set_max_mapnr(pfn_to_page(max_pfn) - mem_map);
530 
531 	/* this will put all unused low memory onto the freelists */
532 	free_unused_memmap();
533 	free_all_bootmem();
534 
535 #ifdef CONFIG_SA1111
536 	/* now that our DMA memory is actually so designated, we can free it */
537 	free_reserved_area(__va(PHYS_OFFSET), swapper_pg_dir, -1, NULL);
538 #endif
539 
540 	free_highpages();
541 
542 	mem_init_print_info(NULL);
543 
544 #define MLK(b, t) b, t, ((t) - (b)) >> 10
545 #define MLM(b, t) b, t, ((t) - (b)) >> 20
546 #define MLK_ROUNDUP(b, t) b, t, DIV_ROUND_UP(((t) - (b)), SZ_1K)
547 
548 	pr_notice("Virtual kernel memory layout:\n"
549 			"    vector  : 0x%08lx - 0x%08lx   (%4ld kB)\n"
550 #ifdef CONFIG_HAVE_TCM
551 			"    DTCM    : 0x%08lx - 0x%08lx   (%4ld kB)\n"
552 			"    ITCM    : 0x%08lx - 0x%08lx   (%4ld kB)\n"
553 #endif
554 			"    fixmap  : 0x%08lx - 0x%08lx   (%4ld kB)\n"
555 			"    vmalloc : 0x%08lx - 0x%08lx   (%4ld MB)\n"
556 			"    lowmem  : 0x%08lx - 0x%08lx   (%4ld MB)\n"
557 #ifdef CONFIG_HIGHMEM
558 			"    pkmap   : 0x%08lx - 0x%08lx   (%4ld MB)\n"
559 #endif
560 #ifdef CONFIG_MODULES
561 			"    modules : 0x%08lx - 0x%08lx   (%4ld MB)\n"
562 #endif
563 			"      .text : 0x%p" " - 0x%p" "   (%4td kB)\n"
564 			"      .init : 0x%p" " - 0x%p" "   (%4td kB)\n"
565 			"      .data : 0x%p" " - 0x%p" "   (%4td kB)\n"
566 			"       .bss : 0x%p" " - 0x%p" "   (%4td kB)\n",
567 
568 			MLK(UL(CONFIG_VECTORS_BASE), UL(CONFIG_VECTORS_BASE) +
569 				(PAGE_SIZE)),
570 #ifdef CONFIG_HAVE_TCM
571 			MLK(DTCM_OFFSET, (unsigned long) dtcm_end),
572 			MLK(ITCM_OFFSET, (unsigned long) itcm_end),
573 #endif
574 			MLK(FIXADDR_START, FIXADDR_END),
575 			MLM(VMALLOC_START, VMALLOC_END),
576 			MLM(PAGE_OFFSET, (unsigned long)high_memory),
577 #ifdef CONFIG_HIGHMEM
578 			MLM(PKMAP_BASE, (PKMAP_BASE) + (LAST_PKMAP) *
579 				(PAGE_SIZE)),
580 #endif
581 #ifdef CONFIG_MODULES
582 			MLM(MODULES_VADDR, MODULES_END),
583 #endif
584 
585 			MLK_ROUNDUP(_text, _etext),
586 			MLK_ROUNDUP(__init_begin, __init_end),
587 			MLK_ROUNDUP(_sdata, _edata),
588 			MLK_ROUNDUP(__bss_start, __bss_stop));
589 
590 #undef MLK
591 #undef MLM
592 #undef MLK_ROUNDUP
593 
594 	/*
595 	 * Check boundaries twice: Some fundamental inconsistencies can
596 	 * be detected at build time already.
597 	 */
598 #ifdef CONFIG_MMU
599 	BUILD_BUG_ON(TASK_SIZE				> MODULES_VADDR);
600 	BUG_ON(TASK_SIZE 				> MODULES_VADDR);
601 #endif
602 
603 #ifdef CONFIG_HIGHMEM
604 	BUILD_BUG_ON(PKMAP_BASE + LAST_PKMAP * PAGE_SIZE > PAGE_OFFSET);
605 	BUG_ON(PKMAP_BASE + LAST_PKMAP * PAGE_SIZE	> PAGE_OFFSET);
606 #endif
607 
608 	if (PAGE_SIZE >= 16384 && get_num_physpages() <= 128) {
609 		extern int sysctl_overcommit_memory;
610 		/*
611 		 * On a machine this small we won't get
612 		 * anywhere without overcommit, so turn
613 		 * it on by default.
614 		 */
615 		sysctl_overcommit_memory = OVERCOMMIT_ALWAYS;
616 	}
617 }
618 
619 #ifdef CONFIG_ARM_KERNMEM_PERMS
620 struct section_perm {
621 	unsigned long start;
622 	unsigned long end;
623 	pmdval_t mask;
624 	pmdval_t prot;
625 	pmdval_t clear;
626 };
627 
628 static struct section_perm nx_perms[] = {
629 	/* Make pages tables, etc before _stext RW (set NX). */
630 	{
631 		.start	= PAGE_OFFSET,
632 		.end	= (unsigned long)_stext,
633 		.mask	= ~PMD_SECT_XN,
634 		.prot	= PMD_SECT_XN,
635 	},
636 	/* Make init RW (set NX). */
637 	{
638 		.start	= (unsigned long)__init_begin,
639 		.end	= (unsigned long)_sdata,
640 		.mask	= ~PMD_SECT_XN,
641 		.prot	= PMD_SECT_XN,
642 	},
643 #ifdef CONFIG_DEBUG_RODATA
644 	/* Make rodata NX (set RO in ro_perms below). */
645 	{
646 		.start  = (unsigned long)__start_rodata,
647 		.end    = (unsigned long)__init_begin,
648 		.mask   = ~PMD_SECT_XN,
649 		.prot   = PMD_SECT_XN,
650 	},
651 #endif
652 };
653 
654 #ifdef CONFIG_DEBUG_RODATA
655 static struct section_perm ro_perms[] = {
656 	/* Make kernel code and rodata RX (set RO). */
657 	{
658 		.start  = (unsigned long)_stext,
659 		.end    = (unsigned long)__init_begin,
660 #ifdef CONFIG_ARM_LPAE
661 		.mask   = ~L_PMD_SECT_RDONLY,
662 		.prot   = L_PMD_SECT_RDONLY,
663 #else
664 		.mask   = ~(PMD_SECT_APX | PMD_SECT_AP_WRITE),
665 		.prot   = PMD_SECT_APX | PMD_SECT_AP_WRITE,
666 		.clear  = PMD_SECT_AP_WRITE,
667 #endif
668 	},
669 };
670 #endif
671 
672 /*
673  * Updates section permissions only for the current mm (sections are
674  * copied into each mm). During startup, this is the init_mm. Is only
675  * safe to be called with preemption disabled, as under stop_machine().
676  */
677 static inline void section_update(unsigned long addr, pmdval_t mask,
678 				  pmdval_t prot)
679 {
680 	struct mm_struct *mm;
681 	pmd_t *pmd;
682 
683 	mm = current->active_mm;
684 	pmd = pmd_offset(pud_offset(pgd_offset(mm, addr), addr), addr);
685 
686 #ifdef CONFIG_ARM_LPAE
687 	pmd[0] = __pmd((pmd_val(pmd[0]) & mask) | prot);
688 #else
689 	if (addr & SECTION_SIZE)
690 		pmd[1] = __pmd((pmd_val(pmd[1]) & mask) | prot);
691 	else
692 		pmd[0] = __pmd((pmd_val(pmd[0]) & mask) | prot);
693 #endif
694 	flush_pmd_entry(pmd);
695 	local_flush_tlb_kernel_range(addr, addr + SECTION_SIZE);
696 }
697 
698 /* Make sure extended page tables are in use. */
699 static inline bool arch_has_strict_perms(void)
700 {
701 	if (cpu_architecture() < CPU_ARCH_ARMv6)
702 		return false;
703 
704 	return !!(get_cr() & CR_XP);
705 }
706 
707 #define set_section_perms(perms, field)	{				\
708 	size_t i;							\
709 	unsigned long addr;						\
710 									\
711 	if (!arch_has_strict_perms())					\
712 		return;							\
713 									\
714 	for (i = 0; i < ARRAY_SIZE(perms); i++) {			\
715 		if (!IS_ALIGNED(perms[i].start, SECTION_SIZE) ||	\
716 		    !IS_ALIGNED(perms[i].end, SECTION_SIZE)) {		\
717 			pr_err("BUG: section %lx-%lx not aligned to %lx\n", \
718 				perms[i].start, perms[i].end,		\
719 				SECTION_SIZE);				\
720 			continue;					\
721 		}							\
722 									\
723 		for (addr = perms[i].start;				\
724 		     addr < perms[i].end;				\
725 		     addr += SECTION_SIZE)				\
726 			section_update(addr, perms[i].mask,		\
727 				       perms[i].field);			\
728 	}								\
729 }
730 
731 static inline void fix_kernmem_perms(void)
732 {
733 	set_section_perms(nx_perms, prot);
734 }
735 
736 #ifdef CONFIG_DEBUG_RODATA
737 void mark_rodata_ro(void)
738 {
739 	set_section_perms(ro_perms, prot);
740 }
741 
742 void set_kernel_text_rw(void)
743 {
744 	set_section_perms(ro_perms, clear);
745 }
746 
747 void set_kernel_text_ro(void)
748 {
749 	set_section_perms(ro_perms, prot);
750 }
751 #endif /* CONFIG_DEBUG_RODATA */
752 
753 #else
754 static inline void fix_kernmem_perms(void) { }
755 #endif /* CONFIG_ARM_KERNMEM_PERMS */
756 
757 void free_tcmmem(void)
758 {
759 #ifdef CONFIG_HAVE_TCM
760 	extern char __tcm_start, __tcm_end;
761 
762 	poison_init_mem(&__tcm_start, &__tcm_end - &__tcm_start);
763 	free_reserved_area(&__tcm_start, &__tcm_end, -1, "TCM link");
764 #endif
765 }
766 
767 void free_initmem(void)
768 {
769 	fix_kernmem_perms();
770 	free_tcmmem();
771 
772 	poison_init_mem(__init_begin, __init_end - __init_begin);
773 	if (!machine_is_integrator() && !machine_is_cintegrator())
774 		free_initmem_default(-1);
775 }
776 
777 #ifdef CONFIG_BLK_DEV_INITRD
778 
779 static int keep_initrd;
780 
781 void free_initrd_mem(unsigned long start, unsigned long end)
782 {
783 	if (!keep_initrd) {
784 		if (start == initrd_start)
785 			start = round_down(start, PAGE_SIZE);
786 		if (end == initrd_end)
787 			end = round_up(end, PAGE_SIZE);
788 
789 		poison_init_mem((void *)start, PAGE_ALIGN(end) - start);
790 		free_reserved_area((void *)start, (void *)end, -1, "initrd");
791 	}
792 }
793 
794 static int __init keepinitrd_setup(char *__unused)
795 {
796 	keep_initrd = 1;
797 	return 1;
798 }
799 
800 __setup("keepinitrd", keepinitrd_setup);
801 #endif
802