11da177e4SLinus Torvalds /* 21da177e4SLinus Torvalds * linux/arch/arm/mm/init.c 31da177e4SLinus Torvalds * 490072059SRussell King * Copyright (C) 1995-2005 Russell King 51da177e4SLinus Torvalds * 61da177e4SLinus Torvalds * This program is free software; you can redistribute it and/or modify 71da177e4SLinus Torvalds * it under the terms of the GNU General Public License version 2 as 81da177e4SLinus Torvalds * published by the Free Software Foundation. 91da177e4SLinus Torvalds */ 101da177e4SLinus Torvalds #include <linux/kernel.h> 111da177e4SLinus Torvalds #include <linux/errno.h> 121da177e4SLinus Torvalds #include <linux/swap.h> 131da177e4SLinus Torvalds #include <linux/init.h> 141da177e4SLinus Torvalds #include <linux/mman.h> 153f07c014SIngo Molnar #include <linux/sched/signal.h> 1629930025SIngo Molnar #include <linux/sched/task.h> 17dc28094bSPaul Gortmaker #include <linux/export.h> 181da177e4SLinus Torvalds #include <linux/nodemask.h> 191da177e4SLinus Torvalds #include <linux/initrd.h> 209eb8f674SGrant Likely #include <linux/of_fdt.h> 213835f6cbSNicolas Pitre #include <linux/highmem.h> 225a0e3ad6STejun Heo #include <linux/gfp.h> 232778f620SRussell King #include <linux/memblock.h> 24c7909509SMarek Szyprowski #include <linux/dma-contiguous.h> 25158e8bfeSAlessandro Rubini #include <linux/sizes.h> 2608925c2fSLaura Abbott #include <linux/stop_machine.h> 271da177e4SLinus Torvalds 28b4b20ad8SRussell King #include <asm/cp15.h> 291da177e4SLinus Torvalds #include <asm/mach-types.h> 30716a3dc2SRussell King #include <asm/memblock.h> 31d2ca5f24SAfzal Mohammed #include <asm/memory.h> 3293c02ab4SGrant Likely #include <asm/prom.h> 3337efe642SRussell King #include <asm/sections.h> 341da177e4SLinus Torvalds #include <asm/setup.h> 351e6b4811SKees Cook #include <asm/system_info.h> 361da177e4SLinus Torvalds #include <asm/tlb.h> 37db9ef1afSFenkart/Bostandzhyan #include <asm/fixmap.h> 38a8e53c15SJinbum Park #include <asm/ptdump.h> 391da177e4SLinus Torvalds 401da177e4SLinus Torvalds #include <asm/mach/arch.h> 411da177e4SLinus Torvalds #include <asm/mach/map.h> 421da177e4SLinus Torvalds 431b2e2b73SRussell King #include "mm.h" 441b2e2b73SRussell King 45b4b20ad8SRussell King #ifdef CONFIG_CPU_CP15_MMU 46b4b20ad8SRussell King unsigned long __init __clear_cr(unsigned long mask) 47b4b20ad8SRussell King { 48b4b20ad8SRussell King cr_alignment = cr_alignment & ~mask; 49b4b20ad8SRussell King return cr_alignment; 50b4b20ad8SRussell King } 51b4b20ad8SRussell King #endif 52b4b20ad8SRussell King 53b1ab95c6SFlorian Fainelli #ifdef CONFIG_BLK_DEV_INITRD 54012d1f4aSRussell King static int __init parse_tag_initrd(const struct tag *tag) 55012d1f4aSRussell King { 564ed89f22SRussell King pr_warn("ATAG_INITRD is deprecated; " 57012d1f4aSRussell King "please update your bootloader.\n"); 58012d1f4aSRussell King phys_initrd_start = __virt_to_phys(tag->u.initrd.start); 59012d1f4aSRussell King phys_initrd_size = tag->u.initrd.size; 60012d1f4aSRussell King return 0; 61012d1f4aSRussell King } 62012d1f4aSRussell King 63012d1f4aSRussell King __tagtable(ATAG_INITRD, parse_tag_initrd); 64012d1f4aSRussell King 65012d1f4aSRussell King static int __init parse_tag_initrd2(const struct tag *tag) 66012d1f4aSRussell King { 67012d1f4aSRussell King phys_initrd_start = tag->u.initrd.start; 68012d1f4aSRussell King phys_initrd_size = tag->u.initrd.size; 69012d1f4aSRussell King return 0; 70012d1f4aSRussell King } 71012d1f4aSRussell King 72012d1f4aSRussell King __tagtable(ATAG_INITRD2, parse_tag_initrd2); 73b1ab95c6SFlorian Fainelli #endif 741da177e4SLinus Torvalds 75f25b4b4cSRussell King static void __init find_limits(unsigned long *min, unsigned long *max_low, 76f25b4b4cSRussell King unsigned long *max_high) 77dde5828fSRussell King { 781c2f87c2SLaura Abbott *max_low = PFN_DOWN(memblock_get_current_limit()); 791c2f87c2SLaura Abbott *min = PFN_UP(memblock_start_of_DRAM()); 801c2f87c2SLaura Abbott *max_high = PFN_DOWN(memblock_end_of_DRAM()); 81dde5828fSRussell King } 82dde5828fSRussell King 83be20902bSRussell King #ifdef CONFIG_ZONE_DMA 8465032018SNicolas Pitre 85364230b9SRob Herring phys_addr_t arm_dma_zone_size __read_mostly; 8665032018SNicolas Pitre EXPORT_SYMBOL(arm_dma_zone_size); 8765032018SNicolas Pitre 88022ae537SRussell King /* 89022ae537SRussell King * The DMA mask corresponding to the maximum bus address allocatable 90022ae537SRussell King * using GFP_DMA. The default here places no restriction on DMA 91022ae537SRussell King * allocations. This must be the smallest DMA mask in the system, 92022ae537SRussell King * so a successful GFP_DMA allocation will always satisfy this. 93022ae537SRussell King */ 944986e5c7SMarek Szyprowski phys_addr_t arm_dma_limit; 954dcfa600SRussell King unsigned long arm_dma_pfn_limit; 96022ae537SRussell King 97be20902bSRussell King static void __init arm_adjust_dma_zone(unsigned long *size, unsigned long *hole, 98be20902bSRussell King unsigned long dma_size) 99be20902bSRussell King { 100be20902bSRussell King if (size[0] <= dma_size) 101be20902bSRussell King return; 102be20902bSRussell King 103be20902bSRussell King size[ZONE_NORMAL] = size[0] - dma_size; 104be20902bSRussell King size[ZONE_DMA] = dma_size; 105be20902bSRussell King hole[ZONE_NORMAL] = hole[0]; 106be20902bSRussell King hole[ZONE_DMA] = 0; 107be20902bSRussell King } 108be20902bSRussell King #endif 109be20902bSRussell King 110ff69a4c8SRussell King void __init setup_dma_zone(const struct machine_desc *mdesc) 111c7909509SMarek Szyprowski { 112c7909509SMarek Szyprowski #ifdef CONFIG_ZONE_DMA 113c7909509SMarek Szyprowski if (mdesc->dma_zone_size) { 114c7909509SMarek Szyprowski arm_dma_zone_size = mdesc->dma_zone_size; 1156bcac805SRussell King arm_dma_limit = PHYS_OFFSET + arm_dma_zone_size - 1; 116c7909509SMarek Szyprowski } else 117c7909509SMarek Szyprowski arm_dma_limit = 0xffffffff; 1184dcfa600SRussell King arm_dma_pfn_limit = arm_dma_limit >> PAGE_SHIFT; 119c7909509SMarek Szyprowski #endif 120c7909509SMarek Szyprowski } 121c7909509SMarek Szyprowski 12284f452b1SSantosh Shilimkar static void __init zone_sizes_init(unsigned long min, unsigned long max_low, 123a2c54d2aSRussell King unsigned long max_high) 124b7a69ac3SRussell King { 125b7a69ac3SRussell King unsigned long zone_size[MAX_NR_ZONES], zhole_size[MAX_NR_ZONES]; 126a2c54d2aSRussell King struct memblock_region *reg; 127b7a69ac3SRussell King 12890072059SRussell King /* 129be370302SRussell King * initialise the zones. 13090072059SRussell King */ 13190072059SRussell King memset(zone_size, 0, sizeof(zone_size)); 13290072059SRussell King 13390072059SRussell King /* 134be370302SRussell King * The memory size has already been determined. If we need 135be370302SRussell King * to do anything fancy with the allocation of this memory 136be370302SRussell King * to the zones, now is the time to do it. 13790072059SRussell King */ 138dde5828fSRussell King zone_size[0] = max_low - min; 139dde5828fSRussell King #ifdef CONFIG_HIGHMEM 140dde5828fSRussell King zone_size[ZONE_HIGHMEM] = max_high - max_low; 141dde5828fSRussell King #endif 14290072059SRussell King 14390072059SRussell King /* 144be370302SRussell King * Calculate the size of the holes. 145be370302SRussell King * holes = node_size - sum(bank_sizes) 14690072059SRussell King */ 147dde5828fSRussell King memcpy(zhole_size, zone_size, sizeof(zhole_size)); 148a2c54d2aSRussell King for_each_memblock(memory, reg) { 149a2c54d2aSRussell King unsigned long start = memblock_region_memory_base_pfn(reg); 150a2c54d2aSRussell King unsigned long end = memblock_region_memory_end_pfn(reg); 151a2c54d2aSRussell King 152a2c54d2aSRussell King if (start < max_low) { 153a2c54d2aSRussell King unsigned long low_end = min(end, max_low); 154a2c54d2aSRussell King zhole_size[0] -= low_end - start; 155a2c54d2aSRussell King } 156dde5828fSRussell King #ifdef CONFIG_HIGHMEM 157a2c54d2aSRussell King if (end > max_low) { 158a2c54d2aSRussell King unsigned long high_start = max(start, max_low); 159a2c54d2aSRussell King zhole_size[ZONE_HIGHMEM] -= end - high_start; 160a2c54d2aSRussell King } 161dde5828fSRussell King #endif 162dde5828fSRussell King } 16390072059SRussell King 16465032018SNicolas Pitre #ifdef CONFIG_ZONE_DMA 16590072059SRussell King /* 16690072059SRussell King * Adjust the sizes according to any special requirements for 16790072059SRussell King * this machine type. 16890072059SRussell King */ 169c7909509SMarek Szyprowski if (arm_dma_zone_size) 170be20902bSRussell King arm_adjust_dma_zone(zone_size, zhole_size, 17165032018SNicolas Pitre arm_dma_zone_size >> PAGE_SHIFT); 172be20902bSRussell King #endif 17390072059SRussell King 174be370302SRussell King free_area_init_node(0, zone_size, min, zhole_size); 17590072059SRussell King } 17690072059SRussell King 1777b7bf499SWill Deacon #ifdef CONFIG_HAVE_ARCH_PFN_VALID 178b7cfda9fSRussell King int pfn_valid(unsigned long pfn) 179b7cfda9fSRussell King { 18009414d00SArd Biesheuvel return memblock_is_map_memory(__pfn_to_phys(pfn)); 181b7cfda9fSRussell King } 182b7cfda9fSRussell King EXPORT_SYMBOL(pfn_valid); 1837b7bf499SWill Deacon #endif 184657e12fdSRussell King 1857b7bf499SWill Deacon #ifndef CONFIG_SPARSEMEM 18614904927SStephen Boyd static void __init arm_memory_present(void) 187657e12fdSRussell King { 188657e12fdSRussell King } 189657e12fdSRussell King #else 19014904927SStephen Boyd static void __init arm_memory_present(void) 191657e12fdSRussell King { 192719c1514SBenjamin Herrenschmidt struct memblock_region *reg; 193719c1514SBenjamin Herrenschmidt 1947c996361SYinghai Lu for_each_memblock(memory, reg) 195c7fc2de0SYinghai Lu memory_present(0, memblock_region_memory_base_pfn(reg), 196c7fc2de0SYinghai Lu memblock_region_memory_end_pfn(reg)); 197657e12fdSRussell King } 198b7cfda9fSRussell King #endif 199b7cfda9fSRussell King 200716a3dc2SRussell King static bool arm_memblock_steal_permitted = true; 201716a3dc2SRussell King 202bc2827d0SRussell King phys_addr_t __init arm_memblock_steal(phys_addr_t size, phys_addr_t align) 203716a3dc2SRussell King { 204716a3dc2SRussell King phys_addr_t phys; 205716a3dc2SRussell King 206716a3dc2SRussell King BUG_ON(!arm_memblock_steal_permitted); 207716a3dc2SRussell King 208f240ec09SMike Rapoport phys = memblock_phys_alloc(size, align); 209716a3dc2SRussell King memblock_free(phys, size); 210716a3dc2SRussell King memblock_remove(phys, size); 211716a3dc2SRussell King 212716a3dc2SRussell King return phys; 213716a3dc2SRussell King } 214716a3dc2SRussell King 21539286248SRussell King static void __init arm_initrd_init(void) 2162778f620SRussell King { 2172778f620SRussell King #ifdef CONFIG_BLK_DEV_INITRD 218cdcc5fa0SRussell King phys_addr_t start; 219cdcc5fa0SRussell King unsigned long size; 220cdcc5fa0SRussell King 2214c235cb9SBen Peddell initrd_start = initrd_end = 0; 22268b32f36SRussell King 22368b32f36SRussell King if (!phys_initrd_size) 22468b32f36SRussell King return; 22568b32f36SRussell King 226cdcc5fa0SRussell King /* 227cdcc5fa0SRussell King * Round the memory region to page boundaries as per free_initrd_mem() 228cdcc5fa0SRussell King * This allows us to detect whether the pages overlapping the initrd 229cdcc5fa0SRussell King * are in use, but more importantly, reserves the entire set of pages 230cdcc5fa0SRussell King * as we don't want these pages allocated for other purposes. 231cdcc5fa0SRussell King */ 232cdcc5fa0SRussell King start = round_down(phys_initrd_start, PAGE_SIZE); 233cdcc5fa0SRussell King size = phys_initrd_size + (phys_initrd_start - start); 234cdcc5fa0SRussell King size = round_up(size, PAGE_SIZE); 235cdcc5fa0SRussell King 236cdcc5fa0SRussell King if (!memblock_is_region_memory(start, size)) { 237de22cc6eSVitaly Andrianov pr_err("INITRD: 0x%08llx+0x%08lx is not a memory region - disabling initrd\n", 238cdcc5fa0SRussell King (u64)start, size); 23968b32f36SRussell King return; 2408f4b8c76SRussell King } 24168b32f36SRussell King 242cdcc5fa0SRussell King if (memblock_is_region_reserved(start, size)) { 243de22cc6eSVitaly Andrianov pr_err("INITRD: 0x%08llx+0x%08lx overlaps in-use memory region - disabling initrd\n", 244cdcc5fa0SRussell King (u64)start, size); 24568b32f36SRussell King return; 246b0a2679dSRussell King } 24768b32f36SRussell King 248cdcc5fa0SRussell King memblock_reserve(start, size); 2492778f620SRussell King 2502778f620SRussell King /* Now convert initrd to virtual addresses */ 2512778f620SRussell King initrd_start = __phys_to_virt(phys_initrd_start); 2522778f620SRussell King initrd_end = initrd_start + phys_initrd_size; 2532778f620SRussell King #endif 25439286248SRussell King } 25539286248SRussell King 25639286248SRussell King void __init arm_memblock_init(const struct machine_desc *mdesc) 25739286248SRussell King { 25839286248SRussell King /* Register the kernel text, kernel data and initrd with memblock. */ 25939286248SRussell King memblock_reserve(__pa(KERNEL_START), KERNEL_END - KERNEL_START); 26039286248SRussell King 26139286248SRussell King arm_initrd_init(); 2622778f620SRussell King 2632778f620SRussell King arm_mm_memblock_reserve(); 2642778f620SRussell King 2658d717a52SRussell King /* reserve any platform specific memblock areas */ 2668d717a52SRussell King if (mdesc->reserve) 2678d717a52SRussell King mdesc->reserve(); 2688d717a52SRussell King 26924bbd929SArd Biesheuvel early_init_fdt_reserve_self(); 270bcedb5f9SMarek Szyprowski early_init_fdt_scan_reserved_mem(); 271bcedb5f9SMarek Szyprowski 27299a468d7SGeorge G. Davis /* reserve memory for DMA contiguous allocations */ 27395b0e655SMarek Szyprowski dma_contiguous_reserve(arm_dma_limit); 274c7909509SMarek Szyprowski 275716a3dc2SRussell King arm_memblock_steal_permitted = false; 2762778f620SRussell King memblock_dump_all(); 2772778f620SRussell King } 2782778f620SRussell King 2798d717a52SRussell King void __init bootmem_init(void) 28090072059SRussell King { 281dde5828fSRussell King unsigned long min, max_low, max_high; 28290072059SRussell King 2838e58caefSGrygorii Strashko memblock_allow_resize(); 284dde5828fSRussell King max_low = max_high = 0; 285dde5828fSRussell King 286f25b4b4cSRussell King find_limits(&min, &max_low, &max_high); 287dde5828fSRussell King 288d30eae47SVladimir Murzin early_memtest((phys_addr_t)min << PAGE_SHIFT, 289d30eae47SVladimir Murzin (phys_addr_t)max_low << PAGE_SHIFT); 290d30eae47SVladimir Murzin 291b7a69ac3SRussell King /* 292657e12fdSRussell King * Sparsemem tries to allocate bootmem in memory_present(), 293657e12fdSRussell King * so must be done after the fixed reservations 294657e12fdSRussell King */ 295eda2e5dcSRussell King arm_memory_present(); 29690072059SRussell King 297b7a69ac3SRussell King /* 298b7a69ac3SRussell King * sparse_init() needs the bootmem allocator up and running. 299b7a69ac3SRussell King */ 300b7a69ac3SRussell King sparse_init(); 301b7a69ac3SRussell King 302b7a69ac3SRussell King /* 303be370302SRussell King * Now free the memory - free_area_init_node needs 304b7a69ac3SRussell King * the sparse mem_map arrays initialized by sparse_init() 305b7a69ac3SRussell King * for memmap_init_zone(), otherwise all PFNs are invalid. 306b7a69ac3SRussell King */ 30784f452b1SSantosh Shilimkar zone_sizes_init(min, max_low, max_high); 308b7a69ac3SRussell King 30990072059SRussell King /* 31090072059SRussell King * This doesn't seem to be used by the Linux memory manager any 31190072059SRussell King * more, but is used by ll_rw_block. If we can get rid of it, we 31290072059SRussell King * also get rid of some of the stuff above as well. 31390072059SRussell King */ 31426ba47b1SSantosh Shilimkar min_low_pfn = min; 31526ba47b1SSantosh Shilimkar max_low_pfn = max_low; 31626ba47b1SSantosh Shilimkar max_pfn = max_high; 31790072059SRussell King } 31890072059SRussell King 31954d52573SStephen Boyd /* 32054d52573SStephen Boyd * Poison init memory with an undefined instruction (ARM) or a branch to an 32154d52573SStephen Boyd * undefined instruction (Thumb). 32254d52573SStephen Boyd */ 32354d52573SStephen Boyd static inline void poison_init_mem(void *s, size_t count) 32454d52573SStephen Boyd { 32554d52573SStephen Boyd u32 *p = (u32 *)s; 326bf912d99SJamie Iles for (; count != 0; count -= 4) 32754d52573SStephen Boyd *p++ = 0xe7fddef0; 32854d52573SStephen Boyd } 32954d52573SStephen Boyd 330a013053dSRussell King static inline void 331be370302SRussell King free_memmap(unsigned long start_pfn, unsigned long end_pfn) 332a013053dSRussell King { 333a013053dSRussell King struct page *start_pg, *end_pg; 33456bc6286SVitaly Andrianov phys_addr_t pg, pgend; 335a013053dSRussell King 336a013053dSRussell King /* 337a013053dSRussell King * Convert start_pfn/end_pfn to a struct page pointer. 338a013053dSRussell King */ 3393257f43dSCatalin Marinas start_pg = pfn_to_page(start_pfn - 1) + 1; 3409af386c8SWill Deacon end_pg = pfn_to_page(end_pfn - 1) + 1; 341a013053dSRussell King 342a013053dSRussell King /* 343a013053dSRussell King * Convert to physical addresses, and 344a013053dSRussell King * round start upwards and end downwards. 345a013053dSRussell King */ 34656bc6286SVitaly Andrianov pg = PAGE_ALIGN(__pa(start_pg)); 34756bc6286SVitaly Andrianov pgend = __pa(end_pg) & PAGE_MASK; 348a013053dSRussell King 349a013053dSRussell King /* 350a013053dSRussell King * If there are free pages between these, 351a013053dSRussell King * free the section of the memmap array. 352a013053dSRussell King */ 353a013053dSRussell King if (pg < pgend) 354cfb66586SSantosh Shilimkar memblock_free_early(pg, pgend - pg); 355a013053dSRussell King } 356a013053dSRussell King 357a013053dSRussell King /* 358a013053dSRussell King * The mem_map array can get very big. Free the unused area of the memory map. 359a013053dSRussell King */ 3601c2f87c2SLaura Abbott static void __init free_unused_memmap(void) 361a013053dSRussell King { 3621c2f87c2SLaura Abbott unsigned long start, prev_end = 0; 3631c2f87c2SLaura Abbott struct memblock_region *reg; 364a013053dSRussell King 365a013053dSRussell King /* 3663260e529SMichael Bohan * This relies on each bank being in address order. 3673260e529SMichael Bohan * The banks are sorted previously in bootmem_init(). 368a013053dSRussell King */ 3691c2f87c2SLaura Abbott for_each_memblock(memory, reg) { 3701c2f87c2SLaura Abbott start = memblock_region_memory_base_pfn(reg); 371a013053dSRussell King 3729af386c8SWill Deacon #ifdef CONFIG_SPARSEMEM 3739af386c8SWill Deacon /* 3749af386c8SWill Deacon * Take care not to free memmap entries that don't exist 3759af386c8SWill Deacon * due to SPARSEMEM sections which aren't present. 3769af386c8SWill Deacon */ 3771c2f87c2SLaura Abbott start = min(start, 3781c2f87c2SLaura Abbott ALIGN(prev_end, PAGES_PER_SECTION)); 379002ea9eeSLinus Walleij #else 380002ea9eeSLinus Walleij /* 381002ea9eeSLinus Walleij * Align down here since the VM subsystem insists that the 382002ea9eeSLinus Walleij * memmap entries are valid from the bank start aligned to 383002ea9eeSLinus Walleij * MAX_ORDER_NR_PAGES. 384002ea9eeSLinus Walleij */ 3851c2f87c2SLaura Abbott start = round_down(start, MAX_ORDER_NR_PAGES); 3869af386c8SWill Deacon #endif 387a013053dSRussell King /* 388a013053dSRussell King * If we had a previous bank, and there is a space 389a013053dSRussell King * between the current bank and the previous, free it. 390a013053dSRussell King */ 3911c2f87c2SLaura Abbott if (prev_end && prev_end < start) 3921c2f87c2SLaura Abbott free_memmap(prev_end, start); 393a013053dSRussell King 3943260e529SMichael Bohan /* 3953260e529SMichael Bohan * Align up here since the VM subsystem insists that the 3963260e529SMichael Bohan * memmap entries are valid from the bank end aligned to 3973260e529SMichael Bohan * MAX_ORDER_NR_PAGES. 3983260e529SMichael Bohan */ 3991c2f87c2SLaura Abbott prev_end = ALIGN(memblock_region_memory_end_pfn(reg), 4001c2f87c2SLaura Abbott MAX_ORDER_NR_PAGES); 401a013053dSRussell King } 4029af386c8SWill Deacon 4039af386c8SWill Deacon #ifdef CONFIG_SPARSEMEM 4041c2f87c2SLaura Abbott if (!IS_ALIGNED(prev_end, PAGES_PER_SECTION)) 4051c2f87c2SLaura Abbott free_memmap(prev_end, 4061c2f87c2SLaura Abbott ALIGN(prev_end, PAGES_PER_SECTION)); 4079af386c8SWill Deacon #endif 408a013053dSRussell King } 409a013053dSRussell King 41083db0384SJiang Liu #ifdef CONFIG_HIGHMEM 41183db0384SJiang Liu static inline void free_area_high(unsigned long pfn, unsigned long end) 41283db0384SJiang Liu { 413dd6911efSJiang Liu for (; pfn < end; pfn++) 414dd6911efSJiang Liu free_highmem_page(pfn_to_page(pfn)); 41583db0384SJiang Liu } 41683db0384SJiang Liu #endif 41783db0384SJiang Liu 418d0e775afSRussell King static void __init free_highpages(void) 419d0e775afSRussell King { 420d0e775afSRussell King #ifdef CONFIG_HIGHMEM 42126ba47b1SSantosh Shilimkar unsigned long max_low = max_low_pfn; 422df4f14c7SRussell King struct memblock_region *mem, *res; 423d0e775afSRussell King 424d0e775afSRussell King /* set highmem page free */ 425df4f14c7SRussell King for_each_memblock(memory, mem) { 426df4f14c7SRussell King unsigned long start = memblock_region_memory_base_pfn(mem); 427df4f14c7SRussell King unsigned long end = memblock_region_memory_end_pfn(mem); 428df4f14c7SRussell King 429df4f14c7SRussell King /* Ignore complete lowmem entries */ 430df4f14c7SRussell King if (end <= max_low) 431df4f14c7SRussell King continue; 432df4f14c7SRussell King 43309414d00SArd Biesheuvel if (memblock_is_nomap(mem)) 43409414d00SArd Biesheuvel continue; 43509414d00SArd Biesheuvel 436df4f14c7SRussell King /* Truncate partial highmem entries */ 437df4f14c7SRussell King if (start < max_low) 438df4f14c7SRussell King start = max_low; 439df4f14c7SRussell King 440df4f14c7SRussell King /* Find and exclude any reserved regions */ 441df4f14c7SRussell King for_each_memblock(reserved, res) { 442df4f14c7SRussell King unsigned long res_start, res_end; 443df4f14c7SRussell King 444df4f14c7SRussell King res_start = memblock_region_reserved_base_pfn(res); 445df4f14c7SRussell King res_end = memblock_region_reserved_end_pfn(res); 446df4f14c7SRussell King 447df4f14c7SRussell King if (res_end < start) 448df4f14c7SRussell King continue; 449df4f14c7SRussell King if (res_start < start) 450df4f14c7SRussell King res_start = start; 451df4f14c7SRussell King if (res_start > end) 452df4f14c7SRussell King res_start = end; 453df4f14c7SRussell King if (res_end > end) 454df4f14c7SRussell King res_end = end; 455df4f14c7SRussell King if (res_start != start) 45683db0384SJiang Liu free_area_high(start, res_start); 457df4f14c7SRussell King start = res_end; 458df4f14c7SRussell King if (start == end) 459df4f14c7SRussell King break; 460df4f14c7SRussell King } 461df4f14c7SRussell King 462df4f14c7SRussell King /* And now free anything which remains */ 463df4f14c7SRussell King if (start < end) 46483db0384SJiang Liu free_area_high(start, end); 465d0e775afSRussell King } 466d0e775afSRussell King #endif 467d0e775afSRussell King } 468d0e775afSRussell King 4691da177e4SLinus Torvalds /* 4701da177e4SLinus Torvalds * mem_init() marks the free areas in the mem_map and tells us how much 4711da177e4SLinus Torvalds * memory is free. This is done after various parts of the system have 4721da177e4SLinus Torvalds * claimed their memory after the kernel image. 4731da177e4SLinus Torvalds */ 4741da177e4SLinus Torvalds void __init mem_init(void) 4751da177e4SLinus Torvalds { 4761dbd30e9SLinus Walleij #ifdef CONFIG_HAVE_TCM 4771dbd30e9SLinus Walleij /* These pointers are filled in on TCM detection */ 4781dbd30e9SLinus Walleij extern u32 dtcm_end; 4791dbd30e9SLinus Walleij extern u32 itcm_end; 4801dbd30e9SLinus Walleij #endif 4811da177e4SLinus Torvalds 482b3ba41f2SSantosh Shilimkar set_max_mapnr(pfn_to_page(max_pfn) - mem_map); 4831da177e4SLinus Torvalds 4841da177e4SLinus Torvalds /* this will put all unused low memory onto the freelists */ 4851c2f87c2SLaura Abbott free_unused_memmap(); 486c6ffc5caSMike Rapoport memblock_free_all(); 4871da177e4SLinus Torvalds 4881da177e4SLinus Torvalds #ifdef CONFIG_SA1111 4891da177e4SLinus Torvalds /* now that our DMA memory is actually so designated, we can free it */ 490bfd65dd9SLinus Torvalds free_reserved_area(__va(PHYS_OFFSET), swapper_pg_dir, -1, NULL); 4911da177e4SLinus Torvalds #endif 4921da177e4SLinus Torvalds 493d0e775afSRussell King free_highpages(); 4943835f6cbSNicolas Pitre 4952450c973SJiang Liu mem_init_print_info(NULL); 4961da177e4SLinus Torvalds 497db9ef1afSFenkart/Bostandzhyan #define MLK(b, t) b, t, ((t) - (b)) >> 10 498db9ef1afSFenkart/Bostandzhyan #define MLM(b, t) b, t, ((t) - (b)) >> 20 499db9ef1afSFenkart/Bostandzhyan #define MLK_ROUNDUP(b, t) b, t, DIV_ROUND_UP(((t) - (b)), SZ_1K) 500db9ef1afSFenkart/Bostandzhyan 5014ed89f22SRussell King pr_notice("Virtual kernel memory layout:\n" 502db9ef1afSFenkart/Bostandzhyan " vector : 0x%08lx - 0x%08lx (%4ld kB)\n" 50307d2a5c7SLinus Walleij #ifdef CONFIG_HAVE_TCM 50407d2a5c7SLinus Walleij " DTCM : 0x%08lx - 0x%08lx (%4ld kB)\n" 50507d2a5c7SLinus Walleij " ITCM : 0x%08lx - 0x%08lx (%4ld kB)\n" 50607d2a5c7SLinus Walleij #endif 507db9ef1afSFenkart/Bostandzhyan " fixmap : 0x%08lx - 0x%08lx (%4ld kB)\n" 508db9ef1afSFenkart/Bostandzhyan " vmalloc : 0x%08lx - 0x%08lx (%4ld MB)\n" 509db9ef1afSFenkart/Bostandzhyan " lowmem : 0x%08lx - 0x%08lx (%4ld MB)\n" 510db9ef1afSFenkart/Bostandzhyan #ifdef CONFIG_HIGHMEM 511db9ef1afSFenkart/Bostandzhyan " pkmap : 0x%08lx - 0x%08lx (%4ld MB)\n" 512db9ef1afSFenkart/Bostandzhyan #endif 513d9277d51SUwe Kleine-König #ifdef CONFIG_MODULES 514db9ef1afSFenkart/Bostandzhyan " modules : 0x%08lx - 0x%08lx (%4ld MB)\n" 515d9277d51SUwe Kleine-König #endif 516178c3dfeSRussell King " .text : 0x%p" " - 0x%p" " (%4td kB)\n" 517178c3dfeSRussell King " .init : 0x%p" " - 0x%p" " (%4td kB)\n" 518178c3dfeSRussell King " .data : 0x%p" " - 0x%p" " (%4td kB)\n" 519178c3dfeSRussell King " .bss : 0x%p" " - 0x%p" " (%4td kB)\n", 520db9ef1afSFenkart/Bostandzhyan 521d2ca5f24SAfzal Mohammed MLK(VECTORS_BASE, VECTORS_BASE + PAGE_SIZE), 52207d2a5c7SLinus Walleij #ifdef CONFIG_HAVE_TCM 5231dbd30e9SLinus Walleij MLK(DTCM_OFFSET, (unsigned long) dtcm_end), 5241dbd30e9SLinus Walleij MLK(ITCM_OFFSET, (unsigned long) itcm_end), 52507d2a5c7SLinus Walleij #endif 526b615bbbfSMark Salter MLK(FIXADDR_START, FIXADDR_END), 527c931b4f6SFenkart/Bostandzhyan MLM(VMALLOC_START, VMALLOC_END), 528db9ef1afSFenkart/Bostandzhyan MLM(PAGE_OFFSET, (unsigned long)high_memory), 529db9ef1afSFenkart/Bostandzhyan #ifdef CONFIG_HIGHMEM 530db9ef1afSFenkart/Bostandzhyan MLM(PKMAP_BASE, (PKMAP_BASE) + (LAST_PKMAP) * 531db9ef1afSFenkart/Bostandzhyan (PAGE_SIZE)), 532db9ef1afSFenkart/Bostandzhyan #endif 533d9277d51SUwe Kleine-König #ifdef CONFIG_MODULES 534db9ef1afSFenkart/Bostandzhyan MLM(MODULES_VADDR, MODULES_END), 535d9277d51SUwe Kleine-König #endif 536db9ef1afSFenkart/Bostandzhyan 537db9ef1afSFenkart/Bostandzhyan MLK_ROUNDUP(_text, _etext), 5383835d69aSRussell King MLK_ROUNDUP(__init_begin, __init_end), 53945f6d7e0SRabin Vincent MLK_ROUNDUP(_sdata, _edata), 54045f6d7e0SRabin Vincent MLK_ROUNDUP(__bss_start, __bss_stop)); 541db9ef1afSFenkart/Bostandzhyan 542db9ef1afSFenkart/Bostandzhyan #undef MLK 543db9ef1afSFenkart/Bostandzhyan #undef MLM 544db9ef1afSFenkart/Bostandzhyan #undef MLK_ROUNDUP 545db9ef1afSFenkart/Bostandzhyan 546a1839272SFenkart/Bostandzhyan /* 547a1839272SFenkart/Bostandzhyan * Check boundaries twice: Some fundamental inconsistencies can 548a1839272SFenkart/Bostandzhyan * be detected at build time already. 549a1839272SFenkart/Bostandzhyan */ 550a1839272SFenkart/Bostandzhyan #ifdef CONFIG_MMU 551a1839272SFenkart/Bostandzhyan BUILD_BUG_ON(TASK_SIZE > MODULES_VADDR); 552a1839272SFenkart/Bostandzhyan BUG_ON(TASK_SIZE > MODULES_VADDR); 553a1839272SFenkart/Bostandzhyan #endif 554a1839272SFenkart/Bostandzhyan 555a1839272SFenkart/Bostandzhyan #ifdef CONFIG_HIGHMEM 556a1839272SFenkart/Bostandzhyan BUILD_BUG_ON(PKMAP_BASE + LAST_PKMAP * PAGE_SIZE > PAGE_OFFSET); 557a1839272SFenkart/Bostandzhyan BUG_ON(PKMAP_BASE + LAST_PKMAP * PAGE_SIZE > PAGE_OFFSET); 558a1839272SFenkart/Bostandzhyan #endif 5591da177e4SLinus Torvalds } 5601da177e4SLinus Torvalds 5610f5bf6d0SLaura Abbott #ifdef CONFIG_STRICT_KERNEL_RWX 5621e6b4811SKees Cook struct section_perm { 56325362dc4SKees Cook const char *name; 5641e6b4811SKees Cook unsigned long start; 5651e6b4811SKees Cook unsigned long end; 5661e6b4811SKees Cook pmdval_t mask; 5671e6b4811SKees Cook pmdval_t prot; 56880d6b0c2SKees Cook pmdval_t clear; 5691e6b4811SKees Cook }; 5701e6b4811SKees Cook 57164ac2e74SKees Cook /* First section-aligned location at or after __start_rodata. */ 57264ac2e74SKees Cook extern char __start_rodata_section_aligned[]; 57364ac2e74SKees Cook 57480d6b0c2SKees Cook static struct section_perm nx_perms[] = { 5751e6b4811SKees Cook /* Make pages tables, etc before _stext RW (set NX). */ 5761e6b4811SKees Cook { 57725362dc4SKees Cook .name = "pre-text NX", 5781e6b4811SKees Cook .start = PAGE_OFFSET, 5791e6b4811SKees Cook .end = (unsigned long)_stext, 5801e6b4811SKees Cook .mask = ~PMD_SECT_XN, 5811e6b4811SKees Cook .prot = PMD_SECT_XN, 5821e6b4811SKees Cook }, 5831e6b4811SKees Cook /* Make init RW (set NX). */ 5841e6b4811SKees Cook { 58525362dc4SKees Cook .name = "init NX", 5861e6b4811SKees Cook .start = (unsigned long)__init_begin, 5871e6b4811SKees Cook .end = (unsigned long)_sdata, 5881e6b4811SKees Cook .mask = ~PMD_SECT_XN, 5891e6b4811SKees Cook .prot = PMD_SECT_XN, 5901e6b4811SKees Cook }, 59180d6b0c2SKees Cook /* Make rodata NX (set RO in ro_perms below). */ 59280d6b0c2SKees Cook { 59325362dc4SKees Cook .name = "rodata NX", 59464ac2e74SKees Cook .start = (unsigned long)__start_rodata_section_aligned, 59580d6b0c2SKees Cook .end = (unsigned long)__init_begin, 59680d6b0c2SKees Cook .mask = ~PMD_SECT_XN, 59780d6b0c2SKees Cook .prot = PMD_SECT_XN, 59880d6b0c2SKees Cook }, 5991e6b4811SKees Cook }; 6001e6b4811SKees Cook 60180d6b0c2SKees Cook static struct section_perm ro_perms[] = { 60280d6b0c2SKees Cook /* Make kernel code and rodata RX (set RO). */ 60380d6b0c2SKees Cook { 60425362dc4SKees Cook .name = "text/rodata RO", 60580d6b0c2SKees Cook .start = (unsigned long)_stext, 60680d6b0c2SKees Cook .end = (unsigned long)__init_begin, 60780d6b0c2SKees Cook #ifdef CONFIG_ARM_LPAE 608400eeffaSPhilip Derrin .mask = ~(L_PMD_SECT_RDONLY | PMD_SECT_AP2), 609400eeffaSPhilip Derrin .prot = L_PMD_SECT_RDONLY | PMD_SECT_AP2, 61080d6b0c2SKees Cook #else 61180d6b0c2SKees Cook .mask = ~(PMD_SECT_APX | PMD_SECT_AP_WRITE), 61280d6b0c2SKees Cook .prot = PMD_SECT_APX | PMD_SECT_AP_WRITE, 61380d6b0c2SKees Cook .clear = PMD_SECT_AP_WRITE, 61480d6b0c2SKees Cook #endif 61580d6b0c2SKees Cook }, 61680d6b0c2SKees Cook }; 61780d6b0c2SKees Cook 6181e6b4811SKees Cook /* 6191e6b4811SKees Cook * Updates section permissions only for the current mm (sections are 6201e6b4811SKees Cook * copied into each mm). During startup, this is the init_mm. Is only 6211e6b4811SKees Cook * safe to be called with preemption disabled, as under stop_machine(). 6221e6b4811SKees Cook */ 6231e6b4811SKees Cook static inline void section_update(unsigned long addr, pmdval_t mask, 62408925c2fSLaura Abbott pmdval_t prot, struct mm_struct *mm) 6251e6b4811SKees Cook { 6261e6b4811SKees Cook pmd_t *pmd; 6271e6b4811SKees Cook 6281e6b4811SKees Cook pmd = pmd_offset(pud_offset(pgd_offset(mm, addr), addr), addr); 6291e6b4811SKees Cook 6301e6b4811SKees Cook #ifdef CONFIG_ARM_LPAE 6311e6b4811SKees Cook pmd[0] = __pmd((pmd_val(pmd[0]) & mask) | prot); 6321e6b4811SKees Cook #else 6331e6b4811SKees Cook if (addr & SECTION_SIZE) 6341e6b4811SKees Cook pmd[1] = __pmd((pmd_val(pmd[1]) & mask) | prot); 6351e6b4811SKees Cook else 6361e6b4811SKees Cook pmd[0] = __pmd((pmd_val(pmd[0]) & mask) | prot); 6371e6b4811SKees Cook #endif 6381e6b4811SKees Cook flush_pmd_entry(pmd); 6391e6b4811SKees Cook local_flush_tlb_kernel_range(addr, addr + SECTION_SIZE); 6401e6b4811SKees Cook } 6411e6b4811SKees Cook 6421e6b4811SKees Cook /* Make sure extended page tables are in use. */ 6431e6b4811SKees Cook static inline bool arch_has_strict_perms(void) 6441e6b4811SKees Cook { 6451e6b4811SKees Cook if (cpu_architecture() < CPU_ARCH_ARMv6) 6461e6b4811SKees Cook return false; 6471e6b4811SKees Cook 6481e6b4811SKees Cook return !!(get_cr() & CR_XP); 6491e6b4811SKees Cook } 6501e6b4811SKees Cook 65108925c2fSLaura Abbott void set_section_perms(struct section_perm *perms, int n, bool set, 65208925c2fSLaura Abbott struct mm_struct *mm) 65308925c2fSLaura Abbott { 65408925c2fSLaura Abbott size_t i; 65508925c2fSLaura Abbott unsigned long addr; 65608925c2fSLaura Abbott 65708925c2fSLaura Abbott if (!arch_has_strict_perms()) 65808925c2fSLaura Abbott return; 65908925c2fSLaura Abbott 66008925c2fSLaura Abbott for (i = 0; i < n; i++) { 66108925c2fSLaura Abbott if (!IS_ALIGNED(perms[i].start, SECTION_SIZE) || 66208925c2fSLaura Abbott !IS_ALIGNED(perms[i].end, SECTION_SIZE)) { 66325362dc4SKees Cook pr_err("BUG: %s section %lx-%lx not aligned to %lx\n", 66425362dc4SKees Cook perms[i].name, perms[i].start, perms[i].end, 66508925c2fSLaura Abbott SECTION_SIZE); 66608925c2fSLaura Abbott continue; 6671e6b4811SKees Cook } 6681e6b4811SKees Cook 66908925c2fSLaura Abbott for (addr = perms[i].start; 67008925c2fSLaura Abbott addr < perms[i].end; 67108925c2fSLaura Abbott addr += SECTION_SIZE) 67208925c2fSLaura Abbott section_update(addr, perms[i].mask, 67308925c2fSLaura Abbott set ? perms[i].prot : perms[i].clear, mm); 67408925c2fSLaura Abbott } 67508925c2fSLaura Abbott 67608925c2fSLaura Abbott } 67708925c2fSLaura Abbott 67811ce4b33SGrygorii Strashko /** 67911ce4b33SGrygorii Strashko * update_sections_early intended to be called only through stop_machine 68011ce4b33SGrygorii Strashko * framework and executed by only one CPU while all other CPUs will spin and 68111ce4b33SGrygorii Strashko * wait, so no locking is required in this function. 68211ce4b33SGrygorii Strashko */ 68308925c2fSLaura Abbott static void update_sections_early(struct section_perm perms[], int n) 6841e6b4811SKees Cook { 68508925c2fSLaura Abbott struct task_struct *t, *s; 68608925c2fSLaura Abbott 68708925c2fSLaura Abbott for_each_process(t) { 68808925c2fSLaura Abbott if (t->flags & PF_KTHREAD) 68908925c2fSLaura Abbott continue; 69008925c2fSLaura Abbott for_each_thread(t, s) 69108925c2fSLaura Abbott set_section_perms(perms, n, true, s->mm); 69208925c2fSLaura Abbott } 69308925c2fSLaura Abbott set_section_perms(perms, n, true, current->active_mm); 69408925c2fSLaura Abbott set_section_perms(perms, n, true, &init_mm); 69508925c2fSLaura Abbott } 69608925c2fSLaura Abbott 69711ce4b33SGrygorii Strashko static int __fix_kernmem_perms(void *unused) 69808925c2fSLaura Abbott { 69908925c2fSLaura Abbott update_sections_early(nx_perms, ARRAY_SIZE(nx_perms)); 70008925c2fSLaura Abbott return 0; 70108925c2fSLaura Abbott } 70208925c2fSLaura Abbott 70311ce4b33SGrygorii Strashko static void fix_kernmem_perms(void) 70408925c2fSLaura Abbott { 70508925c2fSLaura Abbott stop_machine(__fix_kernmem_perms, NULL, NULL); 7061e6b4811SKees Cook } 70780d6b0c2SKees Cook 70811ce4b33SGrygorii Strashko static int __mark_rodata_ro(void *unused) 70908925c2fSLaura Abbott { 71008925c2fSLaura Abbott update_sections_early(ro_perms, ARRAY_SIZE(ro_perms)); 71108925c2fSLaura Abbott return 0; 71208925c2fSLaura Abbott } 71308925c2fSLaura Abbott 714b4c7e2bdSSteven Rostedt (VMware) static int kernel_set_to_readonly __read_mostly; 715b4c7e2bdSSteven Rostedt (VMware) 71680d6b0c2SKees Cook void mark_rodata_ro(void) 71780d6b0c2SKees Cook { 718b4c7e2bdSSteven Rostedt (VMware) kernel_set_to_readonly = 1; 71908925c2fSLaura Abbott stop_machine(__mark_rodata_ro, NULL, NULL); 720a8e53c15SJinbum Park debug_checkwx(); 72180d6b0c2SKees Cook } 72280d6b0c2SKees Cook 72380d6b0c2SKees Cook void set_kernel_text_rw(void) 72480d6b0c2SKees Cook { 725b4c7e2bdSSteven Rostedt (VMware) if (!kernel_set_to_readonly) 726b4c7e2bdSSteven Rostedt (VMware) return; 727b4c7e2bdSSteven Rostedt (VMware) 72808925c2fSLaura Abbott set_section_perms(ro_perms, ARRAY_SIZE(ro_perms), false, 72908925c2fSLaura Abbott current->active_mm); 73080d6b0c2SKees Cook } 73180d6b0c2SKees Cook 73280d6b0c2SKees Cook void set_kernel_text_ro(void) 73380d6b0c2SKees Cook { 734b4c7e2bdSSteven Rostedt (VMware) if (!kernel_set_to_readonly) 735b4c7e2bdSSteven Rostedt (VMware) return; 736b4c7e2bdSSteven Rostedt (VMware) 73708925c2fSLaura Abbott set_section_perms(ro_perms, ARRAY_SIZE(ro_perms), true, 73808925c2fSLaura Abbott current->active_mm); 73980d6b0c2SKees Cook } 74080d6b0c2SKees Cook 7411e6b4811SKees Cook #else 7421e6b4811SKees Cook static inline void fix_kernmem_perms(void) { } 7430f5bf6d0SLaura Abbott #endif /* CONFIG_STRICT_KERNEL_RWX */ 7441e6b4811SKees Cook 7451e6b4811SKees Cook void free_initmem(void) 7461e6b4811SKees Cook { 7471e6b4811SKees Cook fix_kernmem_perms(); 748bc581770SLinus Walleij 74954d52573SStephen Boyd poison_init_mem(__init_begin, __init_end - __init_begin); 7506db015e4SNicolas Pitre if (!machine_is_integrator() && !machine_is_cintegrator()) 751dbe67df4SJiang Liu free_initmem_default(-1); 7521da177e4SLinus Torvalds } 7531da177e4SLinus Torvalds 7541da177e4SLinus Torvalds #ifdef CONFIG_BLK_DEV_INITRD 7551da177e4SLinus Torvalds 7561da177e4SLinus Torvalds static int keep_initrd; 7571da177e4SLinus Torvalds 7581da177e4SLinus Torvalds void free_initrd_mem(unsigned long start, unsigned long end) 7591da177e4SLinus Torvalds { 76054d52573SStephen Boyd if (!keep_initrd) { 761421520baSYalin Wang if (start == initrd_start) 762421520baSYalin Wang start = round_down(start, PAGE_SIZE); 763421520baSYalin Wang if (end == initrd_end) 764421520baSYalin Wang end = round_up(end, PAGE_SIZE); 765421520baSYalin Wang 76654d52573SStephen Boyd poison_init_mem((void *)start, PAGE_ALIGN(end) - start); 767dbe67df4SJiang Liu free_reserved_area((void *)start, (void *)end, -1, "initrd"); 7681da177e4SLinus Torvalds } 76954d52573SStephen Boyd } 7701da177e4SLinus Torvalds 7711da177e4SLinus Torvalds static int __init keepinitrd_setup(char *__unused) 7721da177e4SLinus Torvalds { 7731da177e4SLinus Torvalds keep_initrd = 1; 7741da177e4SLinus Torvalds return 1; 7751da177e4SLinus Torvalds } 7761da177e4SLinus Torvalds 7771da177e4SLinus Torvalds __setup("keepinitrd", keepinitrd_setup); 7781da177e4SLinus Torvalds #endif 779