1d2912cb1SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only 21da177e4SLinus Torvalds /* 31da177e4SLinus Torvalds * linux/arch/arm/mm/init.c 41da177e4SLinus Torvalds * 590072059SRussell King * Copyright (C) 1995-2005 Russell King 61da177e4SLinus Torvalds */ 71da177e4SLinus Torvalds #include <linux/kernel.h> 81da177e4SLinus Torvalds #include <linux/errno.h> 91da177e4SLinus Torvalds #include <linux/swap.h> 101da177e4SLinus Torvalds #include <linux/init.h> 111da177e4SLinus Torvalds #include <linux/mman.h> 123f07c014SIngo Molnar #include <linux/sched/signal.h> 1329930025SIngo Molnar #include <linux/sched/task.h> 14dc28094bSPaul Gortmaker #include <linux/export.h> 151da177e4SLinus Torvalds #include <linux/nodemask.h> 161da177e4SLinus Torvalds #include <linux/initrd.h> 179eb8f674SGrant Likely #include <linux/of_fdt.h> 183835f6cbSNicolas Pitre #include <linux/highmem.h> 195a0e3ad6STejun Heo #include <linux/gfp.h> 202778f620SRussell King #include <linux/memblock.h> 210b1abd1fSChristoph Hellwig #include <linux/dma-map-ops.h> 22158e8bfeSAlessandro Rubini #include <linux/sizes.h> 2308925c2fSLaura Abbott #include <linux/stop_machine.h> 24ad3c7b18SChristoph Hellwig #include <linux/swiotlb.h> 251da177e4SLinus Torvalds 26b4b20ad8SRussell King #include <asm/cp15.h> 271da177e4SLinus Torvalds #include <asm/mach-types.h> 28716a3dc2SRussell King #include <asm/memblock.h> 29d2ca5f24SAfzal Mohammed #include <asm/memory.h> 3093c02ab4SGrant Likely #include <asm/prom.h> 3137efe642SRussell King #include <asm/sections.h> 321da177e4SLinus Torvalds #include <asm/setup.h> 339110f3e7SBen Dooks (Codethink) #include <asm/set_memory.h> 341e6b4811SKees Cook #include <asm/system_info.h> 351da177e4SLinus Torvalds #include <asm/tlb.h> 36db9ef1afSFenkart/Bostandzhyan #include <asm/fixmap.h> 37a8e53c15SJinbum Park #include <asm/ptdump.h> 381da177e4SLinus Torvalds 391da177e4SLinus Torvalds #include <asm/mach/arch.h> 401da177e4SLinus Torvalds #include <asm/mach/map.h> 411da177e4SLinus Torvalds 421b2e2b73SRussell King #include "mm.h" 431b2e2b73SRussell King 44b4b20ad8SRussell King #ifdef CONFIG_CPU_CP15_MMU 45b4b20ad8SRussell King unsigned long __init __clear_cr(unsigned long mask) 46b4b20ad8SRussell King { 47b4b20ad8SRussell King cr_alignment = cr_alignment & ~mask; 48b4b20ad8SRussell King return cr_alignment; 49b4b20ad8SRussell King } 50b4b20ad8SRussell King #endif 51b4b20ad8SRussell King 52b1ab95c6SFlorian Fainelli #ifdef CONFIG_BLK_DEV_INITRD 53012d1f4aSRussell King static int __init parse_tag_initrd(const struct tag *tag) 54012d1f4aSRussell King { 554ed89f22SRussell King pr_warn("ATAG_INITRD is deprecated; " 56012d1f4aSRussell King "please update your bootloader.\n"); 57012d1f4aSRussell King phys_initrd_start = __virt_to_phys(tag->u.initrd.start); 58012d1f4aSRussell King phys_initrd_size = tag->u.initrd.size; 59012d1f4aSRussell King return 0; 60012d1f4aSRussell King } 61012d1f4aSRussell King 62012d1f4aSRussell King __tagtable(ATAG_INITRD, parse_tag_initrd); 63012d1f4aSRussell King 64012d1f4aSRussell King static int __init parse_tag_initrd2(const struct tag *tag) 65012d1f4aSRussell King { 66012d1f4aSRussell King phys_initrd_start = tag->u.initrd.start; 67012d1f4aSRussell King phys_initrd_size = tag->u.initrd.size; 68012d1f4aSRussell King return 0; 69012d1f4aSRussell King } 70012d1f4aSRussell King 71012d1f4aSRussell King __tagtable(ATAG_INITRD2, parse_tag_initrd2); 72b1ab95c6SFlorian Fainelli #endif 731da177e4SLinus Torvalds 74f25b4b4cSRussell King static void __init find_limits(unsigned long *min, unsigned long *max_low, 75f25b4b4cSRussell King unsigned long *max_high) 76dde5828fSRussell King { 771c2f87c2SLaura Abbott *max_low = PFN_DOWN(memblock_get_current_limit()); 781c2f87c2SLaura Abbott *min = PFN_UP(memblock_start_of_DRAM()); 791c2f87c2SLaura Abbott *max_high = PFN_DOWN(memblock_end_of_DRAM()); 80dde5828fSRussell King } 81dde5828fSRussell King 82be20902bSRussell King #ifdef CONFIG_ZONE_DMA 8365032018SNicolas Pitre 84364230b9SRob Herring phys_addr_t arm_dma_zone_size __read_mostly; 8565032018SNicolas Pitre EXPORT_SYMBOL(arm_dma_zone_size); 8665032018SNicolas Pitre 87022ae537SRussell King /* 88022ae537SRussell King * The DMA mask corresponding to the maximum bus address allocatable 89022ae537SRussell King * using GFP_DMA. The default here places no restriction on DMA 90022ae537SRussell King * allocations. This must be the smallest DMA mask in the system, 91022ae537SRussell King * so a successful GFP_DMA allocation will always satisfy this. 92022ae537SRussell King */ 934986e5c7SMarek Szyprowski phys_addr_t arm_dma_limit; 944dcfa600SRussell King unsigned long arm_dma_pfn_limit; 95be20902bSRussell King #endif 96be20902bSRussell King 97ff69a4c8SRussell King void __init setup_dma_zone(const struct machine_desc *mdesc) 98c7909509SMarek Szyprowski { 99c7909509SMarek Szyprowski #ifdef CONFIG_ZONE_DMA 100c7909509SMarek Szyprowski if (mdesc->dma_zone_size) { 101c7909509SMarek Szyprowski arm_dma_zone_size = mdesc->dma_zone_size; 1026bcac805SRussell King arm_dma_limit = PHYS_OFFSET + arm_dma_zone_size - 1; 103c7909509SMarek Szyprowski } else 104c7909509SMarek Szyprowski arm_dma_limit = 0xffffffff; 1054dcfa600SRussell King arm_dma_pfn_limit = arm_dma_limit >> PAGE_SHIFT; 106c7909509SMarek Szyprowski #endif 107c7909509SMarek Szyprowski } 108c7909509SMarek Szyprowski 10984f452b1SSantosh Shilimkar static void __init zone_sizes_init(unsigned long min, unsigned long max_low, 110a2c54d2aSRussell King unsigned long max_high) 111b7a69ac3SRussell King { 112a32c1c61SMike Rapoport unsigned long max_zone_pfn[MAX_NR_ZONES] = { 0 }; 11390072059SRussell King 11465032018SNicolas Pitre #ifdef CONFIG_ZONE_DMA 115a32c1c61SMike Rapoport max_zone_pfn[ZONE_DMA] = min(arm_dma_pfn_limit, max_low); 116be20902bSRussell King #endif 117a32c1c61SMike Rapoport max_zone_pfn[ZONE_NORMAL] = max_low; 118a32c1c61SMike Rapoport #ifdef CONFIG_HIGHMEM 119a32c1c61SMike Rapoport max_zone_pfn[ZONE_HIGHMEM] = max_high; 120a32c1c61SMike Rapoport #endif 121a32c1c61SMike Rapoport free_area_init(max_zone_pfn); 12290072059SRussell King } 12390072059SRussell King 1247b7bf499SWill Deacon #ifdef CONFIG_HAVE_ARCH_PFN_VALID 125b7cfda9fSRussell King int pfn_valid(unsigned long pfn) 126b7cfda9fSRussell King { 1275b3efa4fSzhaoyang phys_addr_t addr = __pfn_to_phys(pfn); 128*a4d5613cSMike Rapoport unsigned long pageblock_size = PAGE_SIZE * pageblock_nr_pages; 1295b3efa4fSzhaoyang 1305b3efa4fSzhaoyang if (__phys_to_pfn(addr) != pfn) 1315b3efa4fSzhaoyang return 0; 1325b3efa4fSzhaoyang 133*a4d5613cSMike Rapoport /* 134*a4d5613cSMike Rapoport * If address less than pageblock_size bytes away from a present 135*a4d5613cSMike Rapoport * memory chunk there still will be a memory map entry for it 136*a4d5613cSMike Rapoport * because we round freed memory map to the pageblock boundaries. 137*a4d5613cSMike Rapoport */ 138*a4d5613cSMike Rapoport if (memblock_overlaps_region(&memblock.memory, 139*a4d5613cSMike Rapoport ALIGN_DOWN(addr, pageblock_size), 140*a4d5613cSMike Rapoport pageblock_size)) 141*a4d5613cSMike Rapoport return 1; 142*a4d5613cSMike Rapoport 143*a4d5613cSMike Rapoport return 0; 144b7cfda9fSRussell King } 145b7cfda9fSRussell King EXPORT_SYMBOL(pfn_valid); 1467b7bf499SWill Deacon #endif 147657e12fdSRussell King 148716a3dc2SRussell King static bool arm_memblock_steal_permitted = true; 149716a3dc2SRussell King 150bc2827d0SRussell King phys_addr_t __init arm_memblock_steal(phys_addr_t size, phys_addr_t align) 151716a3dc2SRussell King { 152716a3dc2SRussell King phys_addr_t phys; 153716a3dc2SRussell King 154716a3dc2SRussell King BUG_ON(!arm_memblock_steal_permitted); 155716a3dc2SRussell King 156f240ec09SMike Rapoport phys = memblock_phys_alloc(size, align); 157ecc3e771SMike Rapoport if (!phys) 158ecc3e771SMike Rapoport panic("Failed to steal %pa bytes at %pS\n", 159ecc3e771SMike Rapoport &size, (void *)_RET_IP_); 160ecc3e771SMike Rapoport 161716a3dc2SRussell King memblock_free(phys, size); 162716a3dc2SRussell King memblock_remove(phys, size); 163716a3dc2SRussell King 164716a3dc2SRussell King return phys; 165716a3dc2SRussell King } 166716a3dc2SRussell King 16739286248SRussell King static void __init arm_initrd_init(void) 1682778f620SRussell King { 1692778f620SRussell King #ifdef CONFIG_BLK_DEV_INITRD 170cdcc5fa0SRussell King phys_addr_t start; 171cdcc5fa0SRussell King unsigned long size; 172cdcc5fa0SRussell King 1734c235cb9SBen Peddell initrd_start = initrd_end = 0; 17468b32f36SRussell King 17568b32f36SRussell King if (!phys_initrd_size) 17668b32f36SRussell King return; 17768b32f36SRussell King 178cdcc5fa0SRussell King /* 179cdcc5fa0SRussell King * Round the memory region to page boundaries as per free_initrd_mem() 180cdcc5fa0SRussell King * This allows us to detect whether the pages overlapping the initrd 181cdcc5fa0SRussell King * are in use, but more importantly, reserves the entire set of pages 182cdcc5fa0SRussell King * as we don't want these pages allocated for other purposes. 183cdcc5fa0SRussell King */ 184cdcc5fa0SRussell King start = round_down(phys_initrd_start, PAGE_SIZE); 185cdcc5fa0SRussell King size = phys_initrd_size + (phys_initrd_start - start); 186cdcc5fa0SRussell King size = round_up(size, PAGE_SIZE); 187cdcc5fa0SRussell King 188cdcc5fa0SRussell King if (!memblock_is_region_memory(start, size)) { 189de22cc6eSVitaly Andrianov pr_err("INITRD: 0x%08llx+0x%08lx is not a memory region - disabling initrd\n", 190cdcc5fa0SRussell King (u64)start, size); 19168b32f36SRussell King return; 1928f4b8c76SRussell King } 19368b32f36SRussell King 194cdcc5fa0SRussell King if (memblock_is_region_reserved(start, size)) { 195de22cc6eSVitaly Andrianov pr_err("INITRD: 0x%08llx+0x%08lx overlaps in-use memory region - disabling initrd\n", 196cdcc5fa0SRussell King (u64)start, size); 19768b32f36SRussell King return; 198b0a2679dSRussell King } 19968b32f36SRussell King 200cdcc5fa0SRussell King memblock_reserve(start, size); 2012778f620SRussell King 2022778f620SRussell King /* Now convert initrd to virtual addresses */ 2032778f620SRussell King initrd_start = __phys_to_virt(phys_initrd_start); 2042778f620SRussell King initrd_end = initrd_start + phys_initrd_size; 2052778f620SRussell King #endif 20639286248SRussell King } 20739286248SRussell King 2085f41f919SMarek Szyprowski #ifdef CONFIG_CPU_ICACHE_MISMATCH_WORKAROUND 2095f41f919SMarek Szyprowski void check_cpu_icache_size(int cpuid) 2105f41f919SMarek Szyprowski { 2115f41f919SMarek Szyprowski u32 size, ctr; 2125f41f919SMarek Szyprowski 2135f41f919SMarek Szyprowski asm("mrc p15, 0, %0, c0, c0, 1" : "=r" (ctr)); 2145f41f919SMarek Szyprowski 2155f41f919SMarek Szyprowski size = 1 << ((ctr & 0xf) + 2); 2165f41f919SMarek Szyprowski if (cpuid != 0 && icache_size != size) 2175f41f919SMarek Szyprowski pr_info("CPU%u: detected I-Cache line size mismatch, workaround enabled\n", 2185f41f919SMarek Szyprowski cpuid); 2195f41f919SMarek Szyprowski if (icache_size > size) 2205f41f919SMarek Szyprowski icache_size = size; 2215f41f919SMarek Szyprowski } 2225f41f919SMarek Szyprowski #endif 2235f41f919SMarek Szyprowski 22439286248SRussell King void __init arm_memblock_init(const struct machine_desc *mdesc) 22539286248SRussell King { 22639286248SRussell King /* Register the kernel text, kernel data and initrd with memblock. */ 22739286248SRussell King memblock_reserve(__pa(KERNEL_START), KERNEL_END - KERNEL_START); 22839286248SRussell King 22939286248SRussell King arm_initrd_init(); 2302778f620SRussell King 2312778f620SRussell King arm_mm_memblock_reserve(); 2322778f620SRussell King 2338d717a52SRussell King /* reserve any platform specific memblock areas */ 2348d717a52SRussell King if (mdesc->reserve) 2358d717a52SRussell King mdesc->reserve(); 2368d717a52SRussell King 237bcedb5f9SMarek Szyprowski early_init_fdt_scan_reserved_mem(); 238bcedb5f9SMarek Szyprowski 23999a468d7SGeorge G. Davis /* reserve memory for DMA contiguous allocations */ 24095b0e655SMarek Szyprowski dma_contiguous_reserve(arm_dma_limit); 241c7909509SMarek Szyprowski 242716a3dc2SRussell King arm_memblock_steal_permitted = false; 2432778f620SRussell King memblock_dump_all(); 2442778f620SRussell King } 2452778f620SRussell King 2468d717a52SRussell King void __init bootmem_init(void) 24790072059SRussell King { 2488e58caefSGrygorii Strashko memblock_allow_resize(); 249dde5828fSRussell King 250071d184aSDoug Berger find_limits(&min_low_pfn, &max_low_pfn, &max_pfn); 251dde5828fSRussell King 252071d184aSDoug Berger early_memtest((phys_addr_t)min_low_pfn << PAGE_SHIFT, 253071d184aSDoug Berger (phys_addr_t)max_low_pfn << PAGE_SHIFT); 254d30eae47SVladimir Murzin 255b7a69ac3SRussell King /* 256c89ab04fSMike Rapoport * sparse_init() tries to allocate memory from memblock, so must be 257c89ab04fSMike Rapoport * done after the fixed reservations 258b7a69ac3SRussell King */ 259b7a69ac3SRussell King sparse_init(); 260b7a69ac3SRussell King 261b7a69ac3SRussell King /* 262a32c1c61SMike Rapoport * Now free the memory - free_area_init needs 263b7a69ac3SRussell King * the sparse mem_map arrays initialized by sparse_init() 264b7a69ac3SRussell King * for memmap_init_zone(), otherwise all PFNs are invalid. 265b7a69ac3SRussell King */ 266071d184aSDoug Berger zone_sizes_init(min_low_pfn, max_low_pfn, max_pfn); 26790072059SRussell King } 26890072059SRussell King 26954d52573SStephen Boyd /* 27054d52573SStephen Boyd * Poison init memory with an undefined instruction (ARM) or a branch to an 27154d52573SStephen Boyd * undefined instruction (Thumb). 27254d52573SStephen Boyd */ 27354d52573SStephen Boyd static inline void poison_init_mem(void *s, size_t count) 27454d52573SStephen Boyd { 27554d52573SStephen Boyd u32 *p = (u32 *)s; 276bf912d99SJamie Iles for (; count != 0; count -= 4) 27754d52573SStephen Boyd *p++ = 0xe7fddef0; 27854d52573SStephen Boyd } 27954d52573SStephen Boyd 280d0e775afSRussell King static void __init free_highpages(void) 281d0e775afSRussell King { 282d0e775afSRussell King #ifdef CONFIG_HIGHMEM 28326ba47b1SSantosh Shilimkar unsigned long max_low = max_low_pfn; 284cddb5ddfSMike Rapoport phys_addr_t range_start, range_end; 285cddb5ddfSMike Rapoport u64 i; 286d0e775afSRussell King 287d0e775afSRussell King /* set highmem page free */ 288cddb5ddfSMike Rapoport for_each_free_mem_range(i, NUMA_NO_NODE, MEMBLOCK_NONE, 289cddb5ddfSMike Rapoport &range_start, &range_end, NULL) { 290b9bc3670SArd Biesheuvel unsigned long start = PFN_UP(range_start); 291b9bc3670SArd Biesheuvel unsigned long end = PFN_DOWN(range_end); 292df4f14c7SRussell King 293df4f14c7SRussell King /* Ignore complete lowmem entries */ 294df4f14c7SRussell King if (end <= max_low) 295df4f14c7SRussell King continue; 296df4f14c7SRussell King 297df4f14c7SRussell King /* Truncate partial highmem entries */ 298df4f14c7SRussell King if (start < max_low) 299df4f14c7SRussell King start = max_low; 300df4f14c7SRussell King 301cddb5ddfSMike Rapoport for (; start < end; start++) 302cddb5ddfSMike Rapoport free_highmem_page(pfn_to_page(start)); 303d0e775afSRussell King } 304d0e775afSRussell King #endif 305d0e775afSRussell King } 306d0e775afSRussell King 3071da177e4SLinus Torvalds /* 3081da177e4SLinus Torvalds * mem_init() marks the free areas in the mem_map and tells us how much 3091da177e4SLinus Torvalds * memory is free. This is done after various parts of the system have 3101da177e4SLinus Torvalds * claimed their memory after the kernel image. 3111da177e4SLinus Torvalds */ 3121da177e4SLinus Torvalds void __init mem_init(void) 3131da177e4SLinus Torvalds { 314ad3c7b18SChristoph Hellwig #ifdef CONFIG_ARM_LPAE 315fcf04489SFlorian Fainelli if (swiotlb_force == SWIOTLB_FORCE || 316fcf04489SFlorian Fainelli max_pfn > arm_dma_pfn_limit) 317ad3c7b18SChristoph Hellwig swiotlb_init(1); 318fcf04489SFlorian Fainelli else 319fcf04489SFlorian Fainelli swiotlb_force = SWIOTLB_NO_FORCE; 320ad3c7b18SChristoph Hellwig #endif 321ad3c7b18SChristoph Hellwig 322b3ba41f2SSantosh Shilimkar set_max_mapnr(pfn_to_page(max_pfn) - mem_map); 3231da177e4SLinus Torvalds 3241da177e4SLinus Torvalds /* this will put all unused low memory onto the freelists */ 325c6ffc5caSMike Rapoport memblock_free_all(); 3261da177e4SLinus Torvalds 3271da177e4SLinus Torvalds #ifdef CONFIG_SA1111 3281da177e4SLinus Torvalds /* now that our DMA memory is actually so designated, we can free it */ 329bfd65dd9SLinus Torvalds free_reserved_area(__va(PHYS_OFFSET), swapper_pg_dir, -1, NULL); 3301da177e4SLinus Torvalds #endif 3311da177e4SLinus Torvalds 332d0e775afSRussell King free_highpages(); 3333835f6cbSNicolas Pitre 334a1839272SFenkart/Bostandzhyan /* 335a1839272SFenkart/Bostandzhyan * Check boundaries twice: Some fundamental inconsistencies can 336a1839272SFenkart/Bostandzhyan * be detected at build time already. 337a1839272SFenkart/Bostandzhyan */ 338a1839272SFenkart/Bostandzhyan #ifdef CONFIG_MMU 339a1839272SFenkart/Bostandzhyan BUILD_BUG_ON(TASK_SIZE > MODULES_VADDR); 340a1839272SFenkart/Bostandzhyan BUG_ON(TASK_SIZE > MODULES_VADDR); 341a1839272SFenkart/Bostandzhyan #endif 342a1839272SFenkart/Bostandzhyan 343a1839272SFenkart/Bostandzhyan #ifdef CONFIG_HIGHMEM 344a1839272SFenkart/Bostandzhyan BUILD_BUG_ON(PKMAP_BASE + LAST_PKMAP * PAGE_SIZE > PAGE_OFFSET); 345a1839272SFenkart/Bostandzhyan BUG_ON(PKMAP_BASE + LAST_PKMAP * PAGE_SIZE > PAGE_OFFSET); 346a1839272SFenkart/Bostandzhyan #endif 3471da177e4SLinus Torvalds } 3481da177e4SLinus Torvalds 3490f5bf6d0SLaura Abbott #ifdef CONFIG_STRICT_KERNEL_RWX 3501e6b4811SKees Cook struct section_perm { 35125362dc4SKees Cook const char *name; 3521e6b4811SKees Cook unsigned long start; 3531e6b4811SKees Cook unsigned long end; 3541e6b4811SKees Cook pmdval_t mask; 3551e6b4811SKees Cook pmdval_t prot; 35680d6b0c2SKees Cook pmdval_t clear; 3571e6b4811SKees Cook }; 3581e6b4811SKees Cook 35964ac2e74SKees Cook /* First section-aligned location at or after __start_rodata. */ 36064ac2e74SKees Cook extern char __start_rodata_section_aligned[]; 36164ac2e74SKees Cook 36280d6b0c2SKees Cook static struct section_perm nx_perms[] = { 3631e6b4811SKees Cook /* Make pages tables, etc before _stext RW (set NX). */ 3641e6b4811SKees Cook { 36525362dc4SKees Cook .name = "pre-text NX", 3661e6b4811SKees Cook .start = PAGE_OFFSET, 3671e6b4811SKees Cook .end = (unsigned long)_stext, 3681e6b4811SKees Cook .mask = ~PMD_SECT_XN, 3691e6b4811SKees Cook .prot = PMD_SECT_XN, 3701e6b4811SKees Cook }, 3711e6b4811SKees Cook /* Make init RW (set NX). */ 3721e6b4811SKees Cook { 37325362dc4SKees Cook .name = "init NX", 3741e6b4811SKees Cook .start = (unsigned long)__init_begin, 3751e6b4811SKees Cook .end = (unsigned long)_sdata, 3761e6b4811SKees Cook .mask = ~PMD_SECT_XN, 3771e6b4811SKees Cook .prot = PMD_SECT_XN, 3781e6b4811SKees Cook }, 37980d6b0c2SKees Cook /* Make rodata NX (set RO in ro_perms below). */ 38080d6b0c2SKees Cook { 38125362dc4SKees Cook .name = "rodata NX", 38264ac2e74SKees Cook .start = (unsigned long)__start_rodata_section_aligned, 38380d6b0c2SKees Cook .end = (unsigned long)__init_begin, 38480d6b0c2SKees Cook .mask = ~PMD_SECT_XN, 38580d6b0c2SKees Cook .prot = PMD_SECT_XN, 38680d6b0c2SKees Cook }, 3871e6b4811SKees Cook }; 3881e6b4811SKees Cook 38980d6b0c2SKees Cook static struct section_perm ro_perms[] = { 39080d6b0c2SKees Cook /* Make kernel code and rodata RX (set RO). */ 39180d6b0c2SKees Cook { 39225362dc4SKees Cook .name = "text/rodata RO", 39380d6b0c2SKees Cook .start = (unsigned long)_stext, 39480d6b0c2SKees Cook .end = (unsigned long)__init_begin, 39580d6b0c2SKees Cook #ifdef CONFIG_ARM_LPAE 396400eeffaSPhilip Derrin .mask = ~(L_PMD_SECT_RDONLY | PMD_SECT_AP2), 397400eeffaSPhilip Derrin .prot = L_PMD_SECT_RDONLY | PMD_SECT_AP2, 39880d6b0c2SKees Cook #else 39980d6b0c2SKees Cook .mask = ~(PMD_SECT_APX | PMD_SECT_AP_WRITE), 40080d6b0c2SKees Cook .prot = PMD_SECT_APX | PMD_SECT_AP_WRITE, 40180d6b0c2SKees Cook .clear = PMD_SECT_AP_WRITE, 40280d6b0c2SKees Cook #endif 40380d6b0c2SKees Cook }, 40480d6b0c2SKees Cook }; 40580d6b0c2SKees Cook 4061e6b4811SKees Cook /* 4071e6b4811SKees Cook * Updates section permissions only for the current mm (sections are 4081e6b4811SKees Cook * copied into each mm). During startup, this is the init_mm. Is only 4091e6b4811SKees Cook * safe to be called with preemption disabled, as under stop_machine(). 4101e6b4811SKees Cook */ 4111e6b4811SKees Cook static inline void section_update(unsigned long addr, pmdval_t mask, 41208925c2fSLaura Abbott pmdval_t prot, struct mm_struct *mm) 4131e6b4811SKees Cook { 4141e6b4811SKees Cook pmd_t *pmd; 4151e6b4811SKees Cook 41684e6ffb2SMike Rapoport pmd = pmd_offset(pud_offset(p4d_offset(pgd_offset(mm, addr), addr), addr), addr); 4171e6b4811SKees Cook 4181e6b4811SKees Cook #ifdef CONFIG_ARM_LPAE 4191e6b4811SKees Cook pmd[0] = __pmd((pmd_val(pmd[0]) & mask) | prot); 4201e6b4811SKees Cook #else 4211e6b4811SKees Cook if (addr & SECTION_SIZE) 4221e6b4811SKees Cook pmd[1] = __pmd((pmd_val(pmd[1]) & mask) | prot); 4231e6b4811SKees Cook else 4241e6b4811SKees Cook pmd[0] = __pmd((pmd_val(pmd[0]) & mask) | prot); 4251e6b4811SKees Cook #endif 4261e6b4811SKees Cook flush_pmd_entry(pmd); 4271e6b4811SKees Cook local_flush_tlb_kernel_range(addr, addr + SECTION_SIZE); 4281e6b4811SKees Cook } 4291e6b4811SKees Cook 4301e6b4811SKees Cook /* Make sure extended page tables are in use. */ 4311e6b4811SKees Cook static inline bool arch_has_strict_perms(void) 4321e6b4811SKees Cook { 4331e6b4811SKees Cook if (cpu_architecture() < CPU_ARCH_ARMv6) 4341e6b4811SKees Cook return false; 4351e6b4811SKees Cook 4361e6b4811SKees Cook return !!(get_cr() & CR_XP); 4371e6b4811SKees Cook } 4381e6b4811SKees Cook 439ea5379beSBen Dooks (Codethink) static void set_section_perms(struct section_perm *perms, int n, bool set, 44008925c2fSLaura Abbott struct mm_struct *mm) 44108925c2fSLaura Abbott { 44208925c2fSLaura Abbott size_t i; 44308925c2fSLaura Abbott unsigned long addr; 44408925c2fSLaura Abbott 44508925c2fSLaura Abbott if (!arch_has_strict_perms()) 44608925c2fSLaura Abbott return; 44708925c2fSLaura Abbott 44808925c2fSLaura Abbott for (i = 0; i < n; i++) { 44908925c2fSLaura Abbott if (!IS_ALIGNED(perms[i].start, SECTION_SIZE) || 45008925c2fSLaura Abbott !IS_ALIGNED(perms[i].end, SECTION_SIZE)) { 45125362dc4SKees Cook pr_err("BUG: %s section %lx-%lx not aligned to %lx\n", 45225362dc4SKees Cook perms[i].name, perms[i].start, perms[i].end, 45308925c2fSLaura Abbott SECTION_SIZE); 45408925c2fSLaura Abbott continue; 4551e6b4811SKees Cook } 4561e6b4811SKees Cook 45708925c2fSLaura Abbott for (addr = perms[i].start; 45808925c2fSLaura Abbott addr < perms[i].end; 45908925c2fSLaura Abbott addr += SECTION_SIZE) 46008925c2fSLaura Abbott section_update(addr, perms[i].mask, 46108925c2fSLaura Abbott set ? perms[i].prot : perms[i].clear, mm); 46208925c2fSLaura Abbott } 46308925c2fSLaura Abbott 46408925c2fSLaura Abbott } 46508925c2fSLaura Abbott 46611ce4b33SGrygorii Strashko /** 46711ce4b33SGrygorii Strashko * update_sections_early intended to be called only through stop_machine 46811ce4b33SGrygorii Strashko * framework and executed by only one CPU while all other CPUs will spin and 46911ce4b33SGrygorii Strashko * wait, so no locking is required in this function. 47011ce4b33SGrygorii Strashko */ 47108925c2fSLaura Abbott static void update_sections_early(struct section_perm perms[], int n) 4721e6b4811SKees Cook { 47308925c2fSLaura Abbott struct task_struct *t, *s; 47408925c2fSLaura Abbott 47508925c2fSLaura Abbott for_each_process(t) { 47608925c2fSLaura Abbott if (t->flags & PF_KTHREAD) 47708925c2fSLaura Abbott continue; 47808925c2fSLaura Abbott for_each_thread(t, s) 479c51bc12dSDoug Berger if (s->mm) 48008925c2fSLaura Abbott set_section_perms(perms, n, true, s->mm); 48108925c2fSLaura Abbott } 48208925c2fSLaura Abbott set_section_perms(perms, n, true, current->active_mm); 48308925c2fSLaura Abbott set_section_perms(perms, n, true, &init_mm); 48408925c2fSLaura Abbott } 48508925c2fSLaura Abbott 48611ce4b33SGrygorii Strashko static int __fix_kernmem_perms(void *unused) 48708925c2fSLaura Abbott { 48808925c2fSLaura Abbott update_sections_early(nx_perms, ARRAY_SIZE(nx_perms)); 48908925c2fSLaura Abbott return 0; 49008925c2fSLaura Abbott } 49108925c2fSLaura Abbott 49211ce4b33SGrygorii Strashko static void fix_kernmem_perms(void) 49308925c2fSLaura Abbott { 49408925c2fSLaura Abbott stop_machine(__fix_kernmem_perms, NULL, NULL); 4951e6b4811SKees Cook } 49680d6b0c2SKees Cook 49711ce4b33SGrygorii Strashko static int __mark_rodata_ro(void *unused) 49808925c2fSLaura Abbott { 49908925c2fSLaura Abbott update_sections_early(ro_perms, ARRAY_SIZE(ro_perms)); 50008925c2fSLaura Abbott return 0; 50108925c2fSLaura Abbott } 50208925c2fSLaura Abbott 50380d6b0c2SKees Cook void mark_rodata_ro(void) 50480d6b0c2SKees Cook { 50508925c2fSLaura Abbott stop_machine(__mark_rodata_ro, NULL, NULL); 506a8e53c15SJinbum Park debug_checkwx(); 50780d6b0c2SKees Cook } 50880d6b0c2SKees Cook 5091e6b4811SKees Cook #else 5101e6b4811SKees Cook static inline void fix_kernmem_perms(void) { } 5110f5bf6d0SLaura Abbott #endif /* CONFIG_STRICT_KERNEL_RWX */ 5121e6b4811SKees Cook 5131e6b4811SKees Cook void free_initmem(void) 5141e6b4811SKees Cook { 5151e6b4811SKees Cook fix_kernmem_perms(); 516bc581770SLinus Walleij 51754d52573SStephen Boyd poison_init_mem(__init_begin, __init_end - __init_begin); 5186db015e4SNicolas Pitre if (!machine_is_integrator() && !machine_is_cintegrator()) 519dbe67df4SJiang Liu free_initmem_default(-1); 5201da177e4SLinus Torvalds } 5211da177e4SLinus Torvalds 5221da177e4SLinus Torvalds #ifdef CONFIG_BLK_DEV_INITRD 5231da177e4SLinus Torvalds void free_initrd_mem(unsigned long start, unsigned long end) 5241da177e4SLinus Torvalds { 525421520baSYalin Wang if (start == initrd_start) 526421520baSYalin Wang start = round_down(start, PAGE_SIZE); 527421520baSYalin Wang if (end == initrd_end) 528421520baSYalin Wang end = round_up(end, PAGE_SIZE); 529421520baSYalin Wang 53054d52573SStephen Boyd poison_init_mem((void *)start, PAGE_ALIGN(end) - start); 531dbe67df4SJiang Liu free_reserved_area((void *)start, (void *)end, -1, "initrd"); 5321da177e4SLinus Torvalds } 5331da177e4SLinus Torvalds #endif 534