1d2912cb1SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only 21da177e4SLinus Torvalds /* 31da177e4SLinus Torvalds * linux/arch/arm/mm/init.c 41da177e4SLinus Torvalds * 590072059SRussell King * Copyright (C) 1995-2005 Russell King 61da177e4SLinus Torvalds */ 71da177e4SLinus Torvalds #include <linux/kernel.h> 81da177e4SLinus Torvalds #include <linux/errno.h> 91da177e4SLinus Torvalds #include <linux/swap.h> 101da177e4SLinus Torvalds #include <linux/init.h> 111da177e4SLinus Torvalds #include <linux/mman.h> 123f07c014SIngo Molnar #include <linux/sched/signal.h> 1329930025SIngo Molnar #include <linux/sched/task.h> 14dc28094bSPaul Gortmaker #include <linux/export.h> 151da177e4SLinus Torvalds #include <linux/nodemask.h> 161da177e4SLinus Torvalds #include <linux/initrd.h> 179eb8f674SGrant Likely #include <linux/of_fdt.h> 183835f6cbSNicolas Pitre #include <linux/highmem.h> 195a0e3ad6STejun Heo #include <linux/gfp.h> 202778f620SRussell King #include <linux/memblock.h> 210b1abd1fSChristoph Hellwig #include <linux/dma-map-ops.h> 22158e8bfeSAlessandro Rubini #include <linux/sizes.h> 2308925c2fSLaura Abbott #include <linux/stop_machine.h> 24ad3c7b18SChristoph Hellwig #include <linux/swiotlb.h> 251da177e4SLinus Torvalds 26b4b20ad8SRussell King #include <asm/cp15.h> 271da177e4SLinus Torvalds #include <asm/mach-types.h> 28716a3dc2SRussell King #include <asm/memblock.h> 29d2ca5f24SAfzal Mohammed #include <asm/memory.h> 3093c02ab4SGrant Likely #include <asm/prom.h> 3137efe642SRussell King #include <asm/sections.h> 321da177e4SLinus Torvalds #include <asm/setup.h> 339110f3e7SBen Dooks (Codethink) #include <asm/set_memory.h> 341e6b4811SKees Cook #include <asm/system_info.h> 351da177e4SLinus Torvalds #include <asm/tlb.h> 36db9ef1afSFenkart/Bostandzhyan #include <asm/fixmap.h> 37a8e53c15SJinbum Park #include <asm/ptdump.h> 381da177e4SLinus Torvalds 391da177e4SLinus Torvalds #include <asm/mach/arch.h> 401da177e4SLinus Torvalds #include <asm/mach/map.h> 411da177e4SLinus Torvalds 421b2e2b73SRussell King #include "mm.h" 431b2e2b73SRussell King 44b4b20ad8SRussell King #ifdef CONFIG_CPU_CP15_MMU 45b4b20ad8SRussell King unsigned long __init __clear_cr(unsigned long mask) 46b4b20ad8SRussell King { 47b4b20ad8SRussell King cr_alignment = cr_alignment & ~mask; 48b4b20ad8SRussell King return cr_alignment; 49b4b20ad8SRussell King } 50b4b20ad8SRussell King #endif 51b4b20ad8SRussell King 52b1ab95c6SFlorian Fainelli #ifdef CONFIG_BLK_DEV_INITRD 53012d1f4aSRussell King static int __init parse_tag_initrd(const struct tag *tag) 54012d1f4aSRussell King { 554ed89f22SRussell King pr_warn("ATAG_INITRD is deprecated; " 56012d1f4aSRussell King "please update your bootloader.\n"); 57012d1f4aSRussell King phys_initrd_start = __virt_to_phys(tag->u.initrd.start); 58012d1f4aSRussell King phys_initrd_size = tag->u.initrd.size; 59012d1f4aSRussell King return 0; 60012d1f4aSRussell King } 61012d1f4aSRussell King 62012d1f4aSRussell King __tagtable(ATAG_INITRD, parse_tag_initrd); 63012d1f4aSRussell King 64012d1f4aSRussell King static int __init parse_tag_initrd2(const struct tag *tag) 65012d1f4aSRussell King { 66012d1f4aSRussell King phys_initrd_start = tag->u.initrd.start; 67012d1f4aSRussell King phys_initrd_size = tag->u.initrd.size; 68012d1f4aSRussell King return 0; 69012d1f4aSRussell King } 70012d1f4aSRussell King 71012d1f4aSRussell King __tagtable(ATAG_INITRD2, parse_tag_initrd2); 72b1ab95c6SFlorian Fainelli #endif 731da177e4SLinus Torvalds 74f25b4b4cSRussell King static void __init find_limits(unsigned long *min, unsigned long *max_low, 75f25b4b4cSRussell King unsigned long *max_high) 76dde5828fSRussell King { 771c2f87c2SLaura Abbott *max_low = PFN_DOWN(memblock_get_current_limit()); 781c2f87c2SLaura Abbott *min = PFN_UP(memblock_start_of_DRAM()); 791c2f87c2SLaura Abbott *max_high = PFN_DOWN(memblock_end_of_DRAM()); 80dde5828fSRussell King } 81dde5828fSRussell King 82be20902bSRussell King #ifdef CONFIG_ZONE_DMA 8365032018SNicolas Pitre 84364230b9SRob Herring phys_addr_t arm_dma_zone_size __read_mostly; 8565032018SNicolas Pitre EXPORT_SYMBOL(arm_dma_zone_size); 8665032018SNicolas Pitre 87022ae537SRussell King /* 88022ae537SRussell King * The DMA mask corresponding to the maximum bus address allocatable 89022ae537SRussell King * using GFP_DMA. The default here places no restriction on DMA 90022ae537SRussell King * allocations. This must be the smallest DMA mask in the system, 91022ae537SRussell King * so a successful GFP_DMA allocation will always satisfy this. 92022ae537SRussell King */ 934986e5c7SMarek Szyprowski phys_addr_t arm_dma_limit; 944dcfa600SRussell King unsigned long arm_dma_pfn_limit; 95be20902bSRussell King #endif 96be20902bSRussell King 97ff69a4c8SRussell King void __init setup_dma_zone(const struct machine_desc *mdesc) 98c7909509SMarek Szyprowski { 99c7909509SMarek Szyprowski #ifdef CONFIG_ZONE_DMA 100c7909509SMarek Szyprowski if (mdesc->dma_zone_size) { 101c7909509SMarek Szyprowski arm_dma_zone_size = mdesc->dma_zone_size; 1026bcac805SRussell King arm_dma_limit = PHYS_OFFSET + arm_dma_zone_size - 1; 103c7909509SMarek Szyprowski } else 104c7909509SMarek Szyprowski arm_dma_limit = 0xffffffff; 1054dcfa600SRussell King arm_dma_pfn_limit = arm_dma_limit >> PAGE_SHIFT; 106c7909509SMarek Szyprowski #endif 107c7909509SMarek Szyprowski } 108c7909509SMarek Szyprowski 10984f452b1SSantosh Shilimkar static void __init zone_sizes_init(unsigned long min, unsigned long max_low, 110a2c54d2aSRussell King unsigned long max_high) 111b7a69ac3SRussell King { 112a32c1c61SMike Rapoport unsigned long max_zone_pfn[MAX_NR_ZONES] = { 0 }; 11390072059SRussell King 11465032018SNicolas Pitre #ifdef CONFIG_ZONE_DMA 115a32c1c61SMike Rapoport max_zone_pfn[ZONE_DMA] = min(arm_dma_pfn_limit, max_low); 116be20902bSRussell King #endif 117a32c1c61SMike Rapoport max_zone_pfn[ZONE_NORMAL] = max_low; 118a32c1c61SMike Rapoport #ifdef CONFIG_HIGHMEM 119a32c1c61SMike Rapoport max_zone_pfn[ZONE_HIGHMEM] = max_high; 120a32c1c61SMike Rapoport #endif 121a32c1c61SMike Rapoport free_area_init(max_zone_pfn); 12290072059SRussell King } 12390072059SRussell King 1247b7bf499SWill Deacon #ifdef CONFIG_HAVE_ARCH_PFN_VALID 125b7cfda9fSRussell King int pfn_valid(unsigned long pfn) 126b7cfda9fSRussell King { 1275b3efa4fSzhaoyang phys_addr_t addr = __pfn_to_phys(pfn); 1285b3efa4fSzhaoyang 1295b3efa4fSzhaoyang if (__phys_to_pfn(addr) != pfn) 1305b3efa4fSzhaoyang return 0; 1315b3efa4fSzhaoyang 132032be728SClemens Gruber return memblock_is_map_memory(addr); 133b7cfda9fSRussell King } 134b7cfda9fSRussell King EXPORT_SYMBOL(pfn_valid); 1357b7bf499SWill Deacon #endif 136657e12fdSRussell King 137716a3dc2SRussell King static bool arm_memblock_steal_permitted = true; 138716a3dc2SRussell King 139bc2827d0SRussell King phys_addr_t __init arm_memblock_steal(phys_addr_t size, phys_addr_t align) 140716a3dc2SRussell King { 141716a3dc2SRussell King phys_addr_t phys; 142716a3dc2SRussell King 143716a3dc2SRussell King BUG_ON(!arm_memblock_steal_permitted); 144716a3dc2SRussell King 145f240ec09SMike Rapoport phys = memblock_phys_alloc(size, align); 146ecc3e771SMike Rapoport if (!phys) 147ecc3e771SMike Rapoport panic("Failed to steal %pa bytes at %pS\n", 148ecc3e771SMike Rapoport &size, (void *)_RET_IP_); 149ecc3e771SMike Rapoport 150716a3dc2SRussell King memblock_free(phys, size); 151716a3dc2SRussell King memblock_remove(phys, size); 152716a3dc2SRussell King 153716a3dc2SRussell King return phys; 154716a3dc2SRussell King } 155716a3dc2SRussell King 15639286248SRussell King static void __init arm_initrd_init(void) 1572778f620SRussell King { 1582778f620SRussell King #ifdef CONFIG_BLK_DEV_INITRD 159cdcc5fa0SRussell King phys_addr_t start; 160cdcc5fa0SRussell King unsigned long size; 161cdcc5fa0SRussell King 1624c235cb9SBen Peddell initrd_start = initrd_end = 0; 16368b32f36SRussell King 16468b32f36SRussell King if (!phys_initrd_size) 16568b32f36SRussell King return; 16668b32f36SRussell King 167cdcc5fa0SRussell King /* 168cdcc5fa0SRussell King * Round the memory region to page boundaries as per free_initrd_mem() 169cdcc5fa0SRussell King * This allows us to detect whether the pages overlapping the initrd 170cdcc5fa0SRussell King * are in use, but more importantly, reserves the entire set of pages 171cdcc5fa0SRussell King * as we don't want these pages allocated for other purposes. 172cdcc5fa0SRussell King */ 173cdcc5fa0SRussell King start = round_down(phys_initrd_start, PAGE_SIZE); 174cdcc5fa0SRussell King size = phys_initrd_size + (phys_initrd_start - start); 175cdcc5fa0SRussell King size = round_up(size, PAGE_SIZE); 176cdcc5fa0SRussell King 177cdcc5fa0SRussell King if (!memblock_is_region_memory(start, size)) { 178de22cc6eSVitaly Andrianov pr_err("INITRD: 0x%08llx+0x%08lx is not a memory region - disabling initrd\n", 179cdcc5fa0SRussell King (u64)start, size); 18068b32f36SRussell King return; 1818f4b8c76SRussell King } 18268b32f36SRussell King 183cdcc5fa0SRussell King if (memblock_is_region_reserved(start, size)) { 184de22cc6eSVitaly Andrianov pr_err("INITRD: 0x%08llx+0x%08lx overlaps in-use memory region - disabling initrd\n", 185cdcc5fa0SRussell King (u64)start, size); 18668b32f36SRussell King return; 187b0a2679dSRussell King } 18868b32f36SRussell King 189cdcc5fa0SRussell King memblock_reserve(start, size); 1902778f620SRussell King 1912778f620SRussell King /* Now convert initrd to virtual addresses */ 1922778f620SRussell King initrd_start = __phys_to_virt(phys_initrd_start); 1932778f620SRussell King initrd_end = initrd_start + phys_initrd_size; 1942778f620SRussell King #endif 19539286248SRussell King } 19639286248SRussell King 1975f41f919SMarek Szyprowski #ifdef CONFIG_CPU_ICACHE_MISMATCH_WORKAROUND 1985f41f919SMarek Szyprowski void check_cpu_icache_size(int cpuid) 1995f41f919SMarek Szyprowski { 2005f41f919SMarek Szyprowski u32 size, ctr; 2015f41f919SMarek Szyprowski 2025f41f919SMarek Szyprowski asm("mrc p15, 0, %0, c0, c0, 1" : "=r" (ctr)); 2035f41f919SMarek Szyprowski 2045f41f919SMarek Szyprowski size = 1 << ((ctr & 0xf) + 2); 2055f41f919SMarek Szyprowski if (cpuid != 0 && icache_size != size) 2065f41f919SMarek Szyprowski pr_info("CPU%u: detected I-Cache line size mismatch, workaround enabled\n", 2075f41f919SMarek Szyprowski cpuid); 2085f41f919SMarek Szyprowski if (icache_size > size) 2095f41f919SMarek Szyprowski icache_size = size; 2105f41f919SMarek Szyprowski } 2115f41f919SMarek Szyprowski #endif 2125f41f919SMarek Szyprowski 21339286248SRussell King void __init arm_memblock_init(const struct machine_desc *mdesc) 21439286248SRussell King { 21539286248SRussell King /* Register the kernel text, kernel data and initrd with memblock. */ 21639286248SRussell King memblock_reserve(__pa(KERNEL_START), KERNEL_END - KERNEL_START); 21739286248SRussell King 21839286248SRussell King arm_initrd_init(); 2192778f620SRussell King 2202778f620SRussell King arm_mm_memblock_reserve(); 2212778f620SRussell King 2228d717a52SRussell King /* reserve any platform specific memblock areas */ 2238d717a52SRussell King if (mdesc->reserve) 2248d717a52SRussell King mdesc->reserve(); 2258d717a52SRussell King 226bcedb5f9SMarek Szyprowski early_init_fdt_scan_reserved_mem(); 227bcedb5f9SMarek Szyprowski 22899a468d7SGeorge G. Davis /* reserve memory for DMA contiguous allocations */ 22995b0e655SMarek Szyprowski dma_contiguous_reserve(arm_dma_limit); 230c7909509SMarek Szyprowski 231716a3dc2SRussell King arm_memblock_steal_permitted = false; 2322778f620SRussell King memblock_dump_all(); 2332778f620SRussell King } 2342778f620SRussell King 2358d717a52SRussell King void __init bootmem_init(void) 23690072059SRussell King { 2378e58caefSGrygorii Strashko memblock_allow_resize(); 238dde5828fSRussell King 239071d184aSDoug Berger find_limits(&min_low_pfn, &max_low_pfn, &max_pfn); 240dde5828fSRussell King 241071d184aSDoug Berger early_memtest((phys_addr_t)min_low_pfn << PAGE_SHIFT, 242071d184aSDoug Berger (phys_addr_t)max_low_pfn << PAGE_SHIFT); 243d30eae47SVladimir Murzin 244b7a69ac3SRussell King /* 245c89ab04fSMike Rapoport * sparse_init() tries to allocate memory from memblock, so must be 246c89ab04fSMike Rapoport * done after the fixed reservations 247b7a69ac3SRussell King */ 248b7a69ac3SRussell King sparse_init(); 249b7a69ac3SRussell King 250b7a69ac3SRussell King /* 251a32c1c61SMike Rapoport * Now free the memory - free_area_init needs 252b7a69ac3SRussell King * the sparse mem_map arrays initialized by sparse_init() 253b7a69ac3SRussell King * for memmap_init_zone(), otherwise all PFNs are invalid. 254b7a69ac3SRussell King */ 255071d184aSDoug Berger zone_sizes_init(min_low_pfn, max_low_pfn, max_pfn); 25690072059SRussell King } 25790072059SRussell King 25854d52573SStephen Boyd /* 25954d52573SStephen Boyd * Poison init memory with an undefined instruction (ARM) or a branch to an 26054d52573SStephen Boyd * undefined instruction (Thumb). 26154d52573SStephen Boyd */ 26254d52573SStephen Boyd static inline void poison_init_mem(void *s, size_t count) 26354d52573SStephen Boyd { 26454d52573SStephen Boyd u32 *p = (u32 *)s; 265bf912d99SJamie Iles for (; count != 0; count -= 4) 26654d52573SStephen Boyd *p++ = 0xe7fddef0; 26754d52573SStephen Boyd } 26854d52573SStephen Boyd 269d0e775afSRussell King static void __init free_highpages(void) 270d0e775afSRussell King { 271d0e775afSRussell King #ifdef CONFIG_HIGHMEM 27226ba47b1SSantosh Shilimkar unsigned long max_low = max_low_pfn; 273cddb5ddfSMike Rapoport phys_addr_t range_start, range_end; 274cddb5ddfSMike Rapoport u64 i; 275d0e775afSRussell King 276d0e775afSRussell King /* set highmem page free */ 277cddb5ddfSMike Rapoport for_each_free_mem_range(i, NUMA_NO_NODE, MEMBLOCK_NONE, 278cddb5ddfSMike Rapoport &range_start, &range_end, NULL) { 279b9bc3670SArd Biesheuvel unsigned long start = PFN_UP(range_start); 280b9bc3670SArd Biesheuvel unsigned long end = PFN_DOWN(range_end); 281df4f14c7SRussell King 282df4f14c7SRussell King /* Ignore complete lowmem entries */ 283df4f14c7SRussell King if (end <= max_low) 284df4f14c7SRussell King continue; 285df4f14c7SRussell King 286df4f14c7SRussell King /* Truncate partial highmem entries */ 287df4f14c7SRussell King if (start < max_low) 288df4f14c7SRussell King start = max_low; 289df4f14c7SRussell King 290cddb5ddfSMike Rapoport for (; start < end; start++) 291cddb5ddfSMike Rapoport free_highmem_page(pfn_to_page(start)); 292d0e775afSRussell King } 293d0e775afSRussell King #endif 294d0e775afSRussell King } 295d0e775afSRussell King 2961da177e4SLinus Torvalds /* 2971da177e4SLinus Torvalds * mem_init() marks the free areas in the mem_map and tells us how much 2981da177e4SLinus Torvalds * memory is free. This is done after various parts of the system have 2991da177e4SLinus Torvalds * claimed their memory after the kernel image. 3001da177e4SLinus Torvalds */ 3011da177e4SLinus Torvalds void __init mem_init(void) 3021da177e4SLinus Torvalds { 303ad3c7b18SChristoph Hellwig #ifdef CONFIG_ARM_LPAE 304*fcf04489SFlorian Fainelli if (swiotlb_force == SWIOTLB_FORCE || 305*fcf04489SFlorian Fainelli max_pfn > arm_dma_pfn_limit) 306ad3c7b18SChristoph Hellwig swiotlb_init(1); 307*fcf04489SFlorian Fainelli else 308*fcf04489SFlorian Fainelli swiotlb_force = SWIOTLB_NO_FORCE; 309ad3c7b18SChristoph Hellwig #endif 310ad3c7b18SChristoph Hellwig 311b3ba41f2SSantosh Shilimkar set_max_mapnr(pfn_to_page(max_pfn) - mem_map); 3121da177e4SLinus Torvalds 3131da177e4SLinus Torvalds /* this will put all unused low memory onto the freelists */ 314c6ffc5caSMike Rapoport memblock_free_all(); 3151da177e4SLinus Torvalds 3161da177e4SLinus Torvalds #ifdef CONFIG_SA1111 3171da177e4SLinus Torvalds /* now that our DMA memory is actually so designated, we can free it */ 318bfd65dd9SLinus Torvalds free_reserved_area(__va(PHYS_OFFSET), swapper_pg_dir, -1, NULL); 3191da177e4SLinus Torvalds #endif 3201da177e4SLinus Torvalds 321d0e775afSRussell King free_highpages(); 3223835f6cbSNicolas Pitre 3232450c973SJiang Liu mem_init_print_info(NULL); 3241da177e4SLinus Torvalds 325a1839272SFenkart/Bostandzhyan /* 326a1839272SFenkart/Bostandzhyan * Check boundaries twice: Some fundamental inconsistencies can 327a1839272SFenkart/Bostandzhyan * be detected at build time already. 328a1839272SFenkart/Bostandzhyan */ 329a1839272SFenkart/Bostandzhyan #ifdef CONFIG_MMU 330a1839272SFenkart/Bostandzhyan BUILD_BUG_ON(TASK_SIZE > MODULES_VADDR); 331a1839272SFenkart/Bostandzhyan BUG_ON(TASK_SIZE > MODULES_VADDR); 332a1839272SFenkart/Bostandzhyan #endif 333a1839272SFenkart/Bostandzhyan 334a1839272SFenkart/Bostandzhyan #ifdef CONFIG_HIGHMEM 335a1839272SFenkart/Bostandzhyan BUILD_BUG_ON(PKMAP_BASE + LAST_PKMAP * PAGE_SIZE > PAGE_OFFSET); 336a1839272SFenkart/Bostandzhyan BUG_ON(PKMAP_BASE + LAST_PKMAP * PAGE_SIZE > PAGE_OFFSET); 337a1839272SFenkart/Bostandzhyan #endif 3381da177e4SLinus Torvalds } 3391da177e4SLinus Torvalds 3400f5bf6d0SLaura Abbott #ifdef CONFIG_STRICT_KERNEL_RWX 3411e6b4811SKees Cook struct section_perm { 34225362dc4SKees Cook const char *name; 3431e6b4811SKees Cook unsigned long start; 3441e6b4811SKees Cook unsigned long end; 3451e6b4811SKees Cook pmdval_t mask; 3461e6b4811SKees Cook pmdval_t prot; 34780d6b0c2SKees Cook pmdval_t clear; 3481e6b4811SKees Cook }; 3491e6b4811SKees Cook 35064ac2e74SKees Cook /* First section-aligned location at or after __start_rodata. */ 35164ac2e74SKees Cook extern char __start_rodata_section_aligned[]; 35264ac2e74SKees Cook 35380d6b0c2SKees Cook static struct section_perm nx_perms[] = { 3541e6b4811SKees Cook /* Make pages tables, etc before _stext RW (set NX). */ 3551e6b4811SKees Cook { 35625362dc4SKees Cook .name = "pre-text NX", 3571e6b4811SKees Cook .start = PAGE_OFFSET, 3581e6b4811SKees Cook .end = (unsigned long)_stext, 3591e6b4811SKees Cook .mask = ~PMD_SECT_XN, 3601e6b4811SKees Cook .prot = PMD_SECT_XN, 3611e6b4811SKees Cook }, 3621e6b4811SKees Cook /* Make init RW (set NX). */ 3631e6b4811SKees Cook { 36425362dc4SKees Cook .name = "init NX", 3651e6b4811SKees Cook .start = (unsigned long)__init_begin, 3661e6b4811SKees Cook .end = (unsigned long)_sdata, 3671e6b4811SKees Cook .mask = ~PMD_SECT_XN, 3681e6b4811SKees Cook .prot = PMD_SECT_XN, 3691e6b4811SKees Cook }, 37080d6b0c2SKees Cook /* Make rodata NX (set RO in ro_perms below). */ 37180d6b0c2SKees Cook { 37225362dc4SKees Cook .name = "rodata NX", 37364ac2e74SKees Cook .start = (unsigned long)__start_rodata_section_aligned, 37480d6b0c2SKees Cook .end = (unsigned long)__init_begin, 37580d6b0c2SKees Cook .mask = ~PMD_SECT_XN, 37680d6b0c2SKees Cook .prot = PMD_SECT_XN, 37780d6b0c2SKees Cook }, 3781e6b4811SKees Cook }; 3791e6b4811SKees Cook 38080d6b0c2SKees Cook static struct section_perm ro_perms[] = { 38180d6b0c2SKees Cook /* Make kernel code and rodata RX (set RO). */ 38280d6b0c2SKees Cook { 38325362dc4SKees Cook .name = "text/rodata RO", 38480d6b0c2SKees Cook .start = (unsigned long)_stext, 38580d6b0c2SKees Cook .end = (unsigned long)__init_begin, 38680d6b0c2SKees Cook #ifdef CONFIG_ARM_LPAE 387400eeffaSPhilip Derrin .mask = ~(L_PMD_SECT_RDONLY | PMD_SECT_AP2), 388400eeffaSPhilip Derrin .prot = L_PMD_SECT_RDONLY | PMD_SECT_AP2, 38980d6b0c2SKees Cook #else 39080d6b0c2SKees Cook .mask = ~(PMD_SECT_APX | PMD_SECT_AP_WRITE), 39180d6b0c2SKees Cook .prot = PMD_SECT_APX | PMD_SECT_AP_WRITE, 39280d6b0c2SKees Cook .clear = PMD_SECT_AP_WRITE, 39380d6b0c2SKees Cook #endif 39480d6b0c2SKees Cook }, 39580d6b0c2SKees Cook }; 39680d6b0c2SKees Cook 3971e6b4811SKees Cook /* 3981e6b4811SKees Cook * Updates section permissions only for the current mm (sections are 3991e6b4811SKees Cook * copied into each mm). During startup, this is the init_mm. Is only 4001e6b4811SKees Cook * safe to be called with preemption disabled, as under stop_machine(). 4011e6b4811SKees Cook */ 4021e6b4811SKees Cook static inline void section_update(unsigned long addr, pmdval_t mask, 40308925c2fSLaura Abbott pmdval_t prot, struct mm_struct *mm) 4041e6b4811SKees Cook { 4051e6b4811SKees Cook pmd_t *pmd; 4061e6b4811SKees Cook 40784e6ffb2SMike Rapoport pmd = pmd_offset(pud_offset(p4d_offset(pgd_offset(mm, addr), addr), addr), addr); 4081e6b4811SKees Cook 4091e6b4811SKees Cook #ifdef CONFIG_ARM_LPAE 4101e6b4811SKees Cook pmd[0] = __pmd((pmd_val(pmd[0]) & mask) | prot); 4111e6b4811SKees Cook #else 4121e6b4811SKees Cook if (addr & SECTION_SIZE) 4131e6b4811SKees Cook pmd[1] = __pmd((pmd_val(pmd[1]) & mask) | prot); 4141e6b4811SKees Cook else 4151e6b4811SKees Cook pmd[0] = __pmd((pmd_val(pmd[0]) & mask) | prot); 4161e6b4811SKees Cook #endif 4171e6b4811SKees Cook flush_pmd_entry(pmd); 4181e6b4811SKees Cook local_flush_tlb_kernel_range(addr, addr + SECTION_SIZE); 4191e6b4811SKees Cook } 4201e6b4811SKees Cook 4211e6b4811SKees Cook /* Make sure extended page tables are in use. */ 4221e6b4811SKees Cook static inline bool arch_has_strict_perms(void) 4231e6b4811SKees Cook { 4241e6b4811SKees Cook if (cpu_architecture() < CPU_ARCH_ARMv6) 4251e6b4811SKees Cook return false; 4261e6b4811SKees Cook 4271e6b4811SKees Cook return !!(get_cr() & CR_XP); 4281e6b4811SKees Cook } 4291e6b4811SKees Cook 430ea5379beSBen Dooks (Codethink) static void set_section_perms(struct section_perm *perms, int n, bool set, 43108925c2fSLaura Abbott struct mm_struct *mm) 43208925c2fSLaura Abbott { 43308925c2fSLaura Abbott size_t i; 43408925c2fSLaura Abbott unsigned long addr; 43508925c2fSLaura Abbott 43608925c2fSLaura Abbott if (!arch_has_strict_perms()) 43708925c2fSLaura Abbott return; 43808925c2fSLaura Abbott 43908925c2fSLaura Abbott for (i = 0; i < n; i++) { 44008925c2fSLaura Abbott if (!IS_ALIGNED(perms[i].start, SECTION_SIZE) || 44108925c2fSLaura Abbott !IS_ALIGNED(perms[i].end, SECTION_SIZE)) { 44225362dc4SKees Cook pr_err("BUG: %s section %lx-%lx not aligned to %lx\n", 44325362dc4SKees Cook perms[i].name, perms[i].start, perms[i].end, 44408925c2fSLaura Abbott SECTION_SIZE); 44508925c2fSLaura Abbott continue; 4461e6b4811SKees Cook } 4471e6b4811SKees Cook 44808925c2fSLaura Abbott for (addr = perms[i].start; 44908925c2fSLaura Abbott addr < perms[i].end; 45008925c2fSLaura Abbott addr += SECTION_SIZE) 45108925c2fSLaura Abbott section_update(addr, perms[i].mask, 45208925c2fSLaura Abbott set ? perms[i].prot : perms[i].clear, mm); 45308925c2fSLaura Abbott } 45408925c2fSLaura Abbott 45508925c2fSLaura Abbott } 45608925c2fSLaura Abbott 45711ce4b33SGrygorii Strashko /** 45811ce4b33SGrygorii Strashko * update_sections_early intended to be called only through stop_machine 45911ce4b33SGrygorii Strashko * framework and executed by only one CPU while all other CPUs will spin and 46011ce4b33SGrygorii Strashko * wait, so no locking is required in this function. 46111ce4b33SGrygorii Strashko */ 46208925c2fSLaura Abbott static void update_sections_early(struct section_perm perms[], int n) 4631e6b4811SKees Cook { 46408925c2fSLaura Abbott struct task_struct *t, *s; 46508925c2fSLaura Abbott 46608925c2fSLaura Abbott for_each_process(t) { 46708925c2fSLaura Abbott if (t->flags & PF_KTHREAD) 46808925c2fSLaura Abbott continue; 46908925c2fSLaura Abbott for_each_thread(t, s) 470c51bc12dSDoug Berger if (s->mm) 47108925c2fSLaura Abbott set_section_perms(perms, n, true, s->mm); 47208925c2fSLaura Abbott } 47308925c2fSLaura Abbott set_section_perms(perms, n, true, current->active_mm); 47408925c2fSLaura Abbott set_section_perms(perms, n, true, &init_mm); 47508925c2fSLaura Abbott } 47608925c2fSLaura Abbott 47711ce4b33SGrygorii Strashko static int __fix_kernmem_perms(void *unused) 47808925c2fSLaura Abbott { 47908925c2fSLaura Abbott update_sections_early(nx_perms, ARRAY_SIZE(nx_perms)); 48008925c2fSLaura Abbott return 0; 48108925c2fSLaura Abbott } 48208925c2fSLaura Abbott 48311ce4b33SGrygorii Strashko static void fix_kernmem_perms(void) 48408925c2fSLaura Abbott { 48508925c2fSLaura Abbott stop_machine(__fix_kernmem_perms, NULL, NULL); 4861e6b4811SKees Cook } 48780d6b0c2SKees Cook 48811ce4b33SGrygorii Strashko static int __mark_rodata_ro(void *unused) 48908925c2fSLaura Abbott { 49008925c2fSLaura Abbott update_sections_early(ro_perms, ARRAY_SIZE(ro_perms)); 49108925c2fSLaura Abbott return 0; 49208925c2fSLaura Abbott } 49308925c2fSLaura Abbott 494b4c7e2bdSSteven Rostedt (VMware) static int kernel_set_to_readonly __read_mostly; 495b4c7e2bdSSteven Rostedt (VMware) 49680d6b0c2SKees Cook void mark_rodata_ro(void) 49780d6b0c2SKees Cook { 498b4c7e2bdSSteven Rostedt (VMware) kernel_set_to_readonly = 1; 49908925c2fSLaura Abbott stop_machine(__mark_rodata_ro, NULL, NULL); 500a8e53c15SJinbum Park debug_checkwx(); 50180d6b0c2SKees Cook } 50280d6b0c2SKees Cook 50380d6b0c2SKees Cook void set_kernel_text_rw(void) 50480d6b0c2SKees Cook { 505b4c7e2bdSSteven Rostedt (VMware) if (!kernel_set_to_readonly) 506b4c7e2bdSSteven Rostedt (VMware) return; 507b4c7e2bdSSteven Rostedt (VMware) 50808925c2fSLaura Abbott set_section_perms(ro_perms, ARRAY_SIZE(ro_perms), false, 50908925c2fSLaura Abbott current->active_mm); 51080d6b0c2SKees Cook } 51180d6b0c2SKees Cook 51280d6b0c2SKees Cook void set_kernel_text_ro(void) 51380d6b0c2SKees Cook { 514b4c7e2bdSSteven Rostedt (VMware) if (!kernel_set_to_readonly) 515b4c7e2bdSSteven Rostedt (VMware) return; 516b4c7e2bdSSteven Rostedt (VMware) 51708925c2fSLaura Abbott set_section_perms(ro_perms, ARRAY_SIZE(ro_perms), true, 51808925c2fSLaura Abbott current->active_mm); 51980d6b0c2SKees Cook } 52080d6b0c2SKees Cook 5211e6b4811SKees Cook #else 5221e6b4811SKees Cook static inline void fix_kernmem_perms(void) { } 5230f5bf6d0SLaura Abbott #endif /* CONFIG_STRICT_KERNEL_RWX */ 5241e6b4811SKees Cook 5251e6b4811SKees Cook void free_initmem(void) 5261e6b4811SKees Cook { 5271e6b4811SKees Cook fix_kernmem_perms(); 528bc581770SLinus Walleij 52954d52573SStephen Boyd poison_init_mem(__init_begin, __init_end - __init_begin); 5306db015e4SNicolas Pitre if (!machine_is_integrator() && !machine_is_cintegrator()) 531dbe67df4SJiang Liu free_initmem_default(-1); 5321da177e4SLinus Torvalds } 5331da177e4SLinus Torvalds 5341da177e4SLinus Torvalds #ifdef CONFIG_BLK_DEV_INITRD 5351da177e4SLinus Torvalds void free_initrd_mem(unsigned long start, unsigned long end) 5361da177e4SLinus Torvalds { 537421520baSYalin Wang if (start == initrd_start) 538421520baSYalin Wang start = round_down(start, PAGE_SIZE); 539421520baSYalin Wang if (end == initrd_end) 540421520baSYalin Wang end = round_up(end, PAGE_SIZE); 541421520baSYalin Wang 54254d52573SStephen Boyd poison_init_mem((void *)start, PAGE_ALIGN(end) - start); 543dbe67df4SJiang Liu free_reserved_area((void *)start, (void *)end, -1, "initrd"); 5441da177e4SLinus Torvalds } 5451da177e4SLinus Torvalds #endif 546