1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * linux/arch/arm/mm/init.c 4 * 5 * Copyright (C) 1995-2005 Russell King 6 */ 7 #include <linux/kernel.h> 8 #include <linux/errno.h> 9 #include <linux/swap.h> 10 #include <linux/init.h> 11 #include <linux/mman.h> 12 #include <linux/sched/signal.h> 13 #include <linux/sched/task.h> 14 #include <linux/export.h> 15 #include <linux/nodemask.h> 16 #include <linux/initrd.h> 17 #include <linux/of_fdt.h> 18 #include <linux/highmem.h> 19 #include <linux/gfp.h> 20 #include <linux/memblock.h> 21 #include <linux/dma-contiguous.h> 22 #include <linux/sizes.h> 23 #include <linux/stop_machine.h> 24 #include <linux/swiotlb.h> 25 26 #include <asm/cp15.h> 27 #include <asm/mach-types.h> 28 #include <asm/memblock.h> 29 #include <asm/memory.h> 30 #include <asm/prom.h> 31 #include <asm/sections.h> 32 #include <asm/setup.h> 33 #include <asm/set_memory.h> 34 #include <asm/system_info.h> 35 #include <asm/tlb.h> 36 #include <asm/fixmap.h> 37 #include <asm/ptdump.h> 38 39 #include <asm/mach/arch.h> 40 #include <asm/mach/map.h> 41 42 #include "mm.h" 43 44 #ifdef CONFIG_CPU_CP15_MMU 45 unsigned long __init __clear_cr(unsigned long mask) 46 { 47 cr_alignment = cr_alignment & ~mask; 48 return cr_alignment; 49 } 50 #endif 51 52 #ifdef CONFIG_BLK_DEV_INITRD 53 static int __init parse_tag_initrd(const struct tag *tag) 54 { 55 pr_warn("ATAG_INITRD is deprecated; " 56 "please update your bootloader.\n"); 57 phys_initrd_start = __virt_to_phys(tag->u.initrd.start); 58 phys_initrd_size = tag->u.initrd.size; 59 return 0; 60 } 61 62 __tagtable(ATAG_INITRD, parse_tag_initrd); 63 64 static int __init parse_tag_initrd2(const struct tag *tag) 65 { 66 phys_initrd_start = tag->u.initrd.start; 67 phys_initrd_size = tag->u.initrd.size; 68 return 0; 69 } 70 71 __tagtable(ATAG_INITRD2, parse_tag_initrd2); 72 #endif 73 74 static void __init find_limits(unsigned long *min, unsigned long *max_low, 75 unsigned long *max_high) 76 { 77 *max_low = PFN_DOWN(memblock_get_current_limit()); 78 *min = PFN_UP(memblock_start_of_DRAM()); 79 *max_high = PFN_DOWN(memblock_end_of_DRAM()); 80 } 81 82 #ifdef CONFIG_ZONE_DMA 83 84 phys_addr_t arm_dma_zone_size __read_mostly; 85 EXPORT_SYMBOL(arm_dma_zone_size); 86 87 /* 88 * The DMA mask corresponding to the maximum bus address allocatable 89 * using GFP_DMA. The default here places no restriction on DMA 90 * allocations. This must be the smallest DMA mask in the system, 91 * so a successful GFP_DMA allocation will always satisfy this. 92 */ 93 phys_addr_t arm_dma_limit; 94 unsigned long arm_dma_pfn_limit; 95 #endif 96 97 void __init setup_dma_zone(const struct machine_desc *mdesc) 98 { 99 #ifdef CONFIG_ZONE_DMA 100 if (mdesc->dma_zone_size) { 101 arm_dma_zone_size = mdesc->dma_zone_size; 102 arm_dma_limit = PHYS_OFFSET + arm_dma_zone_size - 1; 103 } else 104 arm_dma_limit = 0xffffffff; 105 arm_dma_pfn_limit = arm_dma_limit >> PAGE_SHIFT; 106 #endif 107 } 108 109 static void __init zone_sizes_init(unsigned long min, unsigned long max_low, 110 unsigned long max_high) 111 { 112 unsigned long max_zone_pfn[MAX_NR_ZONES] = { 0 }; 113 114 #ifdef CONFIG_ZONE_DMA 115 max_zone_pfn[ZONE_DMA] = min(arm_dma_pfn_limit, max_low); 116 #endif 117 max_zone_pfn[ZONE_NORMAL] = max_low; 118 #ifdef CONFIG_HIGHMEM 119 max_zone_pfn[ZONE_HIGHMEM] = max_high; 120 #endif 121 free_area_init(max_zone_pfn); 122 } 123 124 #ifdef CONFIG_HAVE_ARCH_PFN_VALID 125 int pfn_valid(unsigned long pfn) 126 { 127 phys_addr_t addr = __pfn_to_phys(pfn); 128 129 if (__phys_to_pfn(addr) != pfn) 130 return 0; 131 132 return memblock_is_map_memory(addr); 133 } 134 EXPORT_SYMBOL(pfn_valid); 135 #endif 136 137 static bool arm_memblock_steal_permitted = true; 138 139 phys_addr_t __init arm_memblock_steal(phys_addr_t size, phys_addr_t align) 140 { 141 phys_addr_t phys; 142 143 BUG_ON(!arm_memblock_steal_permitted); 144 145 phys = memblock_phys_alloc(size, align); 146 if (!phys) 147 panic("Failed to steal %pa bytes at %pS\n", 148 &size, (void *)_RET_IP_); 149 150 memblock_free(phys, size); 151 memblock_remove(phys, size); 152 153 return phys; 154 } 155 156 static void __init arm_initrd_init(void) 157 { 158 #ifdef CONFIG_BLK_DEV_INITRD 159 phys_addr_t start; 160 unsigned long size; 161 162 initrd_start = initrd_end = 0; 163 164 if (!phys_initrd_size) 165 return; 166 167 /* 168 * Round the memory region to page boundaries as per free_initrd_mem() 169 * This allows us to detect whether the pages overlapping the initrd 170 * are in use, but more importantly, reserves the entire set of pages 171 * as we don't want these pages allocated for other purposes. 172 */ 173 start = round_down(phys_initrd_start, PAGE_SIZE); 174 size = phys_initrd_size + (phys_initrd_start - start); 175 size = round_up(size, PAGE_SIZE); 176 177 if (!memblock_is_region_memory(start, size)) { 178 pr_err("INITRD: 0x%08llx+0x%08lx is not a memory region - disabling initrd\n", 179 (u64)start, size); 180 return; 181 } 182 183 if (memblock_is_region_reserved(start, size)) { 184 pr_err("INITRD: 0x%08llx+0x%08lx overlaps in-use memory region - disabling initrd\n", 185 (u64)start, size); 186 return; 187 } 188 189 memblock_reserve(start, size); 190 191 /* Now convert initrd to virtual addresses */ 192 initrd_start = __phys_to_virt(phys_initrd_start); 193 initrd_end = initrd_start + phys_initrd_size; 194 #endif 195 } 196 197 #ifdef CONFIG_CPU_ICACHE_MISMATCH_WORKAROUND 198 void check_cpu_icache_size(int cpuid) 199 { 200 u32 size, ctr; 201 202 asm("mrc p15, 0, %0, c0, c0, 1" : "=r" (ctr)); 203 204 size = 1 << ((ctr & 0xf) + 2); 205 if (cpuid != 0 && icache_size != size) 206 pr_info("CPU%u: detected I-Cache line size mismatch, workaround enabled\n", 207 cpuid); 208 if (icache_size > size) 209 icache_size = size; 210 } 211 #endif 212 213 void __init arm_memblock_init(const struct machine_desc *mdesc) 214 { 215 /* Register the kernel text, kernel data and initrd with memblock. */ 216 memblock_reserve(__pa(KERNEL_START), KERNEL_END - KERNEL_START); 217 218 arm_initrd_init(); 219 220 arm_mm_memblock_reserve(); 221 222 /* reserve any platform specific memblock areas */ 223 if (mdesc->reserve) 224 mdesc->reserve(); 225 226 early_init_fdt_reserve_self(); 227 early_init_fdt_scan_reserved_mem(); 228 229 /* reserve memory for DMA contiguous allocations */ 230 dma_contiguous_reserve(arm_dma_limit); 231 232 arm_memblock_steal_permitted = false; 233 memblock_dump_all(); 234 } 235 236 void __init bootmem_init(void) 237 { 238 memblock_allow_resize(); 239 240 find_limits(&min_low_pfn, &max_low_pfn, &max_pfn); 241 242 early_memtest((phys_addr_t)min_low_pfn << PAGE_SHIFT, 243 (phys_addr_t)max_low_pfn << PAGE_SHIFT); 244 245 /* 246 * sparse_init() tries to allocate memory from memblock, so must be 247 * done after the fixed reservations 248 */ 249 sparse_init(); 250 251 /* 252 * Now free the memory - free_area_init needs 253 * the sparse mem_map arrays initialized by sparse_init() 254 * for memmap_init_zone(), otherwise all PFNs are invalid. 255 */ 256 zone_sizes_init(min_low_pfn, max_low_pfn, max_pfn); 257 } 258 259 /* 260 * Poison init memory with an undefined instruction (ARM) or a branch to an 261 * undefined instruction (Thumb). 262 */ 263 static inline void poison_init_mem(void *s, size_t count) 264 { 265 u32 *p = (u32 *)s; 266 for (; count != 0; count -= 4) 267 *p++ = 0xe7fddef0; 268 } 269 270 static inline void __init 271 free_memmap(unsigned long start_pfn, unsigned long end_pfn) 272 { 273 struct page *start_pg, *end_pg; 274 phys_addr_t pg, pgend; 275 276 /* 277 * Convert start_pfn/end_pfn to a struct page pointer. 278 */ 279 start_pg = pfn_to_page(start_pfn - 1) + 1; 280 end_pg = pfn_to_page(end_pfn - 1) + 1; 281 282 /* 283 * Convert to physical addresses, and 284 * round start upwards and end downwards. 285 */ 286 pg = PAGE_ALIGN(__pa(start_pg)); 287 pgend = __pa(end_pg) & PAGE_MASK; 288 289 /* 290 * If there are free pages between these, 291 * free the section of the memmap array. 292 */ 293 if (pg < pgend) 294 memblock_free_early(pg, pgend - pg); 295 } 296 297 /* 298 * The mem_map array can get very big. Free the unused area of the memory map. 299 */ 300 static void __init free_unused_memmap(void) 301 { 302 unsigned long start, prev_end = 0; 303 struct memblock_region *reg; 304 305 /* 306 * This relies on each bank being in address order. 307 * The banks are sorted previously in bootmem_init(). 308 */ 309 for_each_memblock(memory, reg) { 310 start = memblock_region_memory_base_pfn(reg); 311 312 #ifdef CONFIG_SPARSEMEM 313 /* 314 * Take care not to free memmap entries that don't exist 315 * due to SPARSEMEM sections which aren't present. 316 */ 317 start = min(start, 318 ALIGN(prev_end, PAGES_PER_SECTION)); 319 #else 320 /* 321 * Align down here since the VM subsystem insists that the 322 * memmap entries are valid from the bank start aligned to 323 * MAX_ORDER_NR_PAGES. 324 */ 325 start = round_down(start, MAX_ORDER_NR_PAGES); 326 #endif 327 /* 328 * If we had a previous bank, and there is a space 329 * between the current bank and the previous, free it. 330 */ 331 if (prev_end && prev_end < start) 332 free_memmap(prev_end, start); 333 334 /* 335 * Align up here since the VM subsystem insists that the 336 * memmap entries are valid from the bank end aligned to 337 * MAX_ORDER_NR_PAGES. 338 */ 339 prev_end = ALIGN(memblock_region_memory_end_pfn(reg), 340 MAX_ORDER_NR_PAGES); 341 } 342 343 #ifdef CONFIG_SPARSEMEM 344 if (!IS_ALIGNED(prev_end, PAGES_PER_SECTION)) 345 free_memmap(prev_end, 346 ALIGN(prev_end, PAGES_PER_SECTION)); 347 #endif 348 } 349 350 static void __init free_highpages(void) 351 { 352 #ifdef CONFIG_HIGHMEM 353 unsigned long max_low = max_low_pfn; 354 phys_addr_t range_start, range_end; 355 u64 i; 356 357 /* set highmem page free */ 358 for_each_free_mem_range(i, NUMA_NO_NODE, MEMBLOCK_NONE, 359 &range_start, &range_end, NULL) { 360 unsigned long start = PHYS_PFN(range_start); 361 unsigned long end = PHYS_PFN(range_end); 362 363 /* Ignore complete lowmem entries */ 364 if (end <= max_low) 365 continue; 366 367 /* Truncate partial highmem entries */ 368 if (start < max_low) 369 start = max_low; 370 371 for (; start < end; start++) 372 free_highmem_page(pfn_to_page(start)); 373 } 374 #endif 375 } 376 377 /* 378 * mem_init() marks the free areas in the mem_map and tells us how much 379 * memory is free. This is done after various parts of the system have 380 * claimed their memory after the kernel image. 381 */ 382 void __init mem_init(void) 383 { 384 #ifdef CONFIG_ARM_LPAE 385 swiotlb_init(1); 386 #endif 387 388 set_max_mapnr(pfn_to_page(max_pfn) - mem_map); 389 390 /* this will put all unused low memory onto the freelists */ 391 free_unused_memmap(); 392 memblock_free_all(); 393 394 #ifdef CONFIG_SA1111 395 /* now that our DMA memory is actually so designated, we can free it */ 396 free_reserved_area(__va(PHYS_OFFSET), swapper_pg_dir, -1, NULL); 397 #endif 398 399 free_highpages(); 400 401 mem_init_print_info(NULL); 402 403 /* 404 * Check boundaries twice: Some fundamental inconsistencies can 405 * be detected at build time already. 406 */ 407 #ifdef CONFIG_MMU 408 BUILD_BUG_ON(TASK_SIZE > MODULES_VADDR); 409 BUG_ON(TASK_SIZE > MODULES_VADDR); 410 #endif 411 412 #ifdef CONFIG_HIGHMEM 413 BUILD_BUG_ON(PKMAP_BASE + LAST_PKMAP * PAGE_SIZE > PAGE_OFFSET); 414 BUG_ON(PKMAP_BASE + LAST_PKMAP * PAGE_SIZE > PAGE_OFFSET); 415 #endif 416 } 417 418 #ifdef CONFIG_STRICT_KERNEL_RWX 419 struct section_perm { 420 const char *name; 421 unsigned long start; 422 unsigned long end; 423 pmdval_t mask; 424 pmdval_t prot; 425 pmdval_t clear; 426 }; 427 428 /* First section-aligned location at or after __start_rodata. */ 429 extern char __start_rodata_section_aligned[]; 430 431 static struct section_perm nx_perms[] = { 432 /* Make pages tables, etc before _stext RW (set NX). */ 433 { 434 .name = "pre-text NX", 435 .start = PAGE_OFFSET, 436 .end = (unsigned long)_stext, 437 .mask = ~PMD_SECT_XN, 438 .prot = PMD_SECT_XN, 439 }, 440 /* Make init RW (set NX). */ 441 { 442 .name = "init NX", 443 .start = (unsigned long)__init_begin, 444 .end = (unsigned long)_sdata, 445 .mask = ~PMD_SECT_XN, 446 .prot = PMD_SECT_XN, 447 }, 448 /* Make rodata NX (set RO in ro_perms below). */ 449 { 450 .name = "rodata NX", 451 .start = (unsigned long)__start_rodata_section_aligned, 452 .end = (unsigned long)__init_begin, 453 .mask = ~PMD_SECT_XN, 454 .prot = PMD_SECT_XN, 455 }, 456 }; 457 458 static struct section_perm ro_perms[] = { 459 /* Make kernel code and rodata RX (set RO). */ 460 { 461 .name = "text/rodata RO", 462 .start = (unsigned long)_stext, 463 .end = (unsigned long)__init_begin, 464 #ifdef CONFIG_ARM_LPAE 465 .mask = ~(L_PMD_SECT_RDONLY | PMD_SECT_AP2), 466 .prot = L_PMD_SECT_RDONLY | PMD_SECT_AP2, 467 #else 468 .mask = ~(PMD_SECT_APX | PMD_SECT_AP_WRITE), 469 .prot = PMD_SECT_APX | PMD_SECT_AP_WRITE, 470 .clear = PMD_SECT_AP_WRITE, 471 #endif 472 }, 473 }; 474 475 /* 476 * Updates section permissions only for the current mm (sections are 477 * copied into each mm). During startup, this is the init_mm. Is only 478 * safe to be called with preemption disabled, as under stop_machine(). 479 */ 480 static inline void section_update(unsigned long addr, pmdval_t mask, 481 pmdval_t prot, struct mm_struct *mm) 482 { 483 pmd_t *pmd; 484 485 pmd = pmd_offset(pud_offset(p4d_offset(pgd_offset(mm, addr), addr), addr), addr); 486 487 #ifdef CONFIG_ARM_LPAE 488 pmd[0] = __pmd((pmd_val(pmd[0]) & mask) | prot); 489 #else 490 if (addr & SECTION_SIZE) 491 pmd[1] = __pmd((pmd_val(pmd[1]) & mask) | prot); 492 else 493 pmd[0] = __pmd((pmd_val(pmd[0]) & mask) | prot); 494 #endif 495 flush_pmd_entry(pmd); 496 local_flush_tlb_kernel_range(addr, addr + SECTION_SIZE); 497 } 498 499 /* Make sure extended page tables are in use. */ 500 static inline bool arch_has_strict_perms(void) 501 { 502 if (cpu_architecture() < CPU_ARCH_ARMv6) 503 return false; 504 505 return !!(get_cr() & CR_XP); 506 } 507 508 static void set_section_perms(struct section_perm *perms, int n, bool set, 509 struct mm_struct *mm) 510 { 511 size_t i; 512 unsigned long addr; 513 514 if (!arch_has_strict_perms()) 515 return; 516 517 for (i = 0; i < n; i++) { 518 if (!IS_ALIGNED(perms[i].start, SECTION_SIZE) || 519 !IS_ALIGNED(perms[i].end, SECTION_SIZE)) { 520 pr_err("BUG: %s section %lx-%lx not aligned to %lx\n", 521 perms[i].name, perms[i].start, perms[i].end, 522 SECTION_SIZE); 523 continue; 524 } 525 526 for (addr = perms[i].start; 527 addr < perms[i].end; 528 addr += SECTION_SIZE) 529 section_update(addr, perms[i].mask, 530 set ? perms[i].prot : perms[i].clear, mm); 531 } 532 533 } 534 535 /** 536 * update_sections_early intended to be called only through stop_machine 537 * framework and executed by only one CPU while all other CPUs will spin and 538 * wait, so no locking is required in this function. 539 */ 540 static void update_sections_early(struct section_perm perms[], int n) 541 { 542 struct task_struct *t, *s; 543 544 for_each_process(t) { 545 if (t->flags & PF_KTHREAD) 546 continue; 547 for_each_thread(t, s) 548 if (s->mm) 549 set_section_perms(perms, n, true, s->mm); 550 } 551 set_section_perms(perms, n, true, current->active_mm); 552 set_section_perms(perms, n, true, &init_mm); 553 } 554 555 static int __fix_kernmem_perms(void *unused) 556 { 557 update_sections_early(nx_perms, ARRAY_SIZE(nx_perms)); 558 return 0; 559 } 560 561 static void fix_kernmem_perms(void) 562 { 563 stop_machine(__fix_kernmem_perms, NULL, NULL); 564 } 565 566 static int __mark_rodata_ro(void *unused) 567 { 568 update_sections_early(ro_perms, ARRAY_SIZE(ro_perms)); 569 return 0; 570 } 571 572 static int kernel_set_to_readonly __read_mostly; 573 574 void mark_rodata_ro(void) 575 { 576 kernel_set_to_readonly = 1; 577 stop_machine(__mark_rodata_ro, NULL, NULL); 578 debug_checkwx(); 579 } 580 581 void set_kernel_text_rw(void) 582 { 583 if (!kernel_set_to_readonly) 584 return; 585 586 set_section_perms(ro_perms, ARRAY_SIZE(ro_perms), false, 587 current->active_mm); 588 } 589 590 void set_kernel_text_ro(void) 591 { 592 if (!kernel_set_to_readonly) 593 return; 594 595 set_section_perms(ro_perms, ARRAY_SIZE(ro_perms), true, 596 current->active_mm); 597 } 598 599 #else 600 static inline void fix_kernmem_perms(void) { } 601 #endif /* CONFIG_STRICT_KERNEL_RWX */ 602 603 void free_initmem(void) 604 { 605 fix_kernmem_perms(); 606 607 poison_init_mem(__init_begin, __init_end - __init_begin); 608 if (!machine_is_integrator() && !machine_is_cintegrator()) 609 free_initmem_default(-1); 610 } 611 612 #ifdef CONFIG_BLK_DEV_INITRD 613 void free_initrd_mem(unsigned long start, unsigned long end) 614 { 615 if (start == initrd_start) 616 start = round_down(start, PAGE_SIZE); 617 if (end == initrd_end) 618 end = round_up(end, PAGE_SIZE); 619 620 poison_init_mem((void *)start, PAGE_ALIGN(end) - start); 621 free_reserved_area((void *)start, (void *)end, -1, "initrd"); 622 } 623 #endif 624