1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * linux/arch/arm/mm/init.c 4 * 5 * Copyright (C) 1995-2005 Russell King 6 */ 7 #include <linux/kernel.h> 8 #include <linux/errno.h> 9 #include <linux/swap.h> 10 #include <linux/init.h> 11 #include <linux/mman.h> 12 #include <linux/sched/signal.h> 13 #include <linux/sched/task.h> 14 #include <linux/export.h> 15 #include <linux/nodemask.h> 16 #include <linux/initrd.h> 17 #include <linux/of_fdt.h> 18 #include <linux/highmem.h> 19 #include <linux/gfp.h> 20 #include <linux/memblock.h> 21 #include <linux/dma-map-ops.h> 22 #include <linux/sizes.h> 23 #include <linux/stop_machine.h> 24 #include <linux/swiotlb.h> 25 26 #include <asm/cp15.h> 27 #include <asm/mach-types.h> 28 #include <asm/memblock.h> 29 #include <asm/memory.h> 30 #include <asm/prom.h> 31 #include <asm/sections.h> 32 #include <asm/setup.h> 33 #include <asm/set_memory.h> 34 #include <asm/system_info.h> 35 #include <asm/tlb.h> 36 #include <asm/fixmap.h> 37 #include <asm/ptdump.h> 38 39 #include <asm/mach/arch.h> 40 #include <asm/mach/map.h> 41 42 #include "mm.h" 43 44 #ifdef CONFIG_CPU_CP15_MMU 45 unsigned long __init __clear_cr(unsigned long mask) 46 { 47 cr_alignment = cr_alignment & ~mask; 48 return cr_alignment; 49 } 50 #endif 51 52 #ifdef CONFIG_BLK_DEV_INITRD 53 static int __init parse_tag_initrd(const struct tag *tag) 54 { 55 pr_warn("ATAG_INITRD is deprecated; " 56 "please update your bootloader.\n"); 57 phys_initrd_start = __virt_to_phys(tag->u.initrd.start); 58 phys_initrd_size = tag->u.initrd.size; 59 return 0; 60 } 61 62 __tagtable(ATAG_INITRD, parse_tag_initrd); 63 64 static int __init parse_tag_initrd2(const struct tag *tag) 65 { 66 phys_initrd_start = tag->u.initrd.start; 67 phys_initrd_size = tag->u.initrd.size; 68 return 0; 69 } 70 71 __tagtable(ATAG_INITRD2, parse_tag_initrd2); 72 #endif 73 74 static void __init find_limits(unsigned long *min, unsigned long *max_low, 75 unsigned long *max_high) 76 { 77 *max_low = PFN_DOWN(memblock_get_current_limit()); 78 *min = PFN_UP(memblock_start_of_DRAM()); 79 *max_high = PFN_DOWN(memblock_end_of_DRAM()); 80 } 81 82 #ifdef CONFIG_ZONE_DMA 83 84 phys_addr_t arm_dma_zone_size __read_mostly; 85 EXPORT_SYMBOL(arm_dma_zone_size); 86 87 /* 88 * The DMA mask corresponding to the maximum bus address allocatable 89 * using GFP_DMA. The default here places no restriction on DMA 90 * allocations. This must be the smallest DMA mask in the system, 91 * so a successful GFP_DMA allocation will always satisfy this. 92 */ 93 phys_addr_t arm_dma_limit; 94 unsigned long arm_dma_pfn_limit; 95 #endif 96 97 void __init setup_dma_zone(const struct machine_desc *mdesc) 98 { 99 #ifdef CONFIG_ZONE_DMA 100 if (mdesc->dma_zone_size) { 101 arm_dma_zone_size = mdesc->dma_zone_size; 102 arm_dma_limit = PHYS_OFFSET + arm_dma_zone_size - 1; 103 } else 104 arm_dma_limit = 0xffffffff; 105 arm_dma_pfn_limit = arm_dma_limit >> PAGE_SHIFT; 106 #endif 107 } 108 109 static void __init zone_sizes_init(unsigned long min, unsigned long max_low, 110 unsigned long max_high) 111 { 112 unsigned long max_zone_pfn[MAX_NR_ZONES] = { 0 }; 113 114 #ifdef CONFIG_ZONE_DMA 115 max_zone_pfn[ZONE_DMA] = min(arm_dma_pfn_limit, max_low); 116 #endif 117 max_zone_pfn[ZONE_NORMAL] = max_low; 118 #ifdef CONFIG_HIGHMEM 119 max_zone_pfn[ZONE_HIGHMEM] = max_high; 120 #endif 121 free_area_init(max_zone_pfn); 122 } 123 124 #ifdef CONFIG_HAVE_ARCH_PFN_VALID 125 int pfn_valid(unsigned long pfn) 126 { 127 phys_addr_t addr = __pfn_to_phys(pfn); 128 129 if (__phys_to_pfn(addr) != pfn) 130 return 0; 131 132 return memblock_is_map_memory(addr); 133 } 134 EXPORT_SYMBOL(pfn_valid); 135 #endif 136 137 static bool arm_memblock_steal_permitted = true; 138 139 phys_addr_t __init arm_memblock_steal(phys_addr_t size, phys_addr_t align) 140 { 141 phys_addr_t phys; 142 143 BUG_ON(!arm_memblock_steal_permitted); 144 145 phys = memblock_phys_alloc(size, align); 146 if (!phys) 147 panic("Failed to steal %pa bytes at %pS\n", 148 &size, (void *)_RET_IP_); 149 150 memblock_free(phys, size); 151 memblock_remove(phys, size); 152 153 return phys; 154 } 155 156 static void __init arm_initrd_init(void) 157 { 158 #ifdef CONFIG_BLK_DEV_INITRD 159 phys_addr_t start; 160 unsigned long size; 161 162 initrd_start = initrd_end = 0; 163 164 if (!phys_initrd_size) 165 return; 166 167 /* 168 * Round the memory region to page boundaries as per free_initrd_mem() 169 * This allows us to detect whether the pages overlapping the initrd 170 * are in use, but more importantly, reserves the entire set of pages 171 * as we don't want these pages allocated for other purposes. 172 */ 173 start = round_down(phys_initrd_start, PAGE_SIZE); 174 size = phys_initrd_size + (phys_initrd_start - start); 175 size = round_up(size, PAGE_SIZE); 176 177 if (!memblock_is_region_memory(start, size)) { 178 pr_err("INITRD: 0x%08llx+0x%08lx is not a memory region - disabling initrd\n", 179 (u64)start, size); 180 return; 181 } 182 183 if (memblock_is_region_reserved(start, size)) { 184 pr_err("INITRD: 0x%08llx+0x%08lx overlaps in-use memory region - disabling initrd\n", 185 (u64)start, size); 186 return; 187 } 188 189 memblock_reserve(start, size); 190 191 /* Now convert initrd to virtual addresses */ 192 initrd_start = __phys_to_virt(phys_initrd_start); 193 initrd_end = initrd_start + phys_initrd_size; 194 #endif 195 } 196 197 #ifdef CONFIG_CPU_ICACHE_MISMATCH_WORKAROUND 198 void check_cpu_icache_size(int cpuid) 199 { 200 u32 size, ctr; 201 202 asm("mrc p15, 0, %0, c0, c0, 1" : "=r" (ctr)); 203 204 size = 1 << ((ctr & 0xf) + 2); 205 if (cpuid != 0 && icache_size != size) 206 pr_info("CPU%u: detected I-Cache line size mismatch, workaround enabled\n", 207 cpuid); 208 if (icache_size > size) 209 icache_size = size; 210 } 211 #endif 212 213 void __init arm_memblock_init(const struct machine_desc *mdesc) 214 { 215 /* Register the kernel text, kernel data and initrd with memblock. */ 216 memblock_reserve(__pa(KERNEL_START), KERNEL_END - KERNEL_START); 217 218 arm_initrd_init(); 219 220 arm_mm_memblock_reserve(); 221 222 /* reserve any platform specific memblock areas */ 223 if (mdesc->reserve) 224 mdesc->reserve(); 225 226 early_init_fdt_scan_reserved_mem(); 227 228 /* reserve memory for DMA contiguous allocations */ 229 dma_contiguous_reserve(arm_dma_limit); 230 231 arm_memblock_steal_permitted = false; 232 memblock_dump_all(); 233 } 234 235 void __init bootmem_init(void) 236 { 237 memblock_allow_resize(); 238 239 find_limits(&min_low_pfn, &max_low_pfn, &max_pfn); 240 241 early_memtest((phys_addr_t)min_low_pfn << PAGE_SHIFT, 242 (phys_addr_t)max_low_pfn << PAGE_SHIFT); 243 244 /* 245 * sparse_init() tries to allocate memory from memblock, so must be 246 * done after the fixed reservations 247 */ 248 sparse_init(); 249 250 /* 251 * Now free the memory - free_area_init needs 252 * the sparse mem_map arrays initialized by sparse_init() 253 * for memmap_init_zone(), otherwise all PFNs are invalid. 254 */ 255 zone_sizes_init(min_low_pfn, max_low_pfn, max_pfn); 256 } 257 258 /* 259 * Poison init memory with an undefined instruction (ARM) or a branch to an 260 * undefined instruction (Thumb). 261 */ 262 static inline void poison_init_mem(void *s, size_t count) 263 { 264 u32 *p = (u32 *)s; 265 for (; count != 0; count -= 4) 266 *p++ = 0xe7fddef0; 267 } 268 269 static inline void __init 270 free_memmap(unsigned long start_pfn, unsigned long end_pfn) 271 { 272 struct page *start_pg, *end_pg; 273 phys_addr_t pg, pgend; 274 275 /* 276 * Convert start_pfn/end_pfn to a struct page pointer. 277 */ 278 start_pg = pfn_to_page(start_pfn - 1) + 1; 279 end_pg = pfn_to_page(end_pfn - 1) + 1; 280 281 /* 282 * Convert to physical addresses, and 283 * round start upwards and end downwards. 284 */ 285 pg = PAGE_ALIGN(__pa(start_pg)); 286 pgend = __pa(end_pg) & PAGE_MASK; 287 288 /* 289 * If there are free pages between these, 290 * free the section of the memmap array. 291 */ 292 if (pg < pgend) 293 memblock_free_early(pg, pgend - pg); 294 } 295 296 /* 297 * The mem_map array can get very big. Free the unused area of the memory map. 298 */ 299 static void __init free_unused_memmap(void) 300 { 301 unsigned long start, end, prev_end = 0; 302 int i; 303 304 /* 305 * This relies on each bank being in address order. 306 * The banks are sorted previously in bootmem_init(). 307 */ 308 for_each_mem_pfn_range(i, MAX_NUMNODES, &start, &end, NULL) { 309 #ifdef CONFIG_SPARSEMEM 310 /* 311 * Take care not to free memmap entries that don't exist 312 * due to SPARSEMEM sections which aren't present. 313 */ 314 start = min(start, 315 ALIGN(prev_end, PAGES_PER_SECTION)); 316 #else 317 /* 318 * Align down here since the VM subsystem insists that the 319 * memmap entries are valid from the bank start aligned to 320 * MAX_ORDER_NR_PAGES. 321 */ 322 start = round_down(start, MAX_ORDER_NR_PAGES); 323 #endif 324 /* 325 * If we had a previous bank, and there is a space 326 * between the current bank and the previous, free it. 327 */ 328 if (prev_end && prev_end < start) 329 free_memmap(prev_end, start); 330 331 /* 332 * Align up here since the VM subsystem insists that the 333 * memmap entries are valid from the bank end aligned to 334 * MAX_ORDER_NR_PAGES. 335 */ 336 prev_end = ALIGN(end, MAX_ORDER_NR_PAGES); 337 } 338 339 #ifdef CONFIG_SPARSEMEM 340 if (!IS_ALIGNED(prev_end, PAGES_PER_SECTION)) 341 free_memmap(prev_end, 342 ALIGN(prev_end, PAGES_PER_SECTION)); 343 #endif 344 } 345 346 static void __init free_highpages(void) 347 { 348 #ifdef CONFIG_HIGHMEM 349 unsigned long max_low = max_low_pfn; 350 phys_addr_t range_start, range_end; 351 u64 i; 352 353 /* set highmem page free */ 354 for_each_free_mem_range(i, NUMA_NO_NODE, MEMBLOCK_NONE, 355 &range_start, &range_end, NULL) { 356 unsigned long start = PHYS_PFN(range_start); 357 unsigned long end = PHYS_PFN(range_end); 358 359 /* Ignore complete lowmem entries */ 360 if (end <= max_low) 361 continue; 362 363 /* Truncate partial highmem entries */ 364 if (start < max_low) 365 start = max_low; 366 367 for (; start < end; start++) 368 free_highmem_page(pfn_to_page(start)); 369 } 370 #endif 371 } 372 373 /* 374 * mem_init() marks the free areas in the mem_map and tells us how much 375 * memory is free. This is done after various parts of the system have 376 * claimed their memory after the kernel image. 377 */ 378 void __init mem_init(void) 379 { 380 #ifdef CONFIG_ARM_LPAE 381 swiotlb_init(1); 382 #endif 383 384 set_max_mapnr(pfn_to_page(max_pfn) - mem_map); 385 386 /* this will put all unused low memory onto the freelists */ 387 free_unused_memmap(); 388 memblock_free_all(); 389 390 #ifdef CONFIG_SA1111 391 /* now that our DMA memory is actually so designated, we can free it */ 392 free_reserved_area(__va(PHYS_OFFSET), swapper_pg_dir, -1, NULL); 393 #endif 394 395 free_highpages(); 396 397 mem_init_print_info(NULL); 398 399 /* 400 * Check boundaries twice: Some fundamental inconsistencies can 401 * be detected at build time already. 402 */ 403 #ifdef CONFIG_MMU 404 BUILD_BUG_ON(TASK_SIZE > MODULES_VADDR); 405 BUG_ON(TASK_SIZE > MODULES_VADDR); 406 #endif 407 408 #ifdef CONFIG_HIGHMEM 409 BUILD_BUG_ON(PKMAP_BASE + LAST_PKMAP * PAGE_SIZE > PAGE_OFFSET); 410 BUG_ON(PKMAP_BASE + LAST_PKMAP * PAGE_SIZE > PAGE_OFFSET); 411 #endif 412 } 413 414 #ifdef CONFIG_STRICT_KERNEL_RWX 415 struct section_perm { 416 const char *name; 417 unsigned long start; 418 unsigned long end; 419 pmdval_t mask; 420 pmdval_t prot; 421 pmdval_t clear; 422 }; 423 424 /* First section-aligned location at or after __start_rodata. */ 425 extern char __start_rodata_section_aligned[]; 426 427 static struct section_perm nx_perms[] = { 428 /* Make pages tables, etc before _stext RW (set NX). */ 429 { 430 .name = "pre-text NX", 431 .start = PAGE_OFFSET, 432 .end = (unsigned long)_stext, 433 .mask = ~PMD_SECT_XN, 434 .prot = PMD_SECT_XN, 435 }, 436 /* Make init RW (set NX). */ 437 { 438 .name = "init NX", 439 .start = (unsigned long)__init_begin, 440 .end = (unsigned long)_sdata, 441 .mask = ~PMD_SECT_XN, 442 .prot = PMD_SECT_XN, 443 }, 444 /* Make rodata NX (set RO in ro_perms below). */ 445 { 446 .name = "rodata NX", 447 .start = (unsigned long)__start_rodata_section_aligned, 448 .end = (unsigned long)__init_begin, 449 .mask = ~PMD_SECT_XN, 450 .prot = PMD_SECT_XN, 451 }, 452 }; 453 454 static struct section_perm ro_perms[] = { 455 /* Make kernel code and rodata RX (set RO). */ 456 { 457 .name = "text/rodata RO", 458 .start = (unsigned long)_stext, 459 .end = (unsigned long)__init_begin, 460 #ifdef CONFIG_ARM_LPAE 461 .mask = ~(L_PMD_SECT_RDONLY | PMD_SECT_AP2), 462 .prot = L_PMD_SECT_RDONLY | PMD_SECT_AP2, 463 #else 464 .mask = ~(PMD_SECT_APX | PMD_SECT_AP_WRITE), 465 .prot = PMD_SECT_APX | PMD_SECT_AP_WRITE, 466 .clear = PMD_SECT_AP_WRITE, 467 #endif 468 }, 469 }; 470 471 /* 472 * Updates section permissions only for the current mm (sections are 473 * copied into each mm). During startup, this is the init_mm. Is only 474 * safe to be called with preemption disabled, as under stop_machine(). 475 */ 476 static inline void section_update(unsigned long addr, pmdval_t mask, 477 pmdval_t prot, struct mm_struct *mm) 478 { 479 pmd_t *pmd; 480 481 pmd = pmd_offset(pud_offset(p4d_offset(pgd_offset(mm, addr), addr), addr), addr); 482 483 #ifdef CONFIG_ARM_LPAE 484 pmd[0] = __pmd((pmd_val(pmd[0]) & mask) | prot); 485 #else 486 if (addr & SECTION_SIZE) 487 pmd[1] = __pmd((pmd_val(pmd[1]) & mask) | prot); 488 else 489 pmd[0] = __pmd((pmd_val(pmd[0]) & mask) | prot); 490 #endif 491 flush_pmd_entry(pmd); 492 local_flush_tlb_kernel_range(addr, addr + SECTION_SIZE); 493 } 494 495 /* Make sure extended page tables are in use. */ 496 static inline bool arch_has_strict_perms(void) 497 { 498 if (cpu_architecture() < CPU_ARCH_ARMv6) 499 return false; 500 501 return !!(get_cr() & CR_XP); 502 } 503 504 static void set_section_perms(struct section_perm *perms, int n, bool set, 505 struct mm_struct *mm) 506 { 507 size_t i; 508 unsigned long addr; 509 510 if (!arch_has_strict_perms()) 511 return; 512 513 for (i = 0; i < n; i++) { 514 if (!IS_ALIGNED(perms[i].start, SECTION_SIZE) || 515 !IS_ALIGNED(perms[i].end, SECTION_SIZE)) { 516 pr_err("BUG: %s section %lx-%lx not aligned to %lx\n", 517 perms[i].name, perms[i].start, perms[i].end, 518 SECTION_SIZE); 519 continue; 520 } 521 522 for (addr = perms[i].start; 523 addr < perms[i].end; 524 addr += SECTION_SIZE) 525 section_update(addr, perms[i].mask, 526 set ? perms[i].prot : perms[i].clear, mm); 527 } 528 529 } 530 531 /** 532 * update_sections_early intended to be called only through stop_machine 533 * framework and executed by only one CPU while all other CPUs will spin and 534 * wait, so no locking is required in this function. 535 */ 536 static void update_sections_early(struct section_perm perms[], int n) 537 { 538 struct task_struct *t, *s; 539 540 for_each_process(t) { 541 if (t->flags & PF_KTHREAD) 542 continue; 543 for_each_thread(t, s) 544 if (s->mm) 545 set_section_perms(perms, n, true, s->mm); 546 } 547 set_section_perms(perms, n, true, current->active_mm); 548 set_section_perms(perms, n, true, &init_mm); 549 } 550 551 static int __fix_kernmem_perms(void *unused) 552 { 553 update_sections_early(nx_perms, ARRAY_SIZE(nx_perms)); 554 return 0; 555 } 556 557 static void fix_kernmem_perms(void) 558 { 559 stop_machine(__fix_kernmem_perms, NULL, NULL); 560 } 561 562 static int __mark_rodata_ro(void *unused) 563 { 564 update_sections_early(ro_perms, ARRAY_SIZE(ro_perms)); 565 return 0; 566 } 567 568 static int kernel_set_to_readonly __read_mostly; 569 570 void mark_rodata_ro(void) 571 { 572 kernel_set_to_readonly = 1; 573 stop_machine(__mark_rodata_ro, NULL, NULL); 574 debug_checkwx(); 575 } 576 577 void set_kernel_text_rw(void) 578 { 579 if (!kernel_set_to_readonly) 580 return; 581 582 set_section_perms(ro_perms, ARRAY_SIZE(ro_perms), false, 583 current->active_mm); 584 } 585 586 void set_kernel_text_ro(void) 587 { 588 if (!kernel_set_to_readonly) 589 return; 590 591 set_section_perms(ro_perms, ARRAY_SIZE(ro_perms), true, 592 current->active_mm); 593 } 594 595 #else 596 static inline void fix_kernmem_perms(void) { } 597 #endif /* CONFIG_STRICT_KERNEL_RWX */ 598 599 void free_initmem(void) 600 { 601 fix_kernmem_perms(); 602 603 poison_init_mem(__init_begin, __init_end - __init_begin); 604 if (!machine_is_integrator() && !machine_is_cintegrator()) 605 free_initmem_default(-1); 606 } 607 608 #ifdef CONFIG_BLK_DEV_INITRD 609 void free_initrd_mem(unsigned long start, unsigned long end) 610 { 611 if (start == initrd_start) 612 start = round_down(start, PAGE_SIZE); 613 if (end == initrd_end) 614 end = round_up(end, PAGE_SIZE); 615 616 poison_init_mem((void *)start, PAGE_ALIGN(end) - start); 617 free_reserved_area((void *)start, (void *)end, -1, "initrd"); 618 } 619 #endif 620