1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * linux/arch/arm/kernel/setup.c 4 * 5 * Copyright (C) 1995-2001 Russell King 6 */ 7 #include <linux/efi.h> 8 #include <linux/export.h> 9 #include <linux/kernel.h> 10 #include <linux/stddef.h> 11 #include <linux/ioport.h> 12 #include <linux/delay.h> 13 #include <linux/utsname.h> 14 #include <linux/initrd.h> 15 #include <linux/console.h> 16 #include <linux/seq_file.h> 17 #include <linux/screen_info.h> 18 #include <linux/of_platform.h> 19 #include <linux/init.h> 20 #include <linux/kexec.h> 21 #include <linux/libfdt.h> 22 #include <linux/of_fdt.h> 23 #include <linux/cpu.h> 24 #include <linux/interrupt.h> 25 #include <linux/smp.h> 26 #include <linux/proc_fs.h> 27 #include <linux/memblock.h> 28 #include <linux/bug.h> 29 #include <linux/compiler.h> 30 #include <linux/sort.h> 31 #include <linux/psci.h> 32 33 #include <asm/unified.h> 34 #include <asm/cp15.h> 35 #include <asm/cpu.h> 36 #include <asm/cputype.h> 37 #include <asm/efi.h> 38 #include <asm/elf.h> 39 #include <asm/early_ioremap.h> 40 #include <asm/fixmap.h> 41 #include <asm/procinfo.h> 42 #include <asm/psci.h> 43 #include <asm/sections.h> 44 #include <asm/setup.h> 45 #include <asm/smp_plat.h> 46 #include <asm/mach-types.h> 47 #include <asm/cacheflush.h> 48 #include <asm/cachetype.h> 49 #include <asm/tlbflush.h> 50 #include <asm/xen/hypervisor.h> 51 52 #include <asm/prom.h> 53 #include <asm/mach/arch.h> 54 #include <asm/mach/irq.h> 55 #include <asm/mach/time.h> 56 #include <asm/system_info.h> 57 #include <asm/system_misc.h> 58 #include <asm/traps.h> 59 #include <asm/unwind.h> 60 #include <asm/memblock.h> 61 #include <asm/virt.h> 62 #include <asm/kasan.h> 63 64 #include "atags.h" 65 66 67 #if defined(CONFIG_FPE_NWFPE) || defined(CONFIG_FPE_FASTFPE) 68 char fpe_type[8]; 69 70 static int __init fpe_setup(char *line) 71 { 72 memcpy(fpe_type, line, 8); 73 return 1; 74 } 75 76 __setup("fpe=", fpe_setup); 77 #endif 78 79 extern void init_default_cache_policy(unsigned long); 80 extern void paging_init(const struct machine_desc *desc); 81 extern void early_mm_init(const struct machine_desc *); 82 extern void adjust_lowmem_bounds(void); 83 extern enum reboot_mode reboot_mode; 84 extern void setup_dma_zone(const struct machine_desc *desc); 85 86 unsigned int processor_id; 87 EXPORT_SYMBOL(processor_id); 88 unsigned int __machine_arch_type __read_mostly; 89 EXPORT_SYMBOL(__machine_arch_type); 90 unsigned int cacheid __read_mostly; 91 EXPORT_SYMBOL(cacheid); 92 93 unsigned int __atags_pointer __initdata; 94 95 unsigned int system_rev; 96 EXPORT_SYMBOL(system_rev); 97 98 const char *system_serial; 99 EXPORT_SYMBOL(system_serial); 100 101 unsigned int system_serial_low; 102 EXPORT_SYMBOL(system_serial_low); 103 104 unsigned int system_serial_high; 105 EXPORT_SYMBOL(system_serial_high); 106 107 unsigned int elf_hwcap __read_mostly; 108 EXPORT_SYMBOL(elf_hwcap); 109 110 unsigned int elf_hwcap2 __read_mostly; 111 EXPORT_SYMBOL(elf_hwcap2); 112 113 114 #ifdef MULTI_CPU 115 struct processor processor __ro_after_init; 116 #if defined(CONFIG_BIG_LITTLE) && defined(CONFIG_HARDEN_BRANCH_PREDICTOR) 117 struct processor *cpu_vtable[NR_CPUS] = { 118 [0] = &processor, 119 }; 120 #endif 121 #endif 122 #ifdef MULTI_TLB 123 struct cpu_tlb_fns cpu_tlb __ro_after_init; 124 #endif 125 #ifdef MULTI_USER 126 struct cpu_user_fns cpu_user __ro_after_init; 127 #endif 128 #ifdef MULTI_CACHE 129 struct cpu_cache_fns cpu_cache __ro_after_init; 130 #endif 131 #ifdef CONFIG_OUTER_CACHE 132 struct outer_cache_fns outer_cache __ro_after_init; 133 EXPORT_SYMBOL(outer_cache); 134 #endif 135 136 /* 137 * Cached cpu_architecture() result for use by assembler code. 138 * C code should use the cpu_architecture() function instead of accessing this 139 * variable directly. 140 */ 141 int __cpu_architecture __read_mostly = CPU_ARCH_UNKNOWN; 142 143 struct stack { 144 u32 irq[3]; 145 u32 abt[3]; 146 u32 und[3]; 147 u32 fiq[3]; 148 } ____cacheline_aligned; 149 150 #ifndef CONFIG_CPU_V7M 151 static struct stack stacks[NR_CPUS]; 152 #endif 153 154 char elf_platform[ELF_PLATFORM_SIZE]; 155 EXPORT_SYMBOL(elf_platform); 156 157 static const char *cpu_name; 158 static const char *machine_name; 159 static char __initdata cmd_line[COMMAND_LINE_SIZE]; 160 const struct machine_desc *machine_desc __initdata; 161 162 static union { char c[4]; unsigned long l; } endian_test __initdata = { { 'l', '?', '?', 'b' } }; 163 #define ENDIANNESS ((char)endian_test.l) 164 165 DEFINE_PER_CPU(struct cpuinfo_arm, cpu_data); 166 167 /* 168 * Standard memory resources 169 */ 170 static struct resource mem_res[] = { 171 { 172 .name = "Video RAM", 173 .start = 0, 174 .end = 0, 175 .flags = IORESOURCE_MEM 176 }, 177 { 178 .name = "Kernel code", 179 .start = 0, 180 .end = 0, 181 .flags = IORESOURCE_SYSTEM_RAM 182 }, 183 { 184 .name = "Kernel data", 185 .start = 0, 186 .end = 0, 187 .flags = IORESOURCE_SYSTEM_RAM 188 } 189 }; 190 191 #define video_ram mem_res[0] 192 #define kernel_code mem_res[1] 193 #define kernel_data mem_res[2] 194 195 static struct resource io_res[] = { 196 { 197 .name = "reserved", 198 .start = 0x3bc, 199 .end = 0x3be, 200 .flags = IORESOURCE_IO | IORESOURCE_BUSY 201 }, 202 { 203 .name = "reserved", 204 .start = 0x378, 205 .end = 0x37f, 206 .flags = IORESOURCE_IO | IORESOURCE_BUSY 207 }, 208 { 209 .name = "reserved", 210 .start = 0x278, 211 .end = 0x27f, 212 .flags = IORESOURCE_IO | IORESOURCE_BUSY 213 } 214 }; 215 216 #define lp0 io_res[0] 217 #define lp1 io_res[1] 218 #define lp2 io_res[2] 219 220 static const char *proc_arch[] = { 221 "undefined/unknown", 222 "3", 223 "4", 224 "4T", 225 "5", 226 "5T", 227 "5TE", 228 "5TEJ", 229 "6TEJ", 230 "7", 231 "7M", 232 "?(12)", 233 "?(13)", 234 "?(14)", 235 "?(15)", 236 "?(16)", 237 "?(17)", 238 }; 239 240 #ifdef CONFIG_CPU_V7M 241 static int __get_cpu_architecture(void) 242 { 243 return CPU_ARCH_ARMv7M; 244 } 245 #else 246 static int __get_cpu_architecture(void) 247 { 248 int cpu_arch; 249 250 if ((read_cpuid_id() & 0x0008f000) == 0) { 251 cpu_arch = CPU_ARCH_UNKNOWN; 252 } else if ((read_cpuid_id() & 0x0008f000) == 0x00007000) { 253 cpu_arch = (read_cpuid_id() & (1 << 23)) ? CPU_ARCH_ARMv4T : CPU_ARCH_ARMv3; 254 } else if ((read_cpuid_id() & 0x00080000) == 0x00000000) { 255 cpu_arch = (read_cpuid_id() >> 16) & 7; 256 if (cpu_arch) 257 cpu_arch += CPU_ARCH_ARMv3; 258 } else if ((read_cpuid_id() & 0x000f0000) == 0x000f0000) { 259 /* Revised CPUID format. Read the Memory Model Feature 260 * Register 0 and check for VMSAv7 or PMSAv7 */ 261 unsigned int mmfr0 = read_cpuid_ext(CPUID_EXT_MMFR0); 262 if ((mmfr0 & 0x0000000f) >= 0x00000003 || 263 (mmfr0 & 0x000000f0) >= 0x00000030) 264 cpu_arch = CPU_ARCH_ARMv7; 265 else if ((mmfr0 & 0x0000000f) == 0x00000002 || 266 (mmfr0 & 0x000000f0) == 0x00000020) 267 cpu_arch = CPU_ARCH_ARMv6; 268 else 269 cpu_arch = CPU_ARCH_UNKNOWN; 270 } else 271 cpu_arch = CPU_ARCH_UNKNOWN; 272 273 return cpu_arch; 274 } 275 #endif 276 277 int __pure cpu_architecture(void) 278 { 279 BUG_ON(__cpu_architecture == CPU_ARCH_UNKNOWN); 280 281 return __cpu_architecture; 282 } 283 284 static int cpu_has_aliasing_icache(unsigned int arch) 285 { 286 int aliasing_icache; 287 unsigned int id_reg, num_sets, line_size; 288 289 /* PIPT caches never alias. */ 290 if (icache_is_pipt()) 291 return 0; 292 293 /* arch specifies the register format */ 294 switch (arch) { 295 case CPU_ARCH_ARMv7: 296 set_csselr(CSSELR_ICACHE | CSSELR_L1); 297 isb(); 298 id_reg = read_ccsidr(); 299 line_size = 4 << ((id_reg & 0x7) + 2); 300 num_sets = ((id_reg >> 13) & 0x7fff) + 1; 301 aliasing_icache = (line_size * num_sets) > PAGE_SIZE; 302 break; 303 case CPU_ARCH_ARMv6: 304 aliasing_icache = read_cpuid_cachetype() & (1 << 11); 305 break; 306 default: 307 /* I-cache aliases will be handled by D-cache aliasing code */ 308 aliasing_icache = 0; 309 } 310 311 return aliasing_icache; 312 } 313 314 static void __init cacheid_init(void) 315 { 316 unsigned int arch = cpu_architecture(); 317 318 if (arch >= CPU_ARCH_ARMv6) { 319 unsigned int cachetype = read_cpuid_cachetype(); 320 321 if ((arch == CPU_ARCH_ARMv7M) && !(cachetype & 0xf000f)) { 322 cacheid = 0; 323 } else if ((cachetype & (7 << 29)) == 4 << 29) { 324 /* ARMv7 register format */ 325 arch = CPU_ARCH_ARMv7; 326 cacheid = CACHEID_VIPT_NONALIASING; 327 switch (cachetype & (3 << 14)) { 328 case (1 << 14): 329 cacheid |= CACHEID_ASID_TAGGED; 330 break; 331 case (3 << 14): 332 cacheid |= CACHEID_PIPT; 333 break; 334 } 335 } else { 336 arch = CPU_ARCH_ARMv6; 337 if (cachetype & (1 << 23)) 338 cacheid = CACHEID_VIPT_ALIASING; 339 else 340 cacheid = CACHEID_VIPT_NONALIASING; 341 } 342 if (cpu_has_aliasing_icache(arch)) 343 cacheid |= CACHEID_VIPT_I_ALIASING; 344 } else { 345 cacheid = CACHEID_VIVT; 346 } 347 348 pr_info("CPU: %s data cache, %s instruction cache\n", 349 cache_is_vivt() ? "VIVT" : 350 cache_is_vipt_aliasing() ? "VIPT aliasing" : 351 cache_is_vipt_nonaliasing() ? "PIPT / VIPT nonaliasing" : "unknown", 352 cache_is_vivt() ? "VIVT" : 353 icache_is_vivt_asid_tagged() ? "VIVT ASID tagged" : 354 icache_is_vipt_aliasing() ? "VIPT aliasing" : 355 icache_is_pipt() ? "PIPT" : 356 cache_is_vipt_nonaliasing() ? "VIPT nonaliasing" : "unknown"); 357 } 358 359 /* 360 * These functions re-use the assembly code in head.S, which 361 * already provide the required functionality. 362 */ 363 extern struct proc_info_list *lookup_processor_type(unsigned int); 364 365 void __init early_print(const char *str, ...) 366 { 367 extern void printascii(const char *); 368 char buf[256]; 369 va_list ap; 370 371 va_start(ap, str); 372 vsnprintf(buf, sizeof(buf), str, ap); 373 va_end(ap); 374 375 #ifdef CONFIG_DEBUG_LL 376 printascii(buf); 377 #endif 378 printk("%s", buf); 379 } 380 381 #ifdef CONFIG_ARM_PATCH_IDIV 382 383 static inline u32 __attribute_const__ sdiv_instruction(void) 384 { 385 if (IS_ENABLED(CONFIG_THUMB2_KERNEL)) { 386 /* "sdiv r0, r0, r1" */ 387 u32 insn = __opcode_thumb32_compose(0xfb90, 0xf0f1); 388 return __opcode_to_mem_thumb32(insn); 389 } 390 391 /* "sdiv r0, r0, r1" */ 392 return __opcode_to_mem_arm(0xe710f110); 393 } 394 395 static inline u32 __attribute_const__ udiv_instruction(void) 396 { 397 if (IS_ENABLED(CONFIG_THUMB2_KERNEL)) { 398 /* "udiv r0, r0, r1" */ 399 u32 insn = __opcode_thumb32_compose(0xfbb0, 0xf0f1); 400 return __opcode_to_mem_thumb32(insn); 401 } 402 403 /* "udiv r0, r0, r1" */ 404 return __opcode_to_mem_arm(0xe730f110); 405 } 406 407 static inline u32 __attribute_const__ bx_lr_instruction(void) 408 { 409 if (IS_ENABLED(CONFIG_THUMB2_KERNEL)) { 410 /* "bx lr; nop" */ 411 u32 insn = __opcode_thumb32_compose(0x4770, 0x46c0); 412 return __opcode_to_mem_thumb32(insn); 413 } 414 415 /* "bx lr" */ 416 return __opcode_to_mem_arm(0xe12fff1e); 417 } 418 419 static void __init patch_aeabi_idiv(void) 420 { 421 extern void __aeabi_uidiv(void); 422 extern void __aeabi_idiv(void); 423 uintptr_t fn_addr; 424 unsigned int mask; 425 426 mask = IS_ENABLED(CONFIG_THUMB2_KERNEL) ? HWCAP_IDIVT : HWCAP_IDIVA; 427 if (!(elf_hwcap & mask)) 428 return; 429 430 pr_info("CPU: div instructions available: patching division code\n"); 431 432 fn_addr = ((uintptr_t)&__aeabi_uidiv) & ~1; 433 asm ("" : "+g" (fn_addr)); 434 ((u32 *)fn_addr)[0] = udiv_instruction(); 435 ((u32 *)fn_addr)[1] = bx_lr_instruction(); 436 flush_icache_range(fn_addr, fn_addr + 8); 437 438 fn_addr = ((uintptr_t)&__aeabi_idiv) & ~1; 439 asm ("" : "+g" (fn_addr)); 440 ((u32 *)fn_addr)[0] = sdiv_instruction(); 441 ((u32 *)fn_addr)[1] = bx_lr_instruction(); 442 flush_icache_range(fn_addr, fn_addr + 8); 443 } 444 445 #else 446 static inline void patch_aeabi_idiv(void) { } 447 #endif 448 449 static void __init cpuid_init_hwcaps(void) 450 { 451 int block; 452 u32 isar5; 453 454 if (cpu_architecture() < CPU_ARCH_ARMv7) 455 return; 456 457 block = cpuid_feature_extract(CPUID_EXT_ISAR0, 24); 458 if (block >= 2) 459 elf_hwcap |= HWCAP_IDIVA; 460 if (block >= 1) 461 elf_hwcap |= HWCAP_IDIVT; 462 463 /* LPAE implies atomic ldrd/strd instructions */ 464 block = cpuid_feature_extract(CPUID_EXT_MMFR0, 0); 465 if (block >= 5) 466 elf_hwcap |= HWCAP_LPAE; 467 468 /* check for supported v8 Crypto instructions */ 469 isar5 = read_cpuid_ext(CPUID_EXT_ISAR5); 470 471 block = cpuid_feature_extract_field(isar5, 4); 472 if (block >= 2) 473 elf_hwcap2 |= HWCAP2_PMULL; 474 if (block >= 1) 475 elf_hwcap2 |= HWCAP2_AES; 476 477 block = cpuid_feature_extract_field(isar5, 8); 478 if (block >= 1) 479 elf_hwcap2 |= HWCAP2_SHA1; 480 481 block = cpuid_feature_extract_field(isar5, 12); 482 if (block >= 1) 483 elf_hwcap2 |= HWCAP2_SHA2; 484 485 block = cpuid_feature_extract_field(isar5, 16); 486 if (block >= 1) 487 elf_hwcap2 |= HWCAP2_CRC32; 488 } 489 490 static void __init elf_hwcap_fixup(void) 491 { 492 unsigned id = read_cpuid_id(); 493 494 /* 495 * HWCAP_TLS is available only on 1136 r1p0 and later, 496 * see also kuser_get_tls_init. 497 */ 498 if (read_cpuid_part() == ARM_CPU_PART_ARM1136 && 499 ((id >> 20) & 3) == 0) { 500 elf_hwcap &= ~HWCAP_TLS; 501 return; 502 } 503 504 /* Verify if CPUID scheme is implemented */ 505 if ((id & 0x000f0000) != 0x000f0000) 506 return; 507 508 /* 509 * If the CPU supports LDREX/STREX and LDREXB/STREXB, 510 * avoid advertising SWP; it may not be atomic with 511 * multiprocessing cores. 512 */ 513 if (cpuid_feature_extract(CPUID_EXT_ISAR3, 12) > 1 || 514 (cpuid_feature_extract(CPUID_EXT_ISAR3, 12) == 1 && 515 cpuid_feature_extract(CPUID_EXT_ISAR4, 20) >= 3)) 516 elf_hwcap &= ~HWCAP_SWP; 517 } 518 519 /* 520 * cpu_init - initialise one CPU. 521 * 522 * cpu_init sets up the per-CPU stacks. 523 */ 524 void notrace cpu_init(void) 525 { 526 #ifndef CONFIG_CPU_V7M 527 unsigned int cpu = smp_processor_id(); 528 struct stack *stk = &stacks[cpu]; 529 530 if (cpu >= NR_CPUS) { 531 pr_crit("CPU%u: bad primary CPU number\n", cpu); 532 BUG(); 533 } 534 535 /* 536 * This only works on resume and secondary cores. For booting on the 537 * boot cpu, smp_prepare_boot_cpu is called after percpu area setup. 538 */ 539 set_my_cpu_offset(per_cpu_offset(cpu)); 540 541 cpu_proc_init(); 542 543 /* 544 * Define the placement constraint for the inline asm directive below. 545 * In Thumb-2, msr with an immediate value is not allowed. 546 */ 547 #ifdef CONFIG_THUMB2_KERNEL 548 #define PLC "r" 549 #else 550 #define PLC "I" 551 #endif 552 553 /* 554 * setup stacks for re-entrant exception handlers 555 */ 556 __asm__ ( 557 "msr cpsr_c, %1\n\t" 558 "add r14, %0, %2\n\t" 559 "mov sp, r14\n\t" 560 "msr cpsr_c, %3\n\t" 561 "add r14, %0, %4\n\t" 562 "mov sp, r14\n\t" 563 "msr cpsr_c, %5\n\t" 564 "add r14, %0, %6\n\t" 565 "mov sp, r14\n\t" 566 "msr cpsr_c, %7\n\t" 567 "add r14, %0, %8\n\t" 568 "mov sp, r14\n\t" 569 "msr cpsr_c, %9" 570 : 571 : "r" (stk), 572 PLC (PSR_F_BIT | PSR_I_BIT | IRQ_MODE), 573 "I" (offsetof(struct stack, irq[0])), 574 PLC (PSR_F_BIT | PSR_I_BIT | ABT_MODE), 575 "I" (offsetof(struct stack, abt[0])), 576 PLC (PSR_F_BIT | PSR_I_BIT | UND_MODE), 577 "I" (offsetof(struct stack, und[0])), 578 PLC (PSR_F_BIT | PSR_I_BIT | FIQ_MODE), 579 "I" (offsetof(struct stack, fiq[0])), 580 PLC (PSR_F_BIT | PSR_I_BIT | SVC_MODE) 581 : "r14"); 582 #endif 583 } 584 585 u32 __cpu_logical_map[NR_CPUS] = { [0 ... NR_CPUS-1] = MPIDR_INVALID }; 586 587 void __init smp_setup_processor_id(void) 588 { 589 int i; 590 u32 mpidr = is_smp() ? read_cpuid_mpidr() & MPIDR_HWID_BITMASK : 0; 591 u32 cpu = MPIDR_AFFINITY_LEVEL(mpidr, 0); 592 593 cpu_logical_map(0) = cpu; 594 for (i = 1; i < nr_cpu_ids; ++i) 595 cpu_logical_map(i) = i == cpu ? 0 : i; 596 597 /* 598 * clear __my_cpu_offset on boot CPU to avoid hang caused by 599 * using percpu variable early, for example, lockdep will 600 * access percpu variable inside lock_release 601 */ 602 set_my_cpu_offset(0); 603 604 pr_info("Booting Linux on physical CPU 0x%x\n", mpidr); 605 } 606 607 struct mpidr_hash mpidr_hash; 608 #ifdef CONFIG_SMP 609 /** 610 * smp_build_mpidr_hash - Pre-compute shifts required at each affinity 611 * level in order to build a linear index from an 612 * MPIDR value. Resulting algorithm is a collision 613 * free hash carried out through shifting and ORing 614 */ 615 static void __init smp_build_mpidr_hash(void) 616 { 617 u32 i, affinity; 618 u32 fs[3], bits[3], ls, mask = 0; 619 /* 620 * Pre-scan the list of MPIDRS and filter out bits that do 621 * not contribute to affinity levels, ie they never toggle. 622 */ 623 for_each_possible_cpu(i) 624 mask |= (cpu_logical_map(i) ^ cpu_logical_map(0)); 625 pr_debug("mask of set bits 0x%x\n", mask); 626 /* 627 * Find and stash the last and first bit set at all affinity levels to 628 * check how many bits are required to represent them. 629 */ 630 for (i = 0; i < 3; i++) { 631 affinity = MPIDR_AFFINITY_LEVEL(mask, i); 632 /* 633 * Find the MSB bit and LSB bits position 634 * to determine how many bits are required 635 * to express the affinity level. 636 */ 637 ls = fls(affinity); 638 fs[i] = affinity ? ffs(affinity) - 1 : 0; 639 bits[i] = ls - fs[i]; 640 } 641 /* 642 * An index can be created from the MPIDR by isolating the 643 * significant bits at each affinity level and by shifting 644 * them in order to compress the 24 bits values space to a 645 * compressed set of values. This is equivalent to hashing 646 * the MPIDR through shifting and ORing. It is a collision free 647 * hash though not minimal since some levels might contain a number 648 * of CPUs that is not an exact power of 2 and their bit 649 * representation might contain holes, eg MPIDR[7:0] = {0x2, 0x80}. 650 */ 651 mpidr_hash.shift_aff[0] = fs[0]; 652 mpidr_hash.shift_aff[1] = MPIDR_LEVEL_BITS + fs[1] - bits[0]; 653 mpidr_hash.shift_aff[2] = 2*MPIDR_LEVEL_BITS + fs[2] - 654 (bits[1] + bits[0]); 655 mpidr_hash.mask = mask; 656 mpidr_hash.bits = bits[2] + bits[1] + bits[0]; 657 pr_debug("MPIDR hash: aff0[%u] aff1[%u] aff2[%u] mask[0x%x] bits[%u]\n", 658 mpidr_hash.shift_aff[0], 659 mpidr_hash.shift_aff[1], 660 mpidr_hash.shift_aff[2], 661 mpidr_hash.mask, 662 mpidr_hash.bits); 663 /* 664 * 4x is an arbitrary value used to warn on a hash table much bigger 665 * than expected on most systems. 666 */ 667 if (mpidr_hash_size() > 4 * num_possible_cpus()) 668 pr_warn("Large number of MPIDR hash buckets detected\n"); 669 sync_cache_w(&mpidr_hash); 670 } 671 #endif 672 673 /* 674 * locate processor in the list of supported processor types. The linker 675 * builds this table for us from the entries in arch/arm/mm/proc-*.S 676 */ 677 struct proc_info_list *lookup_processor(u32 midr) 678 { 679 struct proc_info_list *list = lookup_processor_type(midr); 680 681 if (!list) { 682 pr_err("CPU%u: configuration botched (ID %08x), CPU halted\n", 683 smp_processor_id(), midr); 684 while (1) 685 /* can't use cpu_relax() here as it may require MMU setup */; 686 } 687 688 return list; 689 } 690 691 static void __init setup_processor(void) 692 { 693 unsigned int midr = read_cpuid_id(); 694 struct proc_info_list *list = lookup_processor(midr); 695 696 cpu_name = list->cpu_name; 697 __cpu_architecture = __get_cpu_architecture(); 698 699 init_proc_vtable(list->proc); 700 #ifdef MULTI_TLB 701 cpu_tlb = *list->tlb; 702 #endif 703 #ifdef MULTI_USER 704 cpu_user = *list->user; 705 #endif 706 #ifdef MULTI_CACHE 707 cpu_cache = *list->cache; 708 #endif 709 710 pr_info("CPU: %s [%08x] revision %d (ARMv%s), cr=%08lx\n", 711 list->cpu_name, midr, midr & 15, 712 proc_arch[cpu_architecture()], get_cr()); 713 714 snprintf(init_utsname()->machine, __NEW_UTS_LEN + 1, "%s%c", 715 list->arch_name, ENDIANNESS); 716 snprintf(elf_platform, ELF_PLATFORM_SIZE, "%s%c", 717 list->elf_name, ENDIANNESS); 718 elf_hwcap = list->elf_hwcap; 719 720 cpuid_init_hwcaps(); 721 patch_aeabi_idiv(); 722 723 #ifndef CONFIG_ARM_THUMB 724 elf_hwcap &= ~(HWCAP_THUMB | HWCAP_IDIVT); 725 #endif 726 #ifdef CONFIG_MMU 727 init_default_cache_policy(list->__cpu_mm_mmu_flags); 728 #endif 729 erratum_a15_798181_init(); 730 731 elf_hwcap_fixup(); 732 733 cacheid_init(); 734 cpu_init(); 735 } 736 737 void __init dump_machine_table(void) 738 { 739 const struct machine_desc *p; 740 741 early_print("Available machine support:\n\nID (hex)\tNAME\n"); 742 for_each_machine_desc(p) 743 early_print("%08x\t%s\n", p->nr, p->name); 744 745 early_print("\nPlease check your kernel config and/or bootloader.\n"); 746 747 while (true) 748 /* can't use cpu_relax() here as it may require MMU setup */; 749 } 750 751 int __init arm_add_memory(u64 start, u64 size) 752 { 753 u64 aligned_start; 754 755 /* 756 * Ensure that start/size are aligned to a page boundary. 757 * Size is rounded down, start is rounded up. 758 */ 759 aligned_start = PAGE_ALIGN(start); 760 if (aligned_start > start + size) 761 size = 0; 762 else 763 size -= aligned_start - start; 764 765 #ifndef CONFIG_PHYS_ADDR_T_64BIT 766 if (aligned_start > ULONG_MAX) { 767 pr_crit("Ignoring memory at 0x%08llx outside 32-bit physical address space\n", 768 start); 769 return -EINVAL; 770 } 771 772 if (aligned_start + size > ULONG_MAX) { 773 pr_crit("Truncating memory at 0x%08llx to fit in 32-bit physical address space\n", 774 (long long)start); 775 /* 776 * To ensure bank->start + bank->size is representable in 777 * 32 bits, we use ULONG_MAX as the upper limit rather than 4GB. 778 * This means we lose a page after masking. 779 */ 780 size = ULONG_MAX - aligned_start; 781 } 782 #endif 783 784 if (aligned_start < PHYS_OFFSET) { 785 if (aligned_start + size <= PHYS_OFFSET) { 786 pr_info("Ignoring memory below PHYS_OFFSET: 0x%08llx-0x%08llx\n", 787 aligned_start, aligned_start + size); 788 return -EINVAL; 789 } 790 791 pr_info("Ignoring memory below PHYS_OFFSET: 0x%08llx-0x%08llx\n", 792 aligned_start, (u64)PHYS_OFFSET); 793 794 size -= PHYS_OFFSET - aligned_start; 795 aligned_start = PHYS_OFFSET; 796 } 797 798 start = aligned_start; 799 size = size & ~(phys_addr_t)(PAGE_SIZE - 1); 800 801 /* 802 * Check whether this memory region has non-zero size or 803 * invalid node number. 804 */ 805 if (size == 0) 806 return -EINVAL; 807 808 memblock_add(start, size); 809 return 0; 810 } 811 812 /* 813 * Pick out the memory size. We look for mem=size@start, 814 * where start and size are "size[KkMm]" 815 */ 816 817 static int __init early_mem(char *p) 818 { 819 static int usermem __initdata = 0; 820 u64 size; 821 u64 start; 822 char *endp; 823 824 /* 825 * If the user specifies memory size, we 826 * blow away any automatically generated 827 * size. 828 */ 829 if (usermem == 0) { 830 usermem = 1; 831 memblock_remove(memblock_start_of_DRAM(), 832 memblock_end_of_DRAM() - memblock_start_of_DRAM()); 833 } 834 835 start = PHYS_OFFSET; 836 size = memparse(p, &endp); 837 if (*endp == '@') 838 start = memparse(endp + 1, NULL); 839 840 arm_add_memory(start, size); 841 842 return 0; 843 } 844 early_param("mem", early_mem); 845 846 static void __init request_standard_resources(const struct machine_desc *mdesc) 847 { 848 phys_addr_t start, end, res_end; 849 struct resource *res; 850 u64 i; 851 852 kernel_code.start = virt_to_phys(_text); 853 kernel_code.end = virt_to_phys(__init_begin - 1); 854 kernel_data.start = virt_to_phys(_sdata); 855 kernel_data.end = virt_to_phys(_end - 1); 856 857 for_each_mem_range(i, &start, &end) { 858 unsigned long boot_alias_start; 859 860 /* 861 * In memblock, end points to the first byte after the 862 * range while in resourses, end points to the last byte in 863 * the range. 864 */ 865 res_end = end - 1; 866 867 /* 868 * Some systems have a special memory alias which is only 869 * used for booting. We need to advertise this region to 870 * kexec-tools so they know where bootable RAM is located. 871 */ 872 boot_alias_start = phys_to_idmap(start); 873 if (arm_has_idmap_alias() && boot_alias_start != IDMAP_INVALID_ADDR) { 874 res = memblock_alloc(sizeof(*res), SMP_CACHE_BYTES); 875 if (!res) 876 panic("%s: Failed to allocate %zu bytes\n", 877 __func__, sizeof(*res)); 878 res->name = "System RAM (boot alias)"; 879 res->start = boot_alias_start; 880 res->end = phys_to_idmap(res_end); 881 res->flags = IORESOURCE_MEM | IORESOURCE_BUSY; 882 request_resource(&iomem_resource, res); 883 } 884 885 res = memblock_alloc(sizeof(*res), SMP_CACHE_BYTES); 886 if (!res) 887 panic("%s: Failed to allocate %zu bytes\n", __func__, 888 sizeof(*res)); 889 res->name = "System RAM"; 890 res->start = start; 891 res->end = res_end; 892 res->flags = IORESOURCE_SYSTEM_RAM | IORESOURCE_BUSY; 893 894 request_resource(&iomem_resource, res); 895 896 if (kernel_code.start >= res->start && 897 kernel_code.end <= res->end) 898 request_resource(res, &kernel_code); 899 if (kernel_data.start >= res->start && 900 kernel_data.end <= res->end) 901 request_resource(res, &kernel_data); 902 } 903 904 if (mdesc->video_start) { 905 video_ram.start = mdesc->video_start; 906 video_ram.end = mdesc->video_end; 907 request_resource(&iomem_resource, &video_ram); 908 } 909 910 /* 911 * Some machines don't have the possibility of ever 912 * possessing lp0, lp1 or lp2 913 */ 914 if (mdesc->reserve_lp0) 915 request_resource(&ioport_resource, &lp0); 916 if (mdesc->reserve_lp1) 917 request_resource(&ioport_resource, &lp1); 918 if (mdesc->reserve_lp2) 919 request_resource(&ioport_resource, &lp2); 920 } 921 922 #if defined(CONFIG_VGA_CONSOLE) || defined(CONFIG_DUMMY_CONSOLE) || \ 923 defined(CONFIG_EFI) 924 struct screen_info screen_info = { 925 .orig_video_lines = 30, 926 .orig_video_cols = 80, 927 .orig_video_mode = 0, 928 .orig_video_ega_bx = 0, 929 .orig_video_isVGA = 1, 930 .orig_video_points = 8 931 }; 932 #endif 933 934 static int __init customize_machine(void) 935 { 936 /* 937 * customizes platform devices, or adds new ones 938 * On DT based machines, we fall back to populating the 939 * machine from the device tree, if no callback is provided, 940 * otherwise we would always need an init_machine callback. 941 */ 942 if (machine_desc->init_machine) 943 machine_desc->init_machine(); 944 945 return 0; 946 } 947 arch_initcall(customize_machine); 948 949 static int __init init_machine_late(void) 950 { 951 struct device_node *root; 952 int ret; 953 954 if (machine_desc->init_late) 955 machine_desc->init_late(); 956 957 root = of_find_node_by_path("/"); 958 if (root) { 959 ret = of_property_read_string(root, "serial-number", 960 &system_serial); 961 if (ret) 962 system_serial = NULL; 963 } 964 965 if (!system_serial) 966 system_serial = kasprintf(GFP_KERNEL, "%08x%08x", 967 system_serial_high, 968 system_serial_low); 969 970 return 0; 971 } 972 late_initcall(init_machine_late); 973 974 #ifdef CONFIG_KEXEC 975 /* 976 * The crash region must be aligned to 128MB to avoid 977 * zImage relocating below the reserved region. 978 */ 979 #define CRASH_ALIGN (128 << 20) 980 981 static inline unsigned long long get_total_mem(void) 982 { 983 unsigned long total; 984 985 total = max_low_pfn - min_low_pfn; 986 return total << PAGE_SHIFT; 987 } 988 989 /** 990 * reserve_crashkernel() - reserves memory are for crash kernel 991 * 992 * This function reserves memory area given in "crashkernel=" kernel command 993 * line parameter. The memory reserved is used by a dump capture kernel when 994 * primary kernel is crashing. 995 */ 996 static void __init reserve_crashkernel(void) 997 { 998 unsigned long long crash_size, crash_base; 999 unsigned long long total_mem; 1000 int ret; 1001 1002 total_mem = get_total_mem(); 1003 ret = parse_crashkernel(boot_command_line, total_mem, 1004 &crash_size, &crash_base); 1005 if (ret) 1006 return; 1007 1008 if (crash_base <= 0) { 1009 unsigned long long crash_max = idmap_to_phys((u32)~0); 1010 unsigned long long lowmem_max = __pa(high_memory - 1) + 1; 1011 if (crash_max > lowmem_max) 1012 crash_max = lowmem_max; 1013 crash_base = memblock_find_in_range(CRASH_ALIGN, crash_max, 1014 crash_size, CRASH_ALIGN); 1015 if (!crash_base) { 1016 pr_err("crashkernel reservation failed - No suitable area found.\n"); 1017 return; 1018 } 1019 } else { 1020 unsigned long long start; 1021 1022 start = memblock_find_in_range(crash_base, 1023 crash_base + crash_size, 1024 crash_size, SECTION_SIZE); 1025 if (start != crash_base) { 1026 pr_err("crashkernel reservation failed - memory is in use.\n"); 1027 return; 1028 } 1029 } 1030 1031 ret = memblock_reserve(crash_base, crash_size); 1032 if (ret < 0) { 1033 pr_warn("crashkernel reservation failed - memory is in use (0x%lx)\n", 1034 (unsigned long)crash_base); 1035 return; 1036 } 1037 1038 pr_info("Reserving %ldMB of memory at %ldMB for crashkernel (System RAM: %ldMB)\n", 1039 (unsigned long)(crash_size >> 20), 1040 (unsigned long)(crash_base >> 20), 1041 (unsigned long)(total_mem >> 20)); 1042 1043 /* The crashk resource must always be located in normal mem */ 1044 crashk_res.start = crash_base; 1045 crashk_res.end = crash_base + crash_size - 1; 1046 insert_resource(&iomem_resource, &crashk_res); 1047 1048 if (arm_has_idmap_alias()) { 1049 /* 1050 * If we have a special RAM alias for use at boot, we 1051 * need to advertise to kexec tools where the alias is. 1052 */ 1053 static struct resource crashk_boot_res = { 1054 .name = "Crash kernel (boot alias)", 1055 .flags = IORESOURCE_BUSY | IORESOURCE_MEM, 1056 }; 1057 1058 crashk_boot_res.start = phys_to_idmap(crash_base); 1059 crashk_boot_res.end = crashk_boot_res.start + crash_size - 1; 1060 insert_resource(&iomem_resource, &crashk_boot_res); 1061 } 1062 } 1063 #else 1064 static inline void reserve_crashkernel(void) {} 1065 #endif /* CONFIG_KEXEC */ 1066 1067 void __init hyp_mode_check(void) 1068 { 1069 #ifdef CONFIG_ARM_VIRT_EXT 1070 sync_boot_mode(); 1071 1072 if (is_hyp_mode_available()) { 1073 pr_info("CPU: All CPU(s) started in HYP mode.\n"); 1074 pr_info("CPU: Virtualization extensions available.\n"); 1075 } else if (is_hyp_mode_mismatched()) { 1076 pr_warn("CPU: WARNING: CPU(s) started in wrong/inconsistent modes (primary CPU mode 0x%x)\n", 1077 __boot_cpu_mode & MODE_MASK); 1078 pr_warn("CPU: This may indicate a broken bootloader or firmware.\n"); 1079 } else 1080 pr_info("CPU: All CPU(s) started in SVC mode.\n"); 1081 #endif 1082 } 1083 1084 void __init setup_arch(char **cmdline_p) 1085 { 1086 const struct machine_desc *mdesc = NULL; 1087 void *atags_vaddr = NULL; 1088 1089 if (__atags_pointer) 1090 atags_vaddr = FDT_VIRT_BASE(__atags_pointer); 1091 1092 setup_processor(); 1093 if (atags_vaddr) { 1094 mdesc = setup_machine_fdt(atags_vaddr); 1095 if (mdesc) 1096 memblock_reserve(__atags_pointer, 1097 fdt_totalsize(atags_vaddr)); 1098 } 1099 if (!mdesc) 1100 mdesc = setup_machine_tags(atags_vaddr, __machine_arch_type); 1101 if (!mdesc) { 1102 early_print("\nError: invalid dtb and unrecognized/unsupported machine ID\n"); 1103 early_print(" r1=0x%08x, r2=0x%08x\n", __machine_arch_type, 1104 __atags_pointer); 1105 if (__atags_pointer) 1106 early_print(" r2[]=%*ph\n", 16, atags_vaddr); 1107 dump_machine_table(); 1108 } 1109 1110 machine_desc = mdesc; 1111 machine_name = mdesc->name; 1112 dump_stack_set_arch_desc("%s", mdesc->name); 1113 1114 if (mdesc->reboot_mode != REBOOT_HARD) 1115 reboot_mode = mdesc->reboot_mode; 1116 1117 init_mm.start_code = (unsigned long) _text; 1118 init_mm.end_code = (unsigned long) _etext; 1119 init_mm.end_data = (unsigned long) _edata; 1120 init_mm.brk = (unsigned long) _end; 1121 1122 /* populate cmd_line too for later use, preserving boot_command_line */ 1123 strlcpy(cmd_line, boot_command_line, COMMAND_LINE_SIZE); 1124 *cmdline_p = cmd_line; 1125 1126 early_fixmap_init(); 1127 early_ioremap_init(); 1128 1129 parse_early_param(); 1130 1131 #ifdef CONFIG_MMU 1132 early_mm_init(mdesc); 1133 #endif 1134 setup_dma_zone(mdesc); 1135 xen_early_init(); 1136 efi_init(); 1137 /* 1138 * Make sure the calculation for lowmem/highmem is set appropriately 1139 * before reserving/allocating any memory 1140 */ 1141 adjust_lowmem_bounds(); 1142 arm_memblock_init(mdesc); 1143 /* Memory may have been removed so recalculate the bounds. */ 1144 adjust_lowmem_bounds(); 1145 1146 early_ioremap_reset(); 1147 1148 paging_init(mdesc); 1149 kasan_init(); 1150 request_standard_resources(mdesc); 1151 1152 if (mdesc->restart) 1153 arm_pm_restart = mdesc->restart; 1154 1155 unflatten_device_tree(); 1156 1157 arm_dt_init_cpu_maps(); 1158 psci_dt_init(); 1159 #ifdef CONFIG_SMP 1160 if (is_smp()) { 1161 if (!mdesc->smp_init || !mdesc->smp_init()) { 1162 if (psci_smp_available()) 1163 smp_set_ops(&psci_smp_ops); 1164 else if (mdesc->smp) 1165 smp_set_ops(mdesc->smp); 1166 } 1167 smp_init_cpus(); 1168 smp_build_mpidr_hash(); 1169 } 1170 #endif 1171 1172 if (!is_smp()) 1173 hyp_mode_check(); 1174 1175 reserve_crashkernel(); 1176 1177 #ifdef CONFIG_GENERIC_IRQ_MULTI_HANDLER 1178 handle_arch_irq = mdesc->handle_irq; 1179 #endif 1180 1181 #ifdef CONFIG_VT 1182 #if defined(CONFIG_VGA_CONSOLE) 1183 conswitchp = &vga_con; 1184 #endif 1185 #endif 1186 1187 if (mdesc->init_early) 1188 mdesc->init_early(); 1189 } 1190 1191 1192 static int __init topology_init(void) 1193 { 1194 int cpu; 1195 1196 for_each_possible_cpu(cpu) { 1197 struct cpuinfo_arm *cpuinfo = &per_cpu(cpu_data, cpu); 1198 cpuinfo->cpu.hotpluggable = platform_can_hotplug_cpu(cpu); 1199 register_cpu(&cpuinfo->cpu, cpu); 1200 } 1201 1202 return 0; 1203 } 1204 subsys_initcall(topology_init); 1205 1206 #ifdef CONFIG_HAVE_PROC_CPU 1207 static int __init proc_cpu_init(void) 1208 { 1209 struct proc_dir_entry *res; 1210 1211 res = proc_mkdir("cpu", NULL); 1212 if (!res) 1213 return -ENOMEM; 1214 return 0; 1215 } 1216 fs_initcall(proc_cpu_init); 1217 #endif 1218 1219 static const char *hwcap_str[] = { 1220 "swp", 1221 "half", 1222 "thumb", 1223 "26bit", 1224 "fastmult", 1225 "fpa", 1226 "vfp", 1227 "edsp", 1228 "java", 1229 "iwmmxt", 1230 "crunch", 1231 "thumbee", 1232 "neon", 1233 "vfpv3", 1234 "vfpv3d16", 1235 "tls", 1236 "vfpv4", 1237 "idiva", 1238 "idivt", 1239 "vfpd32", 1240 "lpae", 1241 "evtstrm", 1242 NULL 1243 }; 1244 1245 static const char *hwcap2_str[] = { 1246 "aes", 1247 "pmull", 1248 "sha1", 1249 "sha2", 1250 "crc32", 1251 NULL 1252 }; 1253 1254 static int c_show(struct seq_file *m, void *v) 1255 { 1256 int i, j; 1257 u32 cpuid; 1258 1259 for_each_online_cpu(i) { 1260 /* 1261 * glibc reads /proc/cpuinfo to determine the number of 1262 * online processors, looking for lines beginning with 1263 * "processor". Give glibc what it expects. 1264 */ 1265 seq_printf(m, "processor\t: %d\n", i); 1266 cpuid = is_smp() ? per_cpu(cpu_data, i).cpuid : read_cpuid_id(); 1267 seq_printf(m, "model name\t: %s rev %d (%s)\n", 1268 cpu_name, cpuid & 15, elf_platform); 1269 1270 #if defined(CONFIG_SMP) 1271 seq_printf(m, "BogoMIPS\t: %lu.%02lu\n", 1272 per_cpu(cpu_data, i).loops_per_jiffy / (500000UL/HZ), 1273 (per_cpu(cpu_data, i).loops_per_jiffy / (5000UL/HZ)) % 100); 1274 #else 1275 seq_printf(m, "BogoMIPS\t: %lu.%02lu\n", 1276 loops_per_jiffy / (500000/HZ), 1277 (loops_per_jiffy / (5000/HZ)) % 100); 1278 #endif 1279 /* dump out the processor features */ 1280 seq_puts(m, "Features\t: "); 1281 1282 for (j = 0; hwcap_str[j]; j++) 1283 if (elf_hwcap & (1 << j)) 1284 seq_printf(m, "%s ", hwcap_str[j]); 1285 1286 for (j = 0; hwcap2_str[j]; j++) 1287 if (elf_hwcap2 & (1 << j)) 1288 seq_printf(m, "%s ", hwcap2_str[j]); 1289 1290 seq_printf(m, "\nCPU implementer\t: 0x%02x\n", cpuid >> 24); 1291 seq_printf(m, "CPU architecture: %s\n", 1292 proc_arch[cpu_architecture()]); 1293 1294 if ((cpuid & 0x0008f000) == 0x00000000) { 1295 /* pre-ARM7 */ 1296 seq_printf(m, "CPU part\t: %07x\n", cpuid >> 4); 1297 } else { 1298 if ((cpuid & 0x0008f000) == 0x00007000) { 1299 /* ARM7 */ 1300 seq_printf(m, "CPU variant\t: 0x%02x\n", 1301 (cpuid >> 16) & 127); 1302 } else { 1303 /* post-ARM7 */ 1304 seq_printf(m, "CPU variant\t: 0x%x\n", 1305 (cpuid >> 20) & 15); 1306 } 1307 seq_printf(m, "CPU part\t: 0x%03x\n", 1308 (cpuid >> 4) & 0xfff); 1309 } 1310 seq_printf(m, "CPU revision\t: %d\n\n", cpuid & 15); 1311 } 1312 1313 seq_printf(m, "Hardware\t: %s\n", machine_name); 1314 seq_printf(m, "Revision\t: %04x\n", system_rev); 1315 seq_printf(m, "Serial\t\t: %s\n", system_serial); 1316 1317 return 0; 1318 } 1319 1320 static void *c_start(struct seq_file *m, loff_t *pos) 1321 { 1322 return *pos < 1 ? (void *)1 : NULL; 1323 } 1324 1325 static void *c_next(struct seq_file *m, void *v, loff_t *pos) 1326 { 1327 ++*pos; 1328 return NULL; 1329 } 1330 1331 static void c_stop(struct seq_file *m, void *v) 1332 { 1333 } 1334 1335 const struct seq_operations cpuinfo_op = { 1336 .start = c_start, 1337 .next = c_next, 1338 .stop = c_stop, 1339 .show = c_show 1340 }; 1341