1/* 2 * Low-level CPU initialisation 3 * Based on arch/arm/kernel/head.S 4 * 5 * Copyright (C) 1994-2002 Russell King 6 * Copyright (C) 2003-2012 ARM Ltd. 7 * Authors: Catalin Marinas <catalin.marinas@arm.com> 8 * Will Deacon <will.deacon@arm.com> 9 * 10 * This program is free software; you can redistribute it and/or modify 11 * it under the terms of the GNU General Public License version 2 as 12 * published by the Free Software Foundation. 13 * 14 * This program is distributed in the hope that it will be useful, 15 * but WITHOUT ANY WARRANTY; without even the implied warranty of 16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 17 * GNU General Public License for more details. 18 * 19 * You should have received a copy of the GNU General Public License 20 * along with this program. If not, see <http://www.gnu.org/licenses/>. 21 */ 22 23#include <linux/linkage.h> 24#include <linux/init.h> 25#include <linux/irqchip/arm-gic-v3.h> 26 27#include <asm/assembler.h> 28#include <asm/ptrace.h> 29#include <asm/asm-offsets.h> 30#include <asm/cache.h> 31#include <asm/cputype.h> 32#include <asm/memory.h> 33#include <asm/thread_info.h> 34#include <asm/pgtable-hwdef.h> 35#include <asm/pgtable.h> 36#include <asm/page.h> 37#include <asm/virt.h> 38 39#define KERNEL_RAM_VADDR (PAGE_OFFSET + TEXT_OFFSET) 40 41#if (TEXT_OFFSET & 0xfff) != 0 42#error TEXT_OFFSET must be at least 4KB aligned 43#elif (PAGE_OFFSET & 0x1fffff) != 0 44#error PAGE_OFFSET must be at least 2MB aligned 45#elif TEXT_OFFSET > 0x1fffff 46#error TEXT_OFFSET must be less than 2MB 47#endif 48 49 .macro pgtbl, ttb0, ttb1, virt_to_phys 50 ldr \ttb1, =swapper_pg_dir 51 ldr \ttb0, =idmap_pg_dir 52 add \ttb1, \ttb1, \virt_to_phys 53 add \ttb0, \ttb0, \virt_to_phys 54 .endm 55 56#ifdef CONFIG_ARM64_64K_PAGES 57#define BLOCK_SHIFT PAGE_SHIFT 58#define BLOCK_SIZE PAGE_SIZE 59#define TABLE_SHIFT PMD_SHIFT 60#else 61#define BLOCK_SHIFT SECTION_SHIFT 62#define BLOCK_SIZE SECTION_SIZE 63#define TABLE_SHIFT PUD_SHIFT 64#endif 65 66#define KERNEL_START KERNEL_RAM_VADDR 67#define KERNEL_END _end 68 69/* 70 * Initial memory map attributes. 71 */ 72#ifndef CONFIG_SMP 73#define PTE_FLAGS PTE_TYPE_PAGE | PTE_AF 74#define PMD_FLAGS PMD_TYPE_SECT | PMD_SECT_AF 75#else 76#define PTE_FLAGS PTE_TYPE_PAGE | PTE_AF | PTE_SHARED 77#define PMD_FLAGS PMD_TYPE_SECT | PMD_SECT_AF | PMD_SECT_S 78#endif 79 80#ifdef CONFIG_ARM64_64K_PAGES 81#define MM_MMUFLAGS PTE_ATTRINDX(MT_NORMAL) | PTE_FLAGS 82#else 83#define MM_MMUFLAGS PMD_ATTRINDX(MT_NORMAL) | PMD_FLAGS 84#endif 85 86/* 87 * Kernel startup entry point. 88 * --------------------------- 89 * 90 * The requirements are: 91 * MMU = off, D-cache = off, I-cache = on or off, 92 * x0 = physical address to the FDT blob. 93 * 94 * This code is mostly position independent so you call this at 95 * __pa(PAGE_OFFSET + TEXT_OFFSET). 96 * 97 * Note that the callee-saved registers are used for storing variables 98 * that are useful before the MMU is enabled. The allocations are described 99 * in the entry routines. 100 */ 101 __HEAD 102 103 /* 104 * DO NOT MODIFY. Image header expected by Linux boot-loaders. 105 */ 106#ifdef CONFIG_EFI 107efi_head: 108 /* 109 * This add instruction has no meaningful effect except that 110 * its opcode forms the magic "MZ" signature required by UEFI. 111 */ 112 add x13, x18, #0x16 113 b stext 114#else 115 b stext // branch to kernel start, magic 116 .long 0 // reserved 117#endif 118 .quad _kernel_offset_le // Image load offset from start of RAM, little-endian 119 .quad _kernel_size_le // Effective size of kernel image, little-endian 120 .quad _kernel_flags_le // Informative flags, little-endian 121 .quad 0 // reserved 122 .quad 0 // reserved 123 .quad 0 // reserved 124 .byte 0x41 // Magic number, "ARM\x64" 125 .byte 0x52 126 .byte 0x4d 127 .byte 0x64 128#ifdef CONFIG_EFI 129 .long pe_header - efi_head // Offset to the PE header. 130#else 131 .word 0 // reserved 132#endif 133 134#ifdef CONFIG_EFI 135 .globl stext_offset 136 .set stext_offset, stext - efi_head 137 .align 3 138pe_header: 139 .ascii "PE" 140 .short 0 141coff_header: 142 .short 0xaa64 // AArch64 143 .short 2 // nr_sections 144 .long 0 // TimeDateStamp 145 .long 0 // PointerToSymbolTable 146 .long 1 // NumberOfSymbols 147 .short section_table - optional_header // SizeOfOptionalHeader 148 .short 0x206 // Characteristics. 149 // IMAGE_FILE_DEBUG_STRIPPED | 150 // IMAGE_FILE_EXECUTABLE_IMAGE | 151 // IMAGE_FILE_LINE_NUMS_STRIPPED 152optional_header: 153 .short 0x20b // PE32+ format 154 .byte 0x02 // MajorLinkerVersion 155 .byte 0x14 // MinorLinkerVersion 156 .long _end - stext // SizeOfCode 157 .long 0 // SizeOfInitializedData 158 .long 0 // SizeOfUninitializedData 159 .long efi_stub_entry - efi_head // AddressOfEntryPoint 160 .long stext_offset // BaseOfCode 161 162extra_header_fields: 163 .quad 0 // ImageBase 164 .long 0x1000 // SectionAlignment 165 .long PECOFF_FILE_ALIGNMENT // FileAlignment 166 .short 0 // MajorOperatingSystemVersion 167 .short 0 // MinorOperatingSystemVersion 168 .short 0 // MajorImageVersion 169 .short 0 // MinorImageVersion 170 .short 0 // MajorSubsystemVersion 171 .short 0 // MinorSubsystemVersion 172 .long 0 // Win32VersionValue 173 174 .long _end - efi_head // SizeOfImage 175 176 // Everything before the kernel image is considered part of the header 177 .long stext_offset // SizeOfHeaders 178 .long 0 // CheckSum 179 .short 0xa // Subsystem (EFI application) 180 .short 0 // DllCharacteristics 181 .quad 0 // SizeOfStackReserve 182 .quad 0 // SizeOfStackCommit 183 .quad 0 // SizeOfHeapReserve 184 .quad 0 // SizeOfHeapCommit 185 .long 0 // LoaderFlags 186 .long 0x6 // NumberOfRvaAndSizes 187 188 .quad 0 // ExportTable 189 .quad 0 // ImportTable 190 .quad 0 // ResourceTable 191 .quad 0 // ExceptionTable 192 .quad 0 // CertificationTable 193 .quad 0 // BaseRelocationTable 194 195 // Section table 196section_table: 197 198 /* 199 * The EFI application loader requires a relocation section 200 * because EFI applications must be relocatable. This is a 201 * dummy section as far as we are concerned. 202 */ 203 .ascii ".reloc" 204 .byte 0 205 .byte 0 // end of 0 padding of section name 206 .long 0 207 .long 0 208 .long 0 // SizeOfRawData 209 .long 0 // PointerToRawData 210 .long 0 // PointerToRelocations 211 .long 0 // PointerToLineNumbers 212 .short 0 // NumberOfRelocations 213 .short 0 // NumberOfLineNumbers 214 .long 0x42100040 // Characteristics (section flags) 215 216 217 .ascii ".text" 218 .byte 0 219 .byte 0 220 .byte 0 // end of 0 padding of section name 221 .long _end - stext // VirtualSize 222 .long stext_offset // VirtualAddress 223 .long _edata - stext // SizeOfRawData 224 .long stext_offset // PointerToRawData 225 226 .long 0 // PointerToRelocations (0 for executables) 227 .long 0 // PointerToLineNumbers (0 for executables) 228 .short 0 // NumberOfRelocations (0 for executables) 229 .short 0 // NumberOfLineNumbers (0 for executables) 230 .long 0xe0500020 // Characteristics (section flags) 231 232 /* 233 * EFI will load stext onwards at the 4k section alignment 234 * described in the PE/COFF header. To ensure that instruction 235 * sequences using an adrp and a :lo12: immediate will function 236 * correctly at this alignment, we must ensure that stext is 237 * placed at a 4k boundary in the Image to begin with. 238 */ 239 .align 12 240#endif 241 242ENTRY(stext) 243 mov x21, x0 // x21=FDT 244 bl el2_setup // Drop to EL1, w20=cpu_boot_mode 245 bl __calc_phys_offset // x24=PHYS_OFFSET, x28=PHYS_OFFSET-PAGE_OFFSET 246 bl set_cpu_boot_mode_flag 247 mrs x22, midr_el1 // x22=cpuid 248 mov x0, x22 249 bl lookup_processor_type 250 mov x23, x0 // x23=current cpu_table 251 /* 252 * __error_p may end up out of range for cbz if text areas are 253 * aligned up to section sizes. 254 */ 255 cbnz x23, 1f // invalid processor (x23=0)? 256 b __error_p 2571: 258 bl __vet_fdt 259 bl __create_page_tables // x25=TTBR0, x26=TTBR1 260 /* 261 * The following calls CPU specific code in a position independent 262 * manner. See arch/arm64/mm/proc.S for details. x23 = base of 263 * cpu_info structure selected by lookup_processor_type above. 264 * On return, the CPU will be ready for the MMU to be turned on and 265 * the TCR will have been set. 266 */ 267 ldr x27, __switch_data // address to jump to after 268 // MMU has been enabled 269 adrp lr, __enable_mmu // return (PIC) address 270 add lr, lr, #:lo12:__enable_mmu 271 ldr x12, [x23, #CPU_INFO_SETUP] 272 add x12, x12, x28 // __virt_to_phys 273 br x12 // initialise processor 274ENDPROC(stext) 275 276/* 277 * Determine validity of the x21 FDT pointer. 278 * The dtb must be 8-byte aligned and live in the first 512M of memory. 279 */ 280__vet_fdt: 281 tst x21, #0x7 282 b.ne 1f 283 cmp x21, x24 284 b.lt 1f 285 mov x0, #(1 << 29) 286 add x0, x0, x24 287 cmp x21, x0 288 b.ge 1f 289 ret 2901: 291 mov x21, #0 292 ret 293ENDPROC(__vet_fdt) 294/* 295 * Macro to create a table entry to the next page. 296 * 297 * tbl: page table address 298 * virt: virtual address 299 * shift: #imm page table shift 300 * ptrs: #imm pointers per table page 301 * 302 * Preserves: virt 303 * Corrupts: tmp1, tmp2 304 * Returns: tbl -> next level table page address 305 */ 306 .macro create_table_entry, tbl, virt, shift, ptrs, tmp1, tmp2 307 lsr \tmp1, \virt, #\shift 308 and \tmp1, \tmp1, #\ptrs - 1 // table index 309 add \tmp2, \tbl, #PAGE_SIZE 310 orr \tmp2, \tmp2, #PMD_TYPE_TABLE // address of next table and entry type 311 str \tmp2, [\tbl, \tmp1, lsl #3] 312 add \tbl, \tbl, #PAGE_SIZE // next level table page 313 .endm 314 315/* 316 * Macro to populate the PGD (and possibily PUD) for the corresponding 317 * block entry in the next level (tbl) for the given virtual address. 318 * 319 * Preserves: tbl, next, virt 320 * Corrupts: tmp1, tmp2 321 */ 322 .macro create_pgd_entry, tbl, virt, tmp1, tmp2 323 create_table_entry \tbl, \virt, PGDIR_SHIFT, PTRS_PER_PGD, \tmp1, \tmp2 324#if SWAPPER_PGTABLE_LEVELS == 3 325 create_table_entry \tbl, \virt, TABLE_SHIFT, PTRS_PER_PTE, \tmp1, \tmp2 326#endif 327 .endm 328 329/* 330 * Macro to populate block entries in the page table for the start..end 331 * virtual range (inclusive). 332 * 333 * Preserves: tbl, flags 334 * Corrupts: phys, start, end, pstate 335 */ 336 .macro create_block_map, tbl, flags, phys, start, end 337 lsr \phys, \phys, #BLOCK_SHIFT 338 lsr \start, \start, #BLOCK_SHIFT 339 and \start, \start, #PTRS_PER_PTE - 1 // table index 340 orr \phys, \flags, \phys, lsl #BLOCK_SHIFT // table entry 341 lsr \end, \end, #BLOCK_SHIFT 342 and \end, \end, #PTRS_PER_PTE - 1 // table end index 3439999: str \phys, [\tbl, \start, lsl #3] // store the entry 344 add \start, \start, #1 // next entry 345 add \phys, \phys, #BLOCK_SIZE // next block 346 cmp \start, \end 347 b.ls 9999b 348 .endm 349 350/* 351 * Setup the initial page tables. We only setup the barest amount which is 352 * required to get the kernel running. The following sections are required: 353 * - identity mapping to enable the MMU (low address, TTBR0) 354 * - first few MB of the kernel linear mapping to jump to once the MMU has 355 * been enabled, including the FDT blob (TTBR1) 356 * - pgd entry for fixed mappings (TTBR1) 357 */ 358__create_page_tables: 359 pgtbl x25, x26, x28 // idmap_pg_dir and swapper_pg_dir addresses 360 mov x27, lr 361 362 /* 363 * Invalidate the idmap and swapper page tables to avoid potential 364 * dirty cache lines being evicted. 365 */ 366 mov x0, x25 367 add x1, x26, #SWAPPER_DIR_SIZE 368 bl __inval_cache_range 369 370 /* 371 * Clear the idmap and swapper page tables. 372 */ 373 mov x0, x25 374 add x6, x26, #SWAPPER_DIR_SIZE 3751: stp xzr, xzr, [x0], #16 376 stp xzr, xzr, [x0], #16 377 stp xzr, xzr, [x0], #16 378 stp xzr, xzr, [x0], #16 379 cmp x0, x6 380 b.lo 1b 381 382 ldr x7, =MM_MMUFLAGS 383 384 /* 385 * Create the identity mapping. 386 */ 387 mov x0, x25 // idmap_pg_dir 388 ldr x3, =KERNEL_START 389 add x3, x3, x28 // __pa(KERNEL_START) 390 create_pgd_entry x0, x3, x5, x6 391 ldr x6, =KERNEL_END 392 mov x5, x3 // __pa(KERNEL_START) 393 add x6, x6, x28 // __pa(KERNEL_END) 394 create_block_map x0, x7, x3, x5, x6 395 396 /* 397 * Map the kernel image (starting with PHYS_OFFSET). 398 */ 399 mov x0, x26 // swapper_pg_dir 400 mov x5, #PAGE_OFFSET 401 create_pgd_entry x0, x5, x3, x6 402 ldr x6, =KERNEL_END 403 mov x3, x24 // phys offset 404 create_block_map x0, x7, x3, x5, x6 405 406 /* 407 * Map the FDT blob (maximum 2MB; must be within 512MB of 408 * PHYS_OFFSET). 409 */ 410 mov x3, x21 // FDT phys address 411 and x3, x3, #~((1 << 21) - 1) // 2MB aligned 412 mov x6, #PAGE_OFFSET 413 sub x5, x3, x24 // subtract PHYS_OFFSET 414 tst x5, #~((1 << 29) - 1) // within 512MB? 415 csel x21, xzr, x21, ne // zero the FDT pointer 416 b.ne 1f 417 add x5, x5, x6 // __va(FDT blob) 418 add x6, x5, #1 << 21 // 2MB for the FDT blob 419 sub x6, x6, #1 // inclusive range 420 create_block_map x0, x7, x3, x5, x6 4211: 422 /* 423 * Since the page tables have been populated with non-cacheable 424 * accesses (MMU disabled), invalidate the idmap and swapper page 425 * tables again to remove any speculatively loaded cache lines. 426 */ 427 mov x0, x25 428 add x1, x26, #SWAPPER_DIR_SIZE 429 bl __inval_cache_range 430 431 mov lr, x27 432 ret 433ENDPROC(__create_page_tables) 434 .ltorg 435 436 .align 3 437 .type __switch_data, %object 438__switch_data: 439 .quad __mmap_switched 440 .quad __bss_start // x6 441 .quad __bss_stop // x7 442 .quad processor_id // x4 443 .quad __fdt_pointer // x5 444 .quad memstart_addr // x6 445 .quad init_thread_union + THREAD_START_SP // sp 446 447/* 448 * The following fragment of code is executed with the MMU on in MMU mode, and 449 * uses absolute addresses; this is not position independent. 450 */ 451__mmap_switched: 452 adr x3, __switch_data + 8 453 454 ldp x6, x7, [x3], #16 4551: cmp x6, x7 456 b.hs 2f 457 str xzr, [x6], #8 // Clear BSS 458 b 1b 4592: 460 ldp x4, x5, [x3], #16 461 ldr x6, [x3], #8 462 ldr x16, [x3] 463 mov sp, x16 464 str x22, [x4] // Save processor ID 465 str x21, [x5] // Save FDT pointer 466 str x24, [x6] // Save PHYS_OFFSET 467 mov x29, #0 468 b start_kernel 469ENDPROC(__mmap_switched) 470 471/* 472 * end early head section, begin head code that is also used for 473 * hotplug and needs to have the same protections as the text region 474 */ 475 .section ".text","ax" 476/* 477 * If we're fortunate enough to boot at EL2, ensure that the world is 478 * sane before dropping to EL1. 479 * 480 * Returns either BOOT_CPU_MODE_EL1 or BOOT_CPU_MODE_EL2 in x20 if 481 * booted in EL1 or EL2 respectively. 482 */ 483ENTRY(el2_setup) 484 mrs x0, CurrentEL 485 cmp x0, #CurrentEL_EL2 486 b.ne 1f 487 mrs x0, sctlr_el2 488CPU_BE( orr x0, x0, #(1 << 25) ) // Set the EE bit for EL2 489CPU_LE( bic x0, x0, #(1 << 25) ) // Clear the EE bit for EL2 490 msr sctlr_el2, x0 491 b 2f 4921: mrs x0, sctlr_el1 493CPU_BE( orr x0, x0, #(3 << 24) ) // Set the EE and E0E bits for EL1 494CPU_LE( bic x0, x0, #(3 << 24) ) // Clear the EE and E0E bits for EL1 495 msr sctlr_el1, x0 496 mov w20, #BOOT_CPU_MODE_EL1 // This cpu booted in EL1 497 isb 498 ret 499 500 /* Hyp configuration. */ 5012: mov x0, #(1 << 31) // 64-bit EL1 502 msr hcr_el2, x0 503 504 /* Generic timers. */ 505 mrs x0, cnthctl_el2 506 orr x0, x0, #3 // Enable EL1 physical timers 507 msr cnthctl_el2, x0 508 msr cntvoff_el2, xzr // Clear virtual offset 509 510#ifdef CONFIG_ARM_GIC_V3 511 /* GICv3 system register access */ 512 mrs x0, id_aa64pfr0_el1 513 ubfx x0, x0, #24, #4 514 cmp x0, #1 515 b.ne 3f 516 517 mrs_s x0, ICC_SRE_EL2 518 orr x0, x0, #ICC_SRE_EL2_SRE // Set ICC_SRE_EL2.SRE==1 519 orr x0, x0, #ICC_SRE_EL2_ENABLE // Set ICC_SRE_EL2.Enable==1 520 msr_s ICC_SRE_EL2, x0 521 isb // Make sure SRE is now set 522 msr_s ICH_HCR_EL2, xzr // Reset ICC_HCR_EL2 to defaults 523 5243: 525#endif 526 527 /* Populate ID registers. */ 528 mrs x0, midr_el1 529 mrs x1, mpidr_el1 530 msr vpidr_el2, x0 531 msr vmpidr_el2, x1 532 533 /* sctlr_el1 */ 534 mov x0, #0x0800 // Set/clear RES{1,0} bits 535CPU_BE( movk x0, #0x33d0, lsl #16 ) // Set EE and E0E on BE systems 536CPU_LE( movk x0, #0x30d0, lsl #16 ) // Clear EE and E0E on LE systems 537 msr sctlr_el1, x0 538 539 /* Coprocessor traps. */ 540 mov x0, #0x33ff 541 msr cptr_el2, x0 // Disable copro. traps to EL2 542 543#ifdef CONFIG_COMPAT 544 msr hstr_el2, xzr // Disable CP15 traps to EL2 545#endif 546 547 /* Stage-2 translation */ 548 msr vttbr_el2, xzr 549 550 /* Hypervisor stub */ 551 adrp x0, __hyp_stub_vectors 552 add x0, x0, #:lo12:__hyp_stub_vectors 553 msr vbar_el2, x0 554 555 /* spsr */ 556 mov x0, #(PSR_F_BIT | PSR_I_BIT | PSR_A_BIT | PSR_D_BIT |\ 557 PSR_MODE_EL1h) 558 msr spsr_el2, x0 559 msr elr_el2, lr 560 mov w20, #BOOT_CPU_MODE_EL2 // This CPU booted in EL2 561 eret 562ENDPROC(el2_setup) 563 564/* 565 * Sets the __boot_cpu_mode flag depending on the CPU boot mode passed 566 * in x20. See arch/arm64/include/asm/virt.h for more info. 567 */ 568ENTRY(set_cpu_boot_mode_flag) 569 ldr x1, =__boot_cpu_mode // Compute __boot_cpu_mode 570 add x1, x1, x28 571 cmp w20, #BOOT_CPU_MODE_EL2 572 b.ne 1f 573 add x1, x1, #4 5741: str w20, [x1] // This CPU has booted in EL1 575 dmb sy 576 dc ivac, x1 // Invalidate potentially stale cache line 577 ret 578ENDPROC(set_cpu_boot_mode_flag) 579 580/* 581 * We need to find out the CPU boot mode long after boot, so we need to 582 * store it in a writable variable. 583 * 584 * This is not in .bss, because we set it sufficiently early that the boot-time 585 * zeroing of .bss would clobber it. 586 */ 587 .pushsection .data..cacheline_aligned 588ENTRY(__boot_cpu_mode) 589 .align L1_CACHE_SHIFT 590 .long BOOT_CPU_MODE_EL2 591 .long 0 592 .popsection 593 594#ifdef CONFIG_SMP 595 .align 3 5961: .quad . 597 .quad secondary_holding_pen_release 598 599 /* 600 * This provides a "holding pen" for platforms to hold all secondary 601 * cores are held until we're ready for them to initialise. 602 */ 603ENTRY(secondary_holding_pen) 604 bl el2_setup // Drop to EL1, w20=cpu_boot_mode 605 bl __calc_phys_offset // x24=PHYS_OFFSET, x28=PHYS_OFFSET-PAGE_OFFSET 606 bl set_cpu_boot_mode_flag 607 mrs x0, mpidr_el1 608 ldr x1, =MPIDR_HWID_BITMASK 609 and x0, x0, x1 610 adr x1, 1b 611 ldp x2, x3, [x1] 612 sub x1, x1, x2 613 add x3, x3, x1 614pen: ldr x4, [x3] 615 cmp x4, x0 616 b.eq secondary_startup 617 wfe 618 b pen 619ENDPROC(secondary_holding_pen) 620 621 /* 622 * Secondary entry point that jumps straight into the kernel. Only to 623 * be used where CPUs are brought online dynamically by the kernel. 624 */ 625ENTRY(secondary_entry) 626 bl el2_setup // Drop to EL1 627 bl __calc_phys_offset // x24=PHYS_OFFSET, x28=PHYS_OFFSET-PAGE_OFFSET 628 bl set_cpu_boot_mode_flag 629 b secondary_startup 630ENDPROC(secondary_entry) 631 632ENTRY(secondary_startup) 633 /* 634 * Common entry point for secondary CPUs. 635 */ 636 mrs x22, midr_el1 // x22=cpuid 637 mov x0, x22 638 bl lookup_processor_type 639 mov x23, x0 // x23=current cpu_table 640 cbz x23, __error_p // invalid processor (x23=0)? 641 642 pgtbl x25, x26, x28 // x25=TTBR0, x26=TTBR1 643 ldr x12, [x23, #CPU_INFO_SETUP] 644 add x12, x12, x28 // __virt_to_phys 645 blr x12 // initialise processor 646 647 ldr x21, =secondary_data 648 ldr x27, =__secondary_switched // address to jump to after enabling the MMU 649 b __enable_mmu 650ENDPROC(secondary_startup) 651 652ENTRY(__secondary_switched) 653 ldr x0, [x21] // get secondary_data.stack 654 mov sp, x0 655 mov x29, #0 656 b secondary_start_kernel 657ENDPROC(__secondary_switched) 658#endif /* CONFIG_SMP */ 659 660/* 661 * Setup common bits before finally enabling the MMU. Essentially this is just 662 * loading the page table pointer and vector base registers. 663 * 664 * On entry to this code, x0 must contain the SCTLR_EL1 value for turning on 665 * the MMU. 666 */ 667__enable_mmu: 668 ldr x5, =vectors 669 msr vbar_el1, x5 670 msr ttbr0_el1, x25 // load TTBR0 671 msr ttbr1_el1, x26 // load TTBR1 672 isb 673 b __turn_mmu_on 674ENDPROC(__enable_mmu) 675 676/* 677 * Enable the MMU. This completely changes the structure of the visible memory 678 * space. You will not be able to trace execution through this. 679 * 680 * x0 = system control register 681 * x27 = *virtual* address to jump to upon completion 682 * 683 * other registers depend on the function called upon completion 684 * 685 * We align the entire function to the smallest power of two larger than it to 686 * ensure it fits within a single block map entry. Otherwise were PHYS_OFFSET 687 * close to the end of a 512MB or 1GB block we might require an additional 688 * table to map the entire function. 689 */ 690 .align 4 691__turn_mmu_on: 692 msr sctlr_el1, x0 693 isb 694 br x27 695ENDPROC(__turn_mmu_on) 696 697/* 698 * Calculate the start of physical memory. 699 */ 700__calc_phys_offset: 701 adr x0, 1f 702 ldp x1, x2, [x0] 703 sub x28, x0, x1 // x28 = PHYS_OFFSET - PAGE_OFFSET 704 add x24, x2, x28 // x24 = PHYS_OFFSET 705 ret 706ENDPROC(__calc_phys_offset) 707 708 .align 3 7091: .quad . 710 .quad PAGE_OFFSET 711 712/* 713 * Exception handling. Something went wrong and we can't proceed. We ought to 714 * tell the user, but since we don't have any guarantee that we're even 715 * running on the right architecture, we do virtually nothing. 716 */ 717__error_p: 718ENDPROC(__error_p) 719 720__error: 7211: nop 722 b 1b 723ENDPROC(__error) 724 725/* 726 * This function gets the processor ID in w0 and searches the cpu_table[] for 727 * a match. It returns a pointer to the struct cpu_info it found. The 728 * cpu_table[] must end with an empty (all zeros) structure. 729 * 730 * This routine can be called via C code and it needs to work with the MMU 731 * both disabled and enabled (the offset is calculated automatically). 732 */ 733ENTRY(lookup_processor_type) 734 adr x1, __lookup_processor_type_data 735 ldp x2, x3, [x1] 736 sub x1, x1, x2 // get offset between VA and PA 737 add x3, x3, x1 // convert VA to PA 7381: 739 ldp w5, w6, [x3] // load cpu_id_val and cpu_id_mask 740 cbz w5, 2f // end of list? 741 and w6, w6, w0 742 cmp w5, w6 743 b.eq 3f 744 add x3, x3, #CPU_INFO_SZ 745 b 1b 7462: 747 mov x3, #0 // unknown processor 7483: 749 mov x0, x3 750 ret 751ENDPROC(lookup_processor_type) 752 753 .align 3 754 .type __lookup_processor_type_data, %object 755__lookup_processor_type_data: 756 .quad . 757 .quad cpu_table 758 .size __lookup_processor_type_data, . - __lookup_processor_type_data 759