1/* 2 * Low-level CPU initialisation 3 * Based on arch/arm/kernel/head.S 4 * 5 * Copyright (C) 1994-2002 Russell King 6 * Copyright (C) 2003-2012 ARM Ltd. 7 * Authors: Catalin Marinas <catalin.marinas@arm.com> 8 * Will Deacon <will.deacon@arm.com> 9 * 10 * This program is free software; you can redistribute it and/or modify 11 * it under the terms of the GNU General Public License version 2 as 12 * published by the Free Software Foundation. 13 * 14 * This program is distributed in the hope that it will be useful, 15 * but WITHOUT ANY WARRANTY; without even the implied warranty of 16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 17 * GNU General Public License for more details. 18 * 19 * You should have received a copy of the GNU General Public License 20 * along with this program. If not, see <http://www.gnu.org/licenses/>. 21 */ 22 23#include <linux/linkage.h> 24#include <linux/init.h> 25 26#include <asm/assembler.h> 27#include <asm/ptrace.h> 28#include <asm/asm-offsets.h> 29#include <asm/cputype.h> 30#include <asm/memory.h> 31#include <asm/thread_info.h> 32#include <asm/pgtable-hwdef.h> 33#include <asm/pgtable.h> 34#include <asm/page.h> 35#include <asm/virt.h> 36 37/* 38 * swapper_pg_dir is the virtual address of the initial page table. We place 39 * the page tables 3 * PAGE_SIZE below KERNEL_RAM_VADDR. The idmap_pg_dir has 40 * 2 pages and is placed below swapper_pg_dir. 41 */ 42#define KERNEL_RAM_VADDR (PAGE_OFFSET + TEXT_OFFSET) 43 44#if (KERNEL_RAM_VADDR & 0xfffff) != 0x80000 45#error KERNEL_RAM_VADDR must start at 0xXXX80000 46#endif 47 48#define SWAPPER_DIR_SIZE (3 * PAGE_SIZE) 49#define IDMAP_DIR_SIZE (2 * PAGE_SIZE) 50 51 .globl swapper_pg_dir 52 .equ swapper_pg_dir, KERNEL_RAM_VADDR - SWAPPER_DIR_SIZE 53 54 .globl idmap_pg_dir 55 .equ idmap_pg_dir, swapper_pg_dir - IDMAP_DIR_SIZE 56 57 .macro pgtbl, ttb0, ttb1, phys 58 add \ttb1, \phys, #TEXT_OFFSET - SWAPPER_DIR_SIZE 59 sub \ttb0, \ttb1, #IDMAP_DIR_SIZE 60 .endm 61 62#ifdef CONFIG_ARM64_64K_PAGES 63#define BLOCK_SHIFT PAGE_SHIFT 64#define BLOCK_SIZE PAGE_SIZE 65#else 66#define BLOCK_SHIFT SECTION_SHIFT 67#define BLOCK_SIZE SECTION_SIZE 68#endif 69 70#define KERNEL_START KERNEL_RAM_VADDR 71#define KERNEL_END _end 72 73/* 74 * Initial memory map attributes. 75 */ 76#ifndef CONFIG_SMP 77#define PTE_FLAGS PTE_TYPE_PAGE | PTE_AF 78#define PMD_FLAGS PMD_TYPE_SECT | PMD_SECT_AF 79#else 80#define PTE_FLAGS PTE_TYPE_PAGE | PTE_AF | PTE_SHARED 81#define PMD_FLAGS PMD_TYPE_SECT | PMD_SECT_AF | PMD_SECT_S 82#endif 83 84#ifdef CONFIG_ARM64_64K_PAGES 85#define MM_MMUFLAGS PTE_ATTRINDX(MT_NORMAL) | PTE_FLAGS 86#else 87#define MM_MMUFLAGS PMD_ATTRINDX(MT_NORMAL) | PMD_FLAGS 88#endif 89 90/* 91 * Kernel startup entry point. 92 * --------------------------- 93 * 94 * The requirements are: 95 * MMU = off, D-cache = off, I-cache = on or off, 96 * x0 = physical address to the FDT blob. 97 * 98 * This code is mostly position independent so you call this at 99 * __pa(PAGE_OFFSET + TEXT_OFFSET). 100 * 101 * Note that the callee-saved registers are used for storing variables 102 * that are useful before the MMU is enabled. The allocations are described 103 * in the entry routines. 104 */ 105 __HEAD 106 107 /* 108 * DO NOT MODIFY. Image header expected by Linux boot-loaders. 109 */ 110 b stext // branch to kernel start, magic 111 .long 0 // reserved 112 .quad TEXT_OFFSET // Image load offset from start of RAM 113 .quad 0 // reserved 114 .quad 0 // reserved 115 .quad 0 // reserved 116 .quad 0 // reserved 117 .quad 0 // reserved 118 .byte 0x41 // Magic number, "ARM\x64" 119 .byte 0x52 120 .byte 0x4d 121 .byte 0x64 122 .word 0 // reserved 123 124ENTRY(stext) 125 mov x21, x0 // x21=FDT 126 bl el2_setup // Drop to EL1, w20=cpu_boot_mode 127 bl __calc_phys_offset // x24=PHYS_OFFSET, x28=PHYS_OFFSET-PAGE_OFFSET 128 bl set_cpu_boot_mode_flag 129 mrs x22, midr_el1 // x22=cpuid 130 mov x0, x22 131 bl lookup_processor_type 132 mov x23, x0 // x23=current cpu_table 133 cbz x23, __error_p // invalid processor (x23=0)? 134 bl __vet_fdt 135 bl __create_page_tables // x25=TTBR0, x26=TTBR1 136 /* 137 * The following calls CPU specific code in a position independent 138 * manner. See arch/arm64/mm/proc.S for details. x23 = base of 139 * cpu_info structure selected by lookup_processor_type above. 140 * On return, the CPU will be ready for the MMU to be turned on and 141 * the TCR will have been set. 142 */ 143 ldr x27, __switch_data // address to jump to after 144 // MMU has been enabled 145 adr lr, __enable_mmu // return (PIC) address 146 ldr x12, [x23, #CPU_INFO_SETUP] 147 add x12, x12, x28 // __virt_to_phys 148 br x12 // initialise processor 149ENDPROC(stext) 150 151/* 152 * If we're fortunate enough to boot at EL2, ensure that the world is 153 * sane before dropping to EL1. 154 * 155 * Returns either BOOT_CPU_MODE_EL1 or BOOT_CPU_MODE_EL2 in x20 if 156 * booted in EL1 or EL2 respectively. 157 */ 158ENTRY(el2_setup) 159 mrs x0, CurrentEL 160 cmp x0, #PSR_MODE_EL2t 161 ccmp x0, #PSR_MODE_EL2h, #0x4, ne 162 b.ne 1f 163 mrs x0, sctlr_el2 164CPU_BE( orr x0, x0, #(1 << 25) ) // Set the EE bit for EL2 165CPU_LE( bic x0, x0, #(1 << 25) ) // Clear the EE bit for EL2 166 msr sctlr_el2, x0 167 b 2f 1681: mrs x0, sctlr_el1 169CPU_BE( orr x0, x0, #(3 << 24) ) // Set the EE and E0E bits for EL1 170CPU_LE( bic x0, x0, #(3 << 24) ) // Clear the EE and E0E bits for EL1 171 msr sctlr_el1, x0 172 mov w20, #BOOT_CPU_MODE_EL1 // This cpu booted in EL1 173 isb 174 ret 175 176 /* Hyp configuration. */ 1772: mov x0, #(1 << 31) // 64-bit EL1 178 msr hcr_el2, x0 179 180 /* Generic timers. */ 181 mrs x0, cnthctl_el2 182 orr x0, x0, #3 // Enable EL1 physical timers 183 msr cnthctl_el2, x0 184 msr cntvoff_el2, xzr // Clear virtual offset 185 186 /* Populate ID registers. */ 187 mrs x0, midr_el1 188 mrs x1, mpidr_el1 189 msr vpidr_el2, x0 190 msr vmpidr_el2, x1 191 192 /* sctlr_el1 */ 193 mov x0, #0x0800 // Set/clear RES{1,0} bits 194CPU_BE( movk x0, #0x33d0, lsl #16 ) // Set EE and E0E on BE systems 195CPU_LE( movk x0, #0x30d0, lsl #16 ) // Clear EE and E0E on LE systems 196 msr sctlr_el1, x0 197 198 /* Coprocessor traps. */ 199 mov x0, #0x33ff 200 msr cptr_el2, x0 // Disable copro. traps to EL2 201 202#ifdef CONFIG_COMPAT 203 msr hstr_el2, xzr // Disable CP15 traps to EL2 204#endif 205 206 /* Stage-2 translation */ 207 msr vttbr_el2, xzr 208 209 /* Hypervisor stub */ 210 adr x0, __hyp_stub_vectors 211 msr vbar_el2, x0 212 213 /* spsr */ 214 mov x0, #(PSR_F_BIT | PSR_I_BIT | PSR_A_BIT | PSR_D_BIT |\ 215 PSR_MODE_EL1h) 216 msr spsr_el2, x0 217 msr elr_el2, lr 218 mov w20, #BOOT_CPU_MODE_EL2 // This CPU booted in EL2 219 eret 220ENDPROC(el2_setup) 221 222/* 223 * Sets the __boot_cpu_mode flag depending on the CPU boot mode passed 224 * in x20. See arch/arm64/include/asm/virt.h for more info. 225 */ 226ENTRY(set_cpu_boot_mode_flag) 227 ldr x1, =__boot_cpu_mode // Compute __boot_cpu_mode 228 add x1, x1, x28 229 cmp w20, #BOOT_CPU_MODE_EL2 230 b.ne 1f 231 add x1, x1, #4 2321: str w20, [x1] // This CPU has booted in EL1 233 ret 234ENDPROC(set_cpu_boot_mode_flag) 235 236/* 237 * We need to find out the CPU boot mode long after boot, so we need to 238 * store it in a writable variable. 239 * 240 * This is not in .bss, because we set it sufficiently early that the boot-time 241 * zeroing of .bss would clobber it. 242 */ 243 .pushsection .data 244ENTRY(__boot_cpu_mode) 245 .long BOOT_CPU_MODE_EL2 246 .long 0 247 .popsection 248 249 .align 3 2502: .quad . 251 .quad PAGE_OFFSET 252 253#ifdef CONFIG_SMP 254 .align 3 2551: .quad . 256 .quad secondary_holding_pen_release 257 258 /* 259 * This provides a "holding pen" for platforms to hold all secondary 260 * cores are held until we're ready for them to initialise. 261 */ 262ENTRY(secondary_holding_pen) 263 bl el2_setup // Drop to EL1, w20=cpu_boot_mode 264 bl __calc_phys_offset // x24=PHYS_OFFSET, x28=PHYS_OFFSET-PAGE_OFFSET 265 bl set_cpu_boot_mode_flag 266 mrs x0, mpidr_el1 267 ldr x1, =MPIDR_HWID_BITMASK 268 and x0, x0, x1 269 adr x1, 1b 270 ldp x2, x3, [x1] 271 sub x1, x1, x2 272 add x3, x3, x1 273pen: ldr x4, [x3] 274 cmp x4, x0 275 b.eq secondary_startup 276 wfe 277 b pen 278ENDPROC(secondary_holding_pen) 279 280 /* 281 * Secondary entry point that jumps straight into the kernel. Only to 282 * be used where CPUs are brought online dynamically by the kernel. 283 */ 284ENTRY(secondary_entry) 285 bl el2_setup // Drop to EL1 286 bl __calc_phys_offset // x24=PHYS_OFFSET, x28=PHYS_OFFSET-PAGE_OFFSET 287 bl set_cpu_boot_mode_flag 288 b secondary_startup 289ENDPROC(secondary_entry) 290 291ENTRY(secondary_startup) 292 /* 293 * Common entry point for secondary CPUs. 294 */ 295 mrs x22, midr_el1 // x22=cpuid 296 mov x0, x22 297 bl lookup_processor_type 298 mov x23, x0 // x23=current cpu_table 299 cbz x23, __error_p // invalid processor (x23=0)? 300 301 pgtbl x25, x26, x24 // x25=TTBR0, x26=TTBR1 302 ldr x12, [x23, #CPU_INFO_SETUP] 303 add x12, x12, x28 // __virt_to_phys 304 blr x12 // initialise processor 305 306 ldr x21, =secondary_data 307 ldr x27, =__secondary_switched // address to jump to after enabling the MMU 308 b __enable_mmu 309ENDPROC(secondary_startup) 310 311ENTRY(__secondary_switched) 312 ldr x0, [x21] // get secondary_data.stack 313 mov sp, x0 314 mov x29, #0 315 b secondary_start_kernel 316ENDPROC(__secondary_switched) 317#endif /* CONFIG_SMP */ 318 319/* 320 * Setup common bits before finally enabling the MMU. Essentially this is just 321 * loading the page table pointer and vector base registers. 322 * 323 * On entry to this code, x0 must contain the SCTLR_EL1 value for turning on 324 * the MMU. 325 */ 326__enable_mmu: 327 ldr x5, =vectors 328 msr vbar_el1, x5 329 msr ttbr0_el1, x25 // load TTBR0 330 msr ttbr1_el1, x26 // load TTBR1 331 isb 332 b __turn_mmu_on 333ENDPROC(__enable_mmu) 334 335/* 336 * Enable the MMU. This completely changes the structure of the visible memory 337 * space. You will not be able to trace execution through this. 338 * 339 * x0 = system control register 340 * x27 = *virtual* address to jump to upon completion 341 * 342 * other registers depend on the function called upon completion 343 */ 344 .align 6 345__turn_mmu_on: 346 msr sctlr_el1, x0 347 isb 348 br x27 349ENDPROC(__turn_mmu_on) 350 351/* 352 * Calculate the start of physical memory. 353 */ 354__calc_phys_offset: 355 adr x0, 1f 356 ldp x1, x2, [x0] 357 sub x28, x0, x1 // x28 = PHYS_OFFSET - PAGE_OFFSET 358 add x24, x2, x28 // x24 = PHYS_OFFSET 359 ret 360ENDPROC(__calc_phys_offset) 361 362 .align 3 3631: .quad . 364 .quad PAGE_OFFSET 365 366/* 367 * Macro to populate the PGD for the corresponding block entry in the next 368 * level (tbl) for the given virtual address. 369 * 370 * Preserves: pgd, tbl, virt 371 * Corrupts: tmp1, tmp2 372 */ 373 .macro create_pgd_entry, pgd, tbl, virt, tmp1, tmp2 374 lsr \tmp1, \virt, #PGDIR_SHIFT 375 and \tmp1, \tmp1, #PTRS_PER_PGD - 1 // PGD index 376 orr \tmp2, \tbl, #3 // PGD entry table type 377 str \tmp2, [\pgd, \tmp1, lsl #3] 378 .endm 379 380/* 381 * Macro to populate block entries in the page table for the start..end 382 * virtual range (inclusive). 383 * 384 * Preserves: tbl, flags 385 * Corrupts: phys, start, end, pstate 386 */ 387 .macro create_block_map, tbl, flags, phys, start, end, idmap=0 388 lsr \phys, \phys, #BLOCK_SHIFT 389 .if \idmap 390 and \start, \phys, #PTRS_PER_PTE - 1 // table index 391 .else 392 lsr \start, \start, #BLOCK_SHIFT 393 and \start, \start, #PTRS_PER_PTE - 1 // table index 394 .endif 395 orr \phys, \flags, \phys, lsl #BLOCK_SHIFT // table entry 396 .ifnc \start,\end 397 lsr \end, \end, #BLOCK_SHIFT 398 and \end, \end, #PTRS_PER_PTE - 1 // table end index 399 .endif 4009999: str \phys, [\tbl, \start, lsl #3] // store the entry 401 .ifnc \start,\end 402 add \start, \start, #1 // next entry 403 add \phys, \phys, #BLOCK_SIZE // next block 404 cmp \start, \end 405 b.ls 9999b 406 .endif 407 .endm 408 409/* 410 * Setup the initial page tables. We only setup the barest amount which is 411 * required to get the kernel running. The following sections are required: 412 * - identity mapping to enable the MMU (low address, TTBR0) 413 * - first few MB of the kernel linear mapping to jump to once the MMU has 414 * been enabled, including the FDT blob (TTBR1) 415 * - UART mapping if CONFIG_EARLY_PRINTK is enabled (TTBR1) 416 */ 417__create_page_tables: 418 pgtbl x25, x26, x24 // idmap_pg_dir and swapper_pg_dir addresses 419 420 /* 421 * Clear the idmap and swapper page tables. 422 */ 423 mov x0, x25 424 add x6, x26, #SWAPPER_DIR_SIZE 4251: stp xzr, xzr, [x0], #16 426 stp xzr, xzr, [x0], #16 427 stp xzr, xzr, [x0], #16 428 stp xzr, xzr, [x0], #16 429 cmp x0, x6 430 b.lo 1b 431 432 ldr x7, =MM_MMUFLAGS 433 434 /* 435 * Create the identity mapping. 436 */ 437 add x0, x25, #PAGE_SIZE // section table address 438 adr x3, __turn_mmu_on // virtual/physical address 439 create_pgd_entry x25, x0, x3, x5, x6 440 create_block_map x0, x7, x3, x5, x5, idmap=1 441 442 /* 443 * Map the kernel image (starting with PHYS_OFFSET). 444 */ 445 add x0, x26, #PAGE_SIZE // section table address 446 mov x5, #PAGE_OFFSET 447 create_pgd_entry x26, x0, x5, x3, x6 448 ldr x6, =KERNEL_END - 1 449 mov x3, x24 // phys offset 450 create_block_map x0, x7, x3, x5, x6 451 452 /* 453 * Map the FDT blob (maximum 2MB; must be within 512MB of 454 * PHYS_OFFSET). 455 */ 456 mov x3, x21 // FDT phys address 457 and x3, x3, #~((1 << 21) - 1) // 2MB aligned 458 mov x6, #PAGE_OFFSET 459 sub x5, x3, x24 // subtract PHYS_OFFSET 460 tst x5, #~((1 << 29) - 1) // within 512MB? 461 csel x21, xzr, x21, ne // zero the FDT pointer 462 b.ne 1f 463 add x5, x5, x6 // __va(FDT blob) 464 add x6, x5, #1 << 21 // 2MB for the FDT blob 465 sub x6, x6, #1 // inclusive range 466 create_block_map x0, x7, x3, x5, x6 4671: 468#ifdef CONFIG_EARLY_PRINTK 469 /* 470 * Create the pgd entry for the UART mapping. The full mapping is done 471 * later based earlyprintk kernel parameter. 472 */ 473 ldr x5, =EARLYCON_IOBASE // UART virtual address 474 add x0, x26, #2 * PAGE_SIZE // section table address 475 create_pgd_entry x26, x0, x5, x6, x7 476#endif 477 ret 478ENDPROC(__create_page_tables) 479 .ltorg 480 481 .align 3 482 .type __switch_data, %object 483__switch_data: 484 .quad __mmap_switched 485 .quad __data_loc // x4 486 .quad _data // x5 487 .quad __bss_start // x6 488 .quad _end // x7 489 .quad processor_id // x4 490 .quad __fdt_pointer // x5 491 .quad memstart_addr // x6 492 .quad init_thread_union + THREAD_START_SP // sp 493 494/* 495 * The following fragment of code is executed with the MMU on in MMU mode, and 496 * uses absolute addresses; this is not position independent. 497 */ 498__mmap_switched: 499 adr x3, __switch_data + 8 500 501 ldp x4, x5, [x3], #16 502 ldp x6, x7, [x3], #16 503 cmp x4, x5 // Copy data segment if needed 5041: ccmp x5, x6, #4, ne 505 b.eq 2f 506 ldr x16, [x4], #8 507 str x16, [x5], #8 508 b 1b 5092: 5101: cmp x6, x7 511 b.hs 2f 512 str xzr, [x6], #8 // Clear BSS 513 b 1b 5142: 515 ldp x4, x5, [x3], #16 516 ldr x6, [x3], #8 517 ldr x16, [x3] 518 mov sp, x16 519 str x22, [x4] // Save processor ID 520 str x21, [x5] // Save FDT pointer 521 str x24, [x6] // Save PHYS_OFFSET 522 mov x29, #0 523 b start_kernel 524ENDPROC(__mmap_switched) 525 526/* 527 * Exception handling. Something went wrong and we can't proceed. We ought to 528 * tell the user, but since we don't have any guarantee that we're even 529 * running on the right architecture, we do virtually nothing. 530 */ 531__error_p: 532ENDPROC(__error_p) 533 534__error: 5351: nop 536 b 1b 537ENDPROC(__error) 538 539/* 540 * This function gets the processor ID in w0 and searches the cpu_table[] for 541 * a match. It returns a pointer to the struct cpu_info it found. The 542 * cpu_table[] must end with an empty (all zeros) structure. 543 * 544 * This routine can be called via C code and it needs to work with the MMU 545 * both disabled and enabled (the offset is calculated automatically). 546 */ 547ENTRY(lookup_processor_type) 548 adr x1, __lookup_processor_type_data 549 ldp x2, x3, [x1] 550 sub x1, x1, x2 // get offset between VA and PA 551 add x3, x3, x1 // convert VA to PA 5521: 553 ldp w5, w6, [x3] // load cpu_id_val and cpu_id_mask 554 cbz w5, 2f // end of list? 555 and w6, w6, w0 556 cmp w5, w6 557 b.eq 3f 558 add x3, x3, #CPU_INFO_SZ 559 b 1b 5602: 561 mov x3, #0 // unknown processor 5623: 563 mov x0, x3 564 ret 565ENDPROC(lookup_processor_type) 566 567 .align 3 568 .type __lookup_processor_type_data, %object 569__lookup_processor_type_data: 570 .quad . 571 .quad cpu_table 572 .size __lookup_processor_type_data, . - __lookup_processor_type_data 573 574/* 575 * Determine validity of the x21 FDT pointer. 576 * The dtb must be 8-byte aligned and live in the first 512M of memory. 577 */ 578__vet_fdt: 579 tst x21, #0x7 580 b.ne 1f 581 cmp x21, x24 582 b.lt 1f 583 mov x0, #(1 << 29) 584 add x0, x0, x24 585 cmp x21, x0 586 b.ge 1f 587 ret 5881: 589 mov x21, #0 590 ret 591ENDPROC(__vet_fdt) 592