1/* 2 * Low-level CPU initialisation 3 * Based on arch/arm/kernel/head.S 4 * 5 * Copyright (C) 1994-2002 Russell King 6 * Copyright (C) 2003-2012 ARM Ltd. 7 * Authors: Catalin Marinas <catalin.marinas@arm.com> 8 * Will Deacon <will.deacon@arm.com> 9 * 10 * This program is free software; you can redistribute it and/or modify 11 * it under the terms of the GNU General Public License version 2 as 12 * published by the Free Software Foundation. 13 * 14 * This program is distributed in the hope that it will be useful, 15 * but WITHOUT ANY WARRANTY; without even the implied warranty of 16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 17 * GNU General Public License for more details. 18 * 19 * You should have received a copy of the GNU General Public License 20 * along with this program. If not, see <http://www.gnu.org/licenses/>. 21 */ 22 23#include <linux/linkage.h> 24#include <linux/init.h> 25 26#include <asm/assembler.h> 27#include <asm/ptrace.h> 28#include <asm/asm-offsets.h> 29#include <asm/cputype.h> 30#include <asm/memory.h> 31#include <asm/thread_info.h> 32#include <asm/pgtable-hwdef.h> 33#include <asm/pgtable.h> 34#include <asm/page.h> 35#include <asm/virt.h> 36 37/* 38 * swapper_pg_dir is the virtual address of the initial page table. We place 39 * the page tables 3 * PAGE_SIZE below KERNEL_RAM_VADDR. The idmap_pg_dir has 40 * 2 pages and is placed below swapper_pg_dir. 41 */ 42#define KERNEL_RAM_VADDR (PAGE_OFFSET + TEXT_OFFSET) 43 44#if (KERNEL_RAM_VADDR & 0xfffff) != 0x80000 45#error KERNEL_RAM_VADDR must start at 0xXXX80000 46#endif 47 48#define SWAPPER_DIR_SIZE (3 * PAGE_SIZE) 49#define IDMAP_DIR_SIZE (2 * PAGE_SIZE) 50 51 .globl swapper_pg_dir 52 .equ swapper_pg_dir, KERNEL_RAM_VADDR - SWAPPER_DIR_SIZE 53 54 .globl idmap_pg_dir 55 .equ idmap_pg_dir, swapper_pg_dir - IDMAP_DIR_SIZE 56 57 .macro pgtbl, ttb0, ttb1, phys 58 add \ttb1, \phys, #TEXT_OFFSET - SWAPPER_DIR_SIZE 59 sub \ttb0, \ttb1, #IDMAP_DIR_SIZE 60 .endm 61 62#ifdef CONFIG_ARM64_64K_PAGES 63#define BLOCK_SHIFT PAGE_SHIFT 64#define BLOCK_SIZE PAGE_SIZE 65#else 66#define BLOCK_SHIFT SECTION_SHIFT 67#define BLOCK_SIZE SECTION_SIZE 68#endif 69 70#define KERNEL_START KERNEL_RAM_VADDR 71#define KERNEL_END _end 72 73/* 74 * Initial memory map attributes. 75 */ 76#ifndef CONFIG_SMP 77#define PTE_FLAGS PTE_TYPE_PAGE | PTE_AF 78#define PMD_FLAGS PMD_TYPE_SECT | PMD_SECT_AF 79#else 80#define PTE_FLAGS PTE_TYPE_PAGE | PTE_AF | PTE_SHARED 81#define PMD_FLAGS PMD_TYPE_SECT | PMD_SECT_AF | PMD_SECT_S 82#endif 83 84#ifdef CONFIG_ARM64_64K_PAGES 85#define MM_MMUFLAGS PTE_ATTRINDX(MT_NORMAL) | PTE_FLAGS 86#else 87#define MM_MMUFLAGS PMD_ATTRINDX(MT_NORMAL) | PMD_FLAGS 88#endif 89 90/* 91 * Kernel startup entry point. 92 * --------------------------- 93 * 94 * The requirements are: 95 * MMU = off, D-cache = off, I-cache = on or off, 96 * x0 = physical address to the FDT blob. 97 * 98 * This code is mostly position independent so you call this at 99 * __pa(PAGE_OFFSET + TEXT_OFFSET). 100 * 101 * Note that the callee-saved registers are used for storing variables 102 * that are useful before the MMU is enabled. The allocations are described 103 * in the entry routines. 104 */ 105 __HEAD 106 107 /* 108 * DO NOT MODIFY. Image header expected by Linux boot-loaders. 109 */ 110 b stext // branch to kernel start, magic 111 .long 0 // reserved 112 .quad TEXT_OFFSET // Image load offset from start of RAM 113 .quad 0 // reserved 114 .quad 0 // reserved 115 116ENTRY(stext) 117 mov x21, x0 // x21=FDT 118 bl __calc_phys_offset // x24=PHYS_OFFSET, x28=PHYS_OFFSET-PAGE_OFFSET 119 bl el2_setup // Drop to EL1 120 mrs x22, midr_el1 // x22=cpuid 121 mov x0, x22 122 bl lookup_processor_type 123 mov x23, x0 // x23=current cpu_table 124 cbz x23, __error_p // invalid processor (x23=0)? 125 bl __vet_fdt 126 bl __create_page_tables // x25=TTBR0, x26=TTBR1 127 /* 128 * The following calls CPU specific code in a position independent 129 * manner. See arch/arm64/mm/proc.S for details. x23 = base of 130 * cpu_info structure selected by lookup_processor_type above. 131 * On return, the CPU will be ready for the MMU to be turned on and 132 * the TCR will have been set. 133 */ 134 ldr x27, __switch_data // address to jump to after 135 // MMU has been enabled 136 adr lr, __enable_mmu // return (PIC) address 137 ldr x12, [x23, #CPU_INFO_SETUP] 138 add x12, x12, x28 // __virt_to_phys 139 br x12 // initialise processor 140ENDPROC(stext) 141 142/* 143 * If we're fortunate enough to boot at EL2, ensure that the world is 144 * sane before dropping to EL1. 145 */ 146ENTRY(el2_setup) 147 mrs x0, CurrentEL 148 cmp x0, #PSR_MODE_EL2t 149 ccmp x0, #PSR_MODE_EL2h, #0x4, ne 150 ldr x0, =__boot_cpu_mode // Compute __boot_cpu_mode 151 add x0, x0, x28 152 b.eq 1f 153 str wzr, [x0] // Remember we don't have EL2... 154 ret 155 156 /* Hyp configuration. */ 1571: ldr w1, =BOOT_CPU_MODE_EL2 158 str w1, [x0, #4] // This CPU has EL2 159 mov x0, #(1 << 31) // 64-bit EL1 160 msr hcr_el2, x0 161 162 /* Generic timers. */ 163 mrs x0, cnthctl_el2 164 orr x0, x0, #3 // Enable EL1 physical timers 165 msr cnthctl_el2, x0 166 msr cntvoff_el2, xzr // Clear virtual offset 167 168 /* Populate ID registers. */ 169 mrs x0, midr_el1 170 mrs x1, mpidr_el1 171 msr vpidr_el2, x0 172 msr vmpidr_el2, x1 173 174 /* sctlr_el1 */ 175 mov x0, #0x0800 // Set/clear RES{1,0} bits 176 movk x0, #0x30d0, lsl #16 177 msr sctlr_el1, x0 178 179 /* Coprocessor traps. */ 180 mov x0, #0x33ff 181 msr cptr_el2, x0 // Disable copro. traps to EL2 182 183#ifdef CONFIG_COMPAT 184 msr hstr_el2, xzr // Disable CP15 traps to EL2 185#endif 186 187 /* Stage-2 translation */ 188 msr vttbr_el2, xzr 189 190 /* Hypervisor stub */ 191 adr x0, __hyp_stub_vectors 192 msr vbar_el2, x0 193 194 /* spsr */ 195 mov x0, #(PSR_F_BIT | PSR_I_BIT | PSR_A_BIT | PSR_D_BIT |\ 196 PSR_MODE_EL1h) 197 msr spsr_el2, x0 198 msr elr_el2, lr 199 eret 200ENDPROC(el2_setup) 201 202/* 203 * We need to find out the CPU boot mode long after boot, so we need to 204 * store it in a writable variable. 205 * 206 * This is not in .bss, because we set it sufficiently early that the boot-time 207 * zeroing of .bss would clobber it. 208 */ 209 .pushsection .data 210ENTRY(__boot_cpu_mode) 211 .long BOOT_CPU_MODE_EL2 212 .long 0 213 .popsection 214 215 .align 3 2162: .quad . 217 .quad PAGE_OFFSET 218 219#ifdef CONFIG_SMP 220 .pushsection .smp.pen.text, "ax" 221 .align 3 2221: .quad . 223 .quad secondary_holding_pen_release 224 225 /* 226 * This provides a "holding pen" for platforms to hold all secondary 227 * cores are held until we're ready for them to initialise. 228 */ 229ENTRY(secondary_holding_pen) 230 bl __calc_phys_offset // x24=phys offset 231 bl el2_setup // Drop to EL1 232 mrs x0, mpidr_el1 233 ldr x1, =MPIDR_HWID_BITMASK 234 and x0, x0, x1 235 adr x1, 1b 236 ldp x2, x3, [x1] 237 sub x1, x1, x2 238 add x3, x3, x1 239pen: ldr x4, [x3] 240 cmp x4, x0 241 b.eq secondary_startup 242 wfe 243 b pen 244ENDPROC(secondary_holding_pen) 245 .popsection 246 247ENTRY(secondary_startup) 248 /* 249 * Common entry point for secondary CPUs. 250 */ 251 mrs x22, midr_el1 // x22=cpuid 252 mov x0, x22 253 bl lookup_processor_type 254 mov x23, x0 // x23=current cpu_table 255 cbz x23, __error_p // invalid processor (x23=0)? 256 257 pgtbl x25, x26, x24 // x25=TTBR0, x26=TTBR1 258 ldr x12, [x23, #CPU_INFO_SETUP] 259 add x12, x12, x28 // __virt_to_phys 260 blr x12 // initialise processor 261 262 ldr x21, =secondary_data 263 ldr x27, =__secondary_switched // address to jump to after enabling the MMU 264 b __enable_mmu 265ENDPROC(secondary_startup) 266 267ENTRY(__secondary_switched) 268 ldr x0, [x21] // get secondary_data.stack 269 mov sp, x0 270 mov x29, #0 271 b secondary_start_kernel 272ENDPROC(__secondary_switched) 273#endif /* CONFIG_SMP */ 274 275/* 276 * Setup common bits before finally enabling the MMU. Essentially this is just 277 * loading the page table pointer and vector base registers. 278 * 279 * On entry to this code, x0 must contain the SCTLR_EL1 value for turning on 280 * the MMU. 281 */ 282__enable_mmu: 283 ldr x5, =vectors 284 msr vbar_el1, x5 285 msr ttbr0_el1, x25 // load TTBR0 286 msr ttbr1_el1, x26 // load TTBR1 287 isb 288 b __turn_mmu_on 289ENDPROC(__enable_mmu) 290 291/* 292 * Enable the MMU. This completely changes the structure of the visible memory 293 * space. You will not be able to trace execution through this. 294 * 295 * x0 = system control register 296 * x27 = *virtual* address to jump to upon completion 297 * 298 * other registers depend on the function called upon completion 299 */ 300 .align 6 301__turn_mmu_on: 302 msr sctlr_el1, x0 303 isb 304 br x27 305ENDPROC(__turn_mmu_on) 306 307/* 308 * Calculate the start of physical memory. 309 */ 310__calc_phys_offset: 311 adr x0, 1f 312 ldp x1, x2, [x0] 313 sub x28, x0, x1 // x28 = PHYS_OFFSET - PAGE_OFFSET 314 add x24, x2, x28 // x24 = PHYS_OFFSET 315 ret 316ENDPROC(__calc_phys_offset) 317 318 .align 3 3191: .quad . 320 .quad PAGE_OFFSET 321 322/* 323 * Macro to populate the PGD for the corresponding block entry in the next 324 * level (tbl) for the given virtual address. 325 * 326 * Preserves: pgd, tbl, virt 327 * Corrupts: tmp1, tmp2 328 */ 329 .macro create_pgd_entry, pgd, tbl, virt, tmp1, tmp2 330 lsr \tmp1, \virt, #PGDIR_SHIFT 331 and \tmp1, \tmp1, #PTRS_PER_PGD - 1 // PGD index 332 orr \tmp2, \tbl, #3 // PGD entry table type 333 str \tmp2, [\pgd, \tmp1, lsl #3] 334 .endm 335 336/* 337 * Macro to populate block entries in the page table for the start..end 338 * virtual range (inclusive). 339 * 340 * Preserves: tbl, flags 341 * Corrupts: phys, start, end, pstate 342 */ 343 .macro create_block_map, tbl, flags, phys, start, end, idmap=0 344 lsr \phys, \phys, #BLOCK_SHIFT 345 .if \idmap 346 and \start, \phys, #PTRS_PER_PTE - 1 // table index 347 .else 348 lsr \start, \start, #BLOCK_SHIFT 349 and \start, \start, #PTRS_PER_PTE - 1 // table index 350 .endif 351 orr \phys, \flags, \phys, lsl #BLOCK_SHIFT // table entry 352 .ifnc \start,\end 353 lsr \end, \end, #BLOCK_SHIFT 354 and \end, \end, #PTRS_PER_PTE - 1 // table end index 355 .endif 3569999: str \phys, [\tbl, \start, lsl #3] // store the entry 357 .ifnc \start,\end 358 add \start, \start, #1 // next entry 359 add \phys, \phys, #BLOCK_SIZE // next block 360 cmp \start, \end 361 b.ls 9999b 362 .endif 363 .endm 364 365/* 366 * Setup the initial page tables. We only setup the barest amount which is 367 * required to get the kernel running. The following sections are required: 368 * - identity mapping to enable the MMU (low address, TTBR0) 369 * - first few MB of the kernel linear mapping to jump to once the MMU has 370 * been enabled, including the FDT blob (TTBR1) 371 * - UART mapping if CONFIG_EARLY_PRINTK is enabled (TTBR1) 372 */ 373__create_page_tables: 374 pgtbl x25, x26, x24 // idmap_pg_dir and swapper_pg_dir addresses 375 376 /* 377 * Clear the idmap and swapper page tables. 378 */ 379 mov x0, x25 380 add x6, x26, #SWAPPER_DIR_SIZE 3811: stp xzr, xzr, [x0], #16 382 stp xzr, xzr, [x0], #16 383 stp xzr, xzr, [x0], #16 384 stp xzr, xzr, [x0], #16 385 cmp x0, x6 386 b.lo 1b 387 388 ldr x7, =MM_MMUFLAGS 389 390 /* 391 * Create the identity mapping. 392 */ 393 add x0, x25, #PAGE_SIZE // section table address 394 adr x3, __turn_mmu_on // virtual/physical address 395 create_pgd_entry x25, x0, x3, x5, x6 396 create_block_map x0, x7, x3, x5, x5, idmap=1 397 398 /* 399 * Map the kernel image (starting with PHYS_OFFSET). 400 */ 401 add x0, x26, #PAGE_SIZE // section table address 402 mov x5, #PAGE_OFFSET 403 create_pgd_entry x26, x0, x5, x3, x6 404 ldr x6, =KERNEL_END - 1 405 mov x3, x24 // phys offset 406 create_block_map x0, x7, x3, x5, x6 407 408 /* 409 * Map the FDT blob (maximum 2MB; must be within 512MB of 410 * PHYS_OFFSET). 411 */ 412 mov x3, x21 // FDT phys address 413 and x3, x3, #~((1 << 21) - 1) // 2MB aligned 414 mov x6, #PAGE_OFFSET 415 sub x5, x3, x24 // subtract PHYS_OFFSET 416 tst x5, #~((1 << 29) - 1) // within 512MB? 417 csel x21, xzr, x21, ne // zero the FDT pointer 418 b.ne 1f 419 add x5, x5, x6 // __va(FDT blob) 420 add x6, x5, #1 << 21 // 2MB for the FDT blob 421 sub x6, x6, #1 // inclusive range 422 create_block_map x0, x7, x3, x5, x6 4231: 424#ifdef CONFIG_EARLY_PRINTK 425 /* 426 * Create the pgd entry for the UART mapping. The full mapping is done 427 * later based earlyprintk kernel parameter. 428 */ 429 ldr x5, =EARLYCON_IOBASE // UART virtual address 430 add x0, x26, #2 * PAGE_SIZE // section table address 431 create_pgd_entry x26, x0, x5, x6, x7 432#endif 433 ret 434ENDPROC(__create_page_tables) 435 .ltorg 436 437 .align 3 438 .type __switch_data, %object 439__switch_data: 440 .quad __mmap_switched 441 .quad __data_loc // x4 442 .quad _data // x5 443 .quad __bss_start // x6 444 .quad _end // x7 445 .quad processor_id // x4 446 .quad __fdt_pointer // x5 447 .quad memstart_addr // x6 448 .quad init_thread_union + THREAD_START_SP // sp 449 450/* 451 * The following fragment of code is executed with the MMU on in MMU mode, and 452 * uses absolute addresses; this is not position independent. 453 */ 454__mmap_switched: 455 adr x3, __switch_data + 8 456 457 ldp x4, x5, [x3], #16 458 ldp x6, x7, [x3], #16 459 cmp x4, x5 // Copy data segment if needed 4601: ccmp x5, x6, #4, ne 461 b.eq 2f 462 ldr x16, [x4], #8 463 str x16, [x5], #8 464 b 1b 4652: 4661: cmp x6, x7 467 b.hs 2f 468 str xzr, [x6], #8 // Clear BSS 469 b 1b 4702: 471 ldp x4, x5, [x3], #16 472 ldr x6, [x3], #8 473 ldr x16, [x3] 474 mov sp, x16 475 str x22, [x4] // Save processor ID 476 str x21, [x5] // Save FDT pointer 477 str x24, [x6] // Save PHYS_OFFSET 478 mov x29, #0 479 b start_kernel 480ENDPROC(__mmap_switched) 481 482/* 483 * Exception handling. Something went wrong and we can't proceed. We ought to 484 * tell the user, but since we don't have any guarantee that we're even 485 * running on the right architecture, we do virtually nothing. 486 */ 487__error_p: 488ENDPROC(__error_p) 489 490__error: 4911: nop 492 b 1b 493ENDPROC(__error) 494 495/* 496 * This function gets the processor ID in w0 and searches the cpu_table[] for 497 * a match. It returns a pointer to the struct cpu_info it found. The 498 * cpu_table[] must end with an empty (all zeros) structure. 499 * 500 * This routine can be called via C code and it needs to work with the MMU 501 * both disabled and enabled (the offset is calculated automatically). 502 */ 503ENTRY(lookup_processor_type) 504 adr x1, __lookup_processor_type_data 505 ldp x2, x3, [x1] 506 sub x1, x1, x2 // get offset between VA and PA 507 add x3, x3, x1 // convert VA to PA 5081: 509 ldp w5, w6, [x3] // load cpu_id_val and cpu_id_mask 510 cbz w5, 2f // end of list? 511 and w6, w6, w0 512 cmp w5, w6 513 b.eq 3f 514 add x3, x3, #CPU_INFO_SZ 515 b 1b 5162: 517 mov x3, #0 // unknown processor 5183: 519 mov x0, x3 520 ret 521ENDPROC(lookup_processor_type) 522 523 .align 3 524 .type __lookup_processor_type_data, %object 525__lookup_processor_type_data: 526 .quad . 527 .quad cpu_table 528 .size __lookup_processor_type_data, . - __lookup_processor_type_data 529 530/* 531 * Determine validity of the x21 FDT pointer. 532 * The dtb must be 8-byte aligned and live in the first 512M of memory. 533 */ 534__vet_fdt: 535 tst x21, #0x7 536 b.ne 1f 537 cmp x21, x24 538 b.lt 1f 539 mov x0, #(1 << 29) 540 add x0, x0, x24 541 cmp x21, x0 542 b.ge 1f 543 ret 5441: 545 mov x21, #0 546 ret 547ENDPROC(__vet_fdt) 548