1/* 2 * Low-level CPU initialisation 3 * Based on arch/arm/kernel/head.S 4 * 5 * Copyright (C) 1994-2002 Russell King 6 * Copyright (C) 2003-2012 ARM Ltd. 7 * Authors: Catalin Marinas <catalin.marinas@arm.com> 8 * Will Deacon <will.deacon@arm.com> 9 * 10 * This program is free software; you can redistribute it and/or modify 11 * it under the terms of the GNU General Public License version 2 as 12 * published by the Free Software Foundation. 13 * 14 * This program is distributed in the hope that it will be useful, 15 * but WITHOUT ANY WARRANTY; without even the implied warranty of 16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 17 * GNU General Public License for more details. 18 * 19 * You should have received a copy of the GNU General Public License 20 * along with this program. If not, see <http://www.gnu.org/licenses/>. 21 */ 22 23#include <linux/linkage.h> 24#include <linux/init.h> 25 26#include <asm/assembler.h> 27#include <asm/ptrace.h> 28#include <asm/asm-offsets.h> 29#include <asm/memory.h> 30#include <asm/thread_info.h> 31#include <asm/pgtable-hwdef.h> 32#include <asm/pgtable.h> 33#include <asm/page.h> 34#include <asm/virt.h> 35 36/* 37 * swapper_pg_dir is the virtual address of the initial page table. We place 38 * the page tables 3 * PAGE_SIZE below KERNEL_RAM_VADDR. The idmap_pg_dir has 39 * 2 pages and is placed below swapper_pg_dir. 40 */ 41#define KERNEL_RAM_VADDR (PAGE_OFFSET + TEXT_OFFSET) 42 43#if (KERNEL_RAM_VADDR & 0xfffff) != 0x80000 44#error KERNEL_RAM_VADDR must start at 0xXXX80000 45#endif 46 47#define SWAPPER_DIR_SIZE (3 * PAGE_SIZE) 48#define IDMAP_DIR_SIZE (2 * PAGE_SIZE) 49 50 .globl swapper_pg_dir 51 .equ swapper_pg_dir, KERNEL_RAM_VADDR - SWAPPER_DIR_SIZE 52 53 .globl idmap_pg_dir 54 .equ idmap_pg_dir, swapper_pg_dir - IDMAP_DIR_SIZE 55 56 .macro pgtbl, ttb0, ttb1, phys 57 add \ttb1, \phys, #TEXT_OFFSET - SWAPPER_DIR_SIZE 58 sub \ttb0, \ttb1, #IDMAP_DIR_SIZE 59 .endm 60 61#ifdef CONFIG_ARM64_64K_PAGES 62#define BLOCK_SHIFT PAGE_SHIFT 63#define BLOCK_SIZE PAGE_SIZE 64#else 65#define BLOCK_SHIFT SECTION_SHIFT 66#define BLOCK_SIZE SECTION_SIZE 67#endif 68 69#define KERNEL_START KERNEL_RAM_VADDR 70#define KERNEL_END _end 71 72/* 73 * Initial memory map attributes. 74 */ 75#ifndef CONFIG_SMP 76#define PTE_FLAGS PTE_TYPE_PAGE | PTE_AF 77#define PMD_FLAGS PMD_TYPE_SECT | PMD_SECT_AF 78#else 79#define PTE_FLAGS PTE_TYPE_PAGE | PTE_AF | PTE_SHARED 80#define PMD_FLAGS PMD_TYPE_SECT | PMD_SECT_AF | PMD_SECT_S 81#endif 82 83#ifdef CONFIG_ARM64_64K_PAGES 84#define MM_MMUFLAGS PTE_ATTRINDX(MT_NORMAL) | PTE_FLAGS 85#else 86#define MM_MMUFLAGS PMD_ATTRINDX(MT_NORMAL) | PMD_FLAGS 87#endif 88 89/* 90 * Kernel startup entry point. 91 * --------------------------- 92 * 93 * The requirements are: 94 * MMU = off, D-cache = off, I-cache = on or off, 95 * x0 = physical address to the FDT blob. 96 * 97 * This code is mostly position independent so you call this at 98 * __pa(PAGE_OFFSET + TEXT_OFFSET). 99 * 100 * Note that the callee-saved registers are used for storing variables 101 * that are useful before the MMU is enabled. The allocations are described 102 * in the entry routines. 103 */ 104 __HEAD 105 106 /* 107 * DO NOT MODIFY. Image header expected by Linux boot-loaders. 108 */ 109 b stext // branch to kernel start, magic 110 .long 0 // reserved 111 .quad TEXT_OFFSET // Image load offset from start of RAM 112 .quad 0 // reserved 113 .quad 0 // reserved 114 115ENTRY(stext) 116 mov x21, x0 // x21=FDT 117 bl __calc_phys_offset // x24=PHYS_OFFSET, x28=PHYS_OFFSET-PAGE_OFFSET 118 bl el2_setup // Drop to EL1 119 mrs x22, midr_el1 // x22=cpuid 120 mov x0, x22 121 bl lookup_processor_type 122 mov x23, x0 // x23=current cpu_table 123 cbz x23, __error_p // invalid processor (x23=0)? 124 bl __vet_fdt 125 bl __create_page_tables // x25=TTBR0, x26=TTBR1 126 /* 127 * The following calls CPU specific code in a position independent 128 * manner. See arch/arm64/mm/proc.S for details. x23 = base of 129 * cpu_info structure selected by lookup_processor_type above. 130 * On return, the CPU will be ready for the MMU to be turned on and 131 * the TCR will have been set. 132 */ 133 ldr x27, __switch_data // address to jump to after 134 // MMU has been enabled 135 adr lr, __enable_mmu // return (PIC) address 136 ldr x12, [x23, #CPU_INFO_SETUP] 137 add x12, x12, x28 // __virt_to_phys 138 br x12 // initialise processor 139ENDPROC(stext) 140 141/* 142 * If we're fortunate enough to boot at EL2, ensure that the world is 143 * sane before dropping to EL1. 144 */ 145ENTRY(el2_setup) 146 mrs x0, CurrentEL 147 cmp x0, #PSR_MODE_EL2t 148 ccmp x0, #PSR_MODE_EL2h, #0x4, ne 149 ldr x0, =__boot_cpu_mode // Compute __boot_cpu_mode 150 add x0, x0, x28 151 b.eq 1f 152 str wzr, [x0] // Remember we don't have EL2... 153 ret 154 155 /* Hyp configuration. */ 1561: ldr w1, =BOOT_CPU_MODE_EL2 157 str w1, [x0, #4] // This CPU has EL2 158 mov x0, #(1 << 31) // 64-bit EL1 159 msr hcr_el2, x0 160 161 /* Generic timers. */ 162 mrs x0, cnthctl_el2 163 orr x0, x0, #3 // Enable EL1 physical timers 164 msr cnthctl_el2, x0 165 msr cntvoff_el2, xzr // Clear virtual offset 166 167 /* Populate ID registers. */ 168 mrs x0, midr_el1 169 mrs x1, mpidr_el1 170 msr vpidr_el2, x0 171 msr vmpidr_el2, x1 172 173 /* sctlr_el1 */ 174 mov x0, #0x0800 // Set/clear RES{1,0} bits 175 movk x0, #0x30d0, lsl #16 176 msr sctlr_el1, x0 177 178 /* Coprocessor traps. */ 179 mov x0, #0x33ff 180 msr cptr_el2, x0 // Disable copro. traps to EL2 181 182#ifdef CONFIG_COMPAT 183 msr hstr_el2, xzr // Disable CP15 traps to EL2 184#endif 185 186 /* Stage-2 translation */ 187 msr vttbr_el2, xzr 188 189 /* Hypervisor stub */ 190 adr x0, __hyp_stub_vectors 191 msr vbar_el2, x0 192 193 /* spsr */ 194 mov x0, #(PSR_F_BIT | PSR_I_BIT | PSR_A_BIT | PSR_D_BIT |\ 195 PSR_MODE_EL1h) 196 msr spsr_el2, x0 197 msr elr_el2, lr 198 eret 199ENDPROC(el2_setup) 200 201/* 202 * We need to find out the CPU boot mode long after boot, so we need to 203 * store it in a writable variable. 204 * 205 * This is not in .bss, because we set it sufficiently early that the boot-time 206 * zeroing of .bss would clobber it. 207 */ 208 .pushsection .data 209ENTRY(__boot_cpu_mode) 210 .long BOOT_CPU_MODE_EL2 211 .long 0 212 .popsection 213 214 .align 3 2152: .quad . 216 .quad PAGE_OFFSET 217 218#ifdef CONFIG_SMP 219 .pushsection .smp.pen.text, "ax" 220 .align 3 2211: .quad . 222 .quad secondary_holding_pen_release 223 224 /* 225 * This provides a "holding pen" for platforms to hold all secondary 226 * cores are held until we're ready for them to initialise. 227 */ 228ENTRY(secondary_holding_pen) 229 bl __calc_phys_offset // x24=phys offset 230 bl el2_setup // Drop to EL1 231 mrs x0, mpidr_el1 232 and x0, x0, #15 // CPU number 233 adr x1, 1b 234 ldp x2, x3, [x1] 235 sub x1, x1, x2 236 add x3, x3, x1 237pen: ldr x4, [x3] 238 cmp x4, x0 239 b.eq secondary_startup 240 wfe 241 b pen 242ENDPROC(secondary_holding_pen) 243 .popsection 244 245ENTRY(secondary_startup) 246 /* 247 * Common entry point for secondary CPUs. 248 */ 249 mrs x22, midr_el1 // x22=cpuid 250 mov x0, x22 251 bl lookup_processor_type 252 mov x23, x0 // x23=current cpu_table 253 cbz x23, __error_p // invalid processor (x23=0)? 254 255 pgtbl x25, x26, x24 // x25=TTBR0, x26=TTBR1 256 ldr x12, [x23, #CPU_INFO_SETUP] 257 add x12, x12, x28 // __virt_to_phys 258 blr x12 // initialise processor 259 260 ldr x21, =secondary_data 261 ldr x27, =__secondary_switched // address to jump to after enabling the MMU 262 b __enable_mmu 263ENDPROC(secondary_startup) 264 265ENTRY(__secondary_switched) 266 ldr x0, [x21] // get secondary_data.stack 267 mov sp, x0 268 mov x29, #0 269 b secondary_start_kernel 270ENDPROC(__secondary_switched) 271#endif /* CONFIG_SMP */ 272 273/* 274 * Setup common bits before finally enabling the MMU. Essentially this is just 275 * loading the page table pointer and vector base registers. 276 * 277 * On entry to this code, x0 must contain the SCTLR_EL1 value for turning on 278 * the MMU. 279 */ 280__enable_mmu: 281 ldr x5, =vectors 282 msr vbar_el1, x5 283 msr ttbr0_el1, x25 // load TTBR0 284 msr ttbr1_el1, x26 // load TTBR1 285 isb 286 b __turn_mmu_on 287ENDPROC(__enable_mmu) 288 289/* 290 * Enable the MMU. This completely changes the structure of the visible memory 291 * space. You will not be able to trace execution through this. 292 * 293 * x0 = system control register 294 * x27 = *virtual* address to jump to upon completion 295 * 296 * other registers depend on the function called upon completion 297 */ 298 .align 6 299__turn_mmu_on: 300 msr sctlr_el1, x0 301 isb 302 br x27 303ENDPROC(__turn_mmu_on) 304 305/* 306 * Calculate the start of physical memory. 307 */ 308__calc_phys_offset: 309 adr x0, 1f 310 ldp x1, x2, [x0] 311 sub x28, x0, x1 // x28 = PHYS_OFFSET - PAGE_OFFSET 312 add x24, x2, x28 // x24 = PHYS_OFFSET 313 ret 314ENDPROC(__calc_phys_offset) 315 316 .align 3 3171: .quad . 318 .quad PAGE_OFFSET 319 320/* 321 * Macro to populate the PGD for the corresponding block entry in the next 322 * level (tbl) for the given virtual address. 323 * 324 * Preserves: pgd, tbl, virt 325 * Corrupts: tmp1, tmp2 326 */ 327 .macro create_pgd_entry, pgd, tbl, virt, tmp1, tmp2 328 lsr \tmp1, \virt, #PGDIR_SHIFT 329 and \tmp1, \tmp1, #PTRS_PER_PGD - 1 // PGD index 330 orr \tmp2, \tbl, #3 // PGD entry table type 331 str \tmp2, [\pgd, \tmp1, lsl #3] 332 .endm 333 334/* 335 * Macro to populate block entries in the page table for the start..end 336 * virtual range (inclusive). 337 * 338 * Preserves: tbl, flags 339 * Corrupts: phys, start, end, pstate 340 */ 341 .macro create_block_map, tbl, flags, phys, start, end, idmap=0 342 lsr \phys, \phys, #BLOCK_SHIFT 343 .if \idmap 344 and \start, \phys, #PTRS_PER_PTE - 1 // table index 345 .else 346 lsr \start, \start, #BLOCK_SHIFT 347 and \start, \start, #PTRS_PER_PTE - 1 // table index 348 .endif 349 orr \phys, \flags, \phys, lsl #BLOCK_SHIFT // table entry 350 .ifnc \start,\end 351 lsr \end, \end, #BLOCK_SHIFT 352 and \end, \end, #PTRS_PER_PTE - 1 // table end index 353 .endif 3549999: str \phys, [\tbl, \start, lsl #3] // store the entry 355 .ifnc \start,\end 356 add \start, \start, #1 // next entry 357 add \phys, \phys, #BLOCK_SIZE // next block 358 cmp \start, \end 359 b.ls 9999b 360 .endif 361 .endm 362 363/* 364 * Setup the initial page tables. We only setup the barest amount which is 365 * required to get the kernel running. The following sections are required: 366 * - identity mapping to enable the MMU (low address, TTBR0) 367 * - first few MB of the kernel linear mapping to jump to once the MMU has 368 * been enabled, including the FDT blob (TTBR1) 369 * - UART mapping if CONFIG_EARLY_PRINTK is enabled (TTBR1) 370 */ 371__create_page_tables: 372 pgtbl x25, x26, x24 // idmap_pg_dir and swapper_pg_dir addresses 373 374 /* 375 * Clear the idmap and swapper page tables. 376 */ 377 mov x0, x25 378 add x6, x26, #SWAPPER_DIR_SIZE 3791: stp xzr, xzr, [x0], #16 380 stp xzr, xzr, [x0], #16 381 stp xzr, xzr, [x0], #16 382 stp xzr, xzr, [x0], #16 383 cmp x0, x6 384 b.lo 1b 385 386 ldr x7, =MM_MMUFLAGS 387 388 /* 389 * Create the identity mapping. 390 */ 391 add x0, x25, #PAGE_SIZE // section table address 392 adr x3, __turn_mmu_on // virtual/physical address 393 create_pgd_entry x25, x0, x3, x5, x6 394 create_block_map x0, x7, x3, x5, x5, idmap=1 395 396 /* 397 * Map the kernel image (starting with PHYS_OFFSET). 398 */ 399 add x0, x26, #PAGE_SIZE // section table address 400 mov x5, #PAGE_OFFSET 401 create_pgd_entry x26, x0, x5, x3, x6 402 ldr x6, =KERNEL_END - 1 403 mov x3, x24 // phys offset 404 create_block_map x0, x7, x3, x5, x6 405 406 /* 407 * Map the FDT blob (maximum 2MB; must be within 512MB of 408 * PHYS_OFFSET). 409 */ 410 mov x3, x21 // FDT phys address 411 and x3, x3, #~((1 << 21) - 1) // 2MB aligned 412 mov x6, #PAGE_OFFSET 413 sub x5, x3, x24 // subtract PHYS_OFFSET 414 tst x5, #~((1 << 29) - 1) // within 512MB? 415 csel x21, xzr, x21, ne // zero the FDT pointer 416 b.ne 1f 417 add x5, x5, x6 // __va(FDT blob) 418 add x6, x5, #1 << 21 // 2MB for the FDT blob 419 sub x6, x6, #1 // inclusive range 420 create_block_map x0, x7, x3, x5, x6 4211: 422#ifdef CONFIG_EARLY_PRINTK 423 /* 424 * Create the pgd entry for the UART mapping. The full mapping is done 425 * later based earlyprintk kernel parameter. 426 */ 427 ldr x5, =EARLYCON_IOBASE // UART virtual address 428 add x0, x26, #2 * PAGE_SIZE // section table address 429 create_pgd_entry x26, x0, x5, x6, x7 430#endif 431 ret 432ENDPROC(__create_page_tables) 433 .ltorg 434 435 .align 3 436 .type __switch_data, %object 437__switch_data: 438 .quad __mmap_switched 439 .quad __data_loc // x4 440 .quad _data // x5 441 .quad __bss_start // x6 442 .quad _end // x7 443 .quad processor_id // x4 444 .quad __fdt_pointer // x5 445 .quad memstart_addr // x6 446 .quad init_thread_union + THREAD_START_SP // sp 447 448/* 449 * The following fragment of code is executed with the MMU on in MMU mode, and 450 * uses absolute addresses; this is not position independent. 451 */ 452__mmap_switched: 453 adr x3, __switch_data + 8 454 455 ldp x4, x5, [x3], #16 456 ldp x6, x7, [x3], #16 457 cmp x4, x5 // Copy data segment if needed 4581: ccmp x5, x6, #4, ne 459 b.eq 2f 460 ldr x16, [x4], #8 461 str x16, [x5], #8 462 b 1b 4632: 4641: cmp x6, x7 465 b.hs 2f 466 str xzr, [x6], #8 // Clear BSS 467 b 1b 4682: 469 ldp x4, x5, [x3], #16 470 ldr x6, [x3], #8 471 ldr x16, [x3] 472 mov sp, x16 473 str x22, [x4] // Save processor ID 474 str x21, [x5] // Save FDT pointer 475 str x24, [x6] // Save PHYS_OFFSET 476 mov x29, #0 477 b start_kernel 478ENDPROC(__mmap_switched) 479 480/* 481 * Exception handling. Something went wrong and we can't proceed. We ought to 482 * tell the user, but since we don't have any guarantee that we're even 483 * running on the right architecture, we do virtually nothing. 484 */ 485__error_p: 486ENDPROC(__error_p) 487 488__error: 4891: nop 490 b 1b 491ENDPROC(__error) 492 493/* 494 * This function gets the processor ID in w0 and searches the cpu_table[] for 495 * a match. It returns a pointer to the struct cpu_info it found. The 496 * cpu_table[] must end with an empty (all zeros) structure. 497 * 498 * This routine can be called via C code and it needs to work with the MMU 499 * both disabled and enabled (the offset is calculated automatically). 500 */ 501ENTRY(lookup_processor_type) 502 adr x1, __lookup_processor_type_data 503 ldp x2, x3, [x1] 504 sub x1, x1, x2 // get offset between VA and PA 505 add x3, x3, x1 // convert VA to PA 5061: 507 ldp w5, w6, [x3] // load cpu_id_val and cpu_id_mask 508 cbz w5, 2f // end of list? 509 and w6, w6, w0 510 cmp w5, w6 511 b.eq 3f 512 add x3, x3, #CPU_INFO_SZ 513 b 1b 5142: 515 mov x3, #0 // unknown processor 5163: 517 mov x0, x3 518 ret 519ENDPROC(lookup_processor_type) 520 521 .align 3 522 .type __lookup_processor_type_data, %object 523__lookup_processor_type_data: 524 .quad . 525 .quad cpu_table 526 .size __lookup_processor_type_data, . - __lookup_processor_type_data 527 528/* 529 * Determine validity of the x21 FDT pointer. 530 * The dtb must be 8-byte aligned and live in the first 512M of memory. 531 */ 532__vet_fdt: 533 tst x21, #0x7 534 b.ne 1f 535 cmp x21, x24 536 b.lt 1f 537 mov x0, #(1 << 29) 538 add x0, x0, x24 539 cmp x21, x0 540 b.ge 1f 541 ret 5421: 543 mov x21, #0 544 ret 545ENDPROC(__vet_fdt) 546