xref: /openbmc/linux/arch/arm64/kernel/head.S (revision 68198dca)
1/*
2 * Low-level CPU initialisation
3 * Based on arch/arm/kernel/head.S
4 *
5 * Copyright (C) 1994-2002 Russell King
6 * Copyright (C) 2003-2012 ARM Ltd.
7 * Authors:	Catalin Marinas <catalin.marinas@arm.com>
8 *		Will Deacon <will.deacon@arm.com>
9 *
10 * This program is free software; you can redistribute it and/or modify
11 * it under the terms of the GNU General Public License version 2 as
12 * published by the Free Software Foundation.
13 *
14 * This program is distributed in the hope that it will be useful,
15 * but WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
17 * GNU General Public License for more details.
18 *
19 * You should have received a copy of the GNU General Public License
20 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
21 */
22
23#include <linux/linkage.h>
24#include <linux/init.h>
25#include <linux/irqchip/arm-gic-v3.h>
26
27#include <asm/assembler.h>
28#include <asm/boot.h>
29#include <asm/ptrace.h>
30#include <asm/asm-offsets.h>
31#include <asm/cache.h>
32#include <asm/cputype.h>
33#include <asm/elf.h>
34#include <asm/kernel-pgtable.h>
35#include <asm/kvm_arm.h>
36#include <asm/memory.h>
37#include <asm/pgtable-hwdef.h>
38#include <asm/pgtable.h>
39#include <asm/page.h>
40#include <asm/smp.h>
41#include <asm/sysreg.h>
42#include <asm/thread_info.h>
43#include <asm/virt.h>
44
45#include "efi-header.S"
46
47#define __PHYS_OFFSET	(KERNEL_START - TEXT_OFFSET)
48
49#if (TEXT_OFFSET & 0xfff) != 0
50#error TEXT_OFFSET must be at least 4KB aligned
51#elif (PAGE_OFFSET & 0x1fffff) != 0
52#error PAGE_OFFSET must be at least 2MB aligned
53#elif TEXT_OFFSET > 0x1fffff
54#error TEXT_OFFSET must be less than 2MB
55#endif
56
57/*
58 * Kernel startup entry point.
59 * ---------------------------
60 *
61 * The requirements are:
62 *   MMU = off, D-cache = off, I-cache = on or off,
63 *   x0 = physical address to the FDT blob.
64 *
65 * This code is mostly position independent so you call this at
66 * __pa(PAGE_OFFSET + TEXT_OFFSET).
67 *
68 * Note that the callee-saved registers are used for storing variables
69 * that are useful before the MMU is enabled. The allocations are described
70 * in the entry routines.
71 */
72	__HEAD
73_head:
74	/*
75	 * DO NOT MODIFY. Image header expected by Linux boot-loaders.
76	 */
77#ifdef CONFIG_EFI
78	/*
79	 * This add instruction has no meaningful effect except that
80	 * its opcode forms the magic "MZ" signature required by UEFI.
81	 */
82	add	x13, x18, #0x16
83	b	stext
84#else
85	b	stext				// branch to kernel start, magic
86	.long	0				// reserved
87#endif
88	le64sym	_kernel_offset_le		// Image load offset from start of RAM, little-endian
89	le64sym	_kernel_size_le			// Effective size of kernel image, little-endian
90	le64sym	_kernel_flags_le		// Informative flags, little-endian
91	.quad	0				// reserved
92	.quad	0				// reserved
93	.quad	0				// reserved
94	.ascii	"ARM\x64"			// Magic number
95#ifdef CONFIG_EFI
96	.long	pe_header - _head		// Offset to the PE header.
97
98pe_header:
99	__EFI_PE_HEADER
100#else
101	.long	0				// reserved
102#endif
103
104	__INIT
105
106	/*
107	 * The following callee saved general purpose registers are used on the
108	 * primary lowlevel boot path:
109	 *
110	 *  Register   Scope                      Purpose
111	 *  x21        stext() .. start_kernel()  FDT pointer passed at boot in x0
112	 *  x23        stext() .. start_kernel()  physical misalignment/KASLR offset
113	 *  x28        __create_page_tables()     callee preserved temp register
114	 *  x19/x20    __primary_switch()         callee preserved temp registers
115	 */
116ENTRY(stext)
117	bl	preserve_boot_args
118	bl	el2_setup			// Drop to EL1, w0=cpu_boot_mode
119	adrp	x23, __PHYS_OFFSET
120	and	x23, x23, MIN_KIMG_ALIGN - 1	// KASLR offset, defaults to 0
121	bl	set_cpu_boot_mode_flag
122	bl	__create_page_tables
123	/*
124	 * The following calls CPU setup code, see arch/arm64/mm/proc.S for
125	 * details.
126	 * On return, the CPU will be ready for the MMU to be turned on and
127	 * the TCR will have been set.
128	 */
129	bl	__cpu_setup			// initialise processor
130	b	__primary_switch
131ENDPROC(stext)
132
133/*
134 * Preserve the arguments passed by the bootloader in x0 .. x3
135 */
136preserve_boot_args:
137	mov	x21, x0				// x21=FDT
138
139	adr_l	x0, boot_args			// record the contents of
140	stp	x21, x1, [x0]			// x0 .. x3 at kernel entry
141	stp	x2, x3, [x0, #16]
142
143	dmb	sy				// needed before dc ivac with
144						// MMU off
145
146	mov	x1, #0x20			// 4 x 8 bytes
147	b	__inval_dcache_area		// tail call
148ENDPROC(preserve_boot_args)
149
150/*
151 * Macro to create a table entry to the next page.
152 *
153 *	tbl:	page table address
154 *	virt:	virtual address
155 *	shift:	#imm page table shift
156 *	ptrs:	#imm pointers per table page
157 *
158 * Preserves:	virt
159 * Corrupts:	tmp1, tmp2
160 * Returns:	tbl -> next level table page address
161 */
162	.macro	create_table_entry, tbl, virt, shift, ptrs, tmp1, tmp2
163	lsr	\tmp1, \virt, #\shift
164	and	\tmp1, \tmp1, #\ptrs - 1	// table index
165	add	\tmp2, \tbl, #PAGE_SIZE
166	orr	\tmp2, \tmp2, #PMD_TYPE_TABLE	// address of next table and entry type
167	str	\tmp2, [\tbl, \tmp1, lsl #3]
168	add	\tbl, \tbl, #PAGE_SIZE		// next level table page
169	.endm
170
171/*
172 * Macro to populate the PGD (and possibily PUD) for the corresponding
173 * block entry in the next level (tbl) for the given virtual address.
174 *
175 * Preserves:	tbl, next, virt
176 * Corrupts:	tmp1, tmp2
177 */
178	.macro	create_pgd_entry, tbl, virt, tmp1, tmp2
179	create_table_entry \tbl, \virt, PGDIR_SHIFT, PTRS_PER_PGD, \tmp1, \tmp2
180#if SWAPPER_PGTABLE_LEVELS > 3
181	create_table_entry \tbl, \virt, PUD_SHIFT, PTRS_PER_PUD, \tmp1, \tmp2
182#endif
183#if SWAPPER_PGTABLE_LEVELS > 2
184	create_table_entry \tbl, \virt, SWAPPER_TABLE_SHIFT, PTRS_PER_PTE, \tmp1, \tmp2
185#endif
186	.endm
187
188/*
189 * Macro to populate block entries in the page table for the start..end
190 * virtual range (inclusive).
191 *
192 * Preserves:	tbl, flags
193 * Corrupts:	phys, start, end, pstate
194 */
195	.macro	create_block_map, tbl, flags, phys, start, end
196	lsr	\phys, \phys, #SWAPPER_BLOCK_SHIFT
197	lsr	\start, \start, #SWAPPER_BLOCK_SHIFT
198	and	\start, \start, #PTRS_PER_PTE - 1	// table index
199	orr	\phys, \flags, \phys, lsl #SWAPPER_BLOCK_SHIFT	// table entry
200	lsr	\end, \end, #SWAPPER_BLOCK_SHIFT
201	and	\end, \end, #PTRS_PER_PTE - 1		// table end index
2029999:	str	\phys, [\tbl, \start, lsl #3]		// store the entry
203	add	\start, \start, #1			// next entry
204	add	\phys, \phys, #SWAPPER_BLOCK_SIZE		// next block
205	cmp	\start, \end
206	b.ls	9999b
207	.endm
208
209/*
210 * Setup the initial page tables. We only setup the barest amount which is
211 * required to get the kernel running. The following sections are required:
212 *   - identity mapping to enable the MMU (low address, TTBR0)
213 *   - first few MB of the kernel linear mapping to jump to once the MMU has
214 *     been enabled
215 */
216__create_page_tables:
217	mov	x28, lr
218
219	/*
220	 * Invalidate the idmap and swapper page tables to avoid potential
221	 * dirty cache lines being evicted.
222	 */
223	adrp	x0, idmap_pg_dir
224	ldr	x1, =(IDMAP_DIR_SIZE + SWAPPER_DIR_SIZE + RESERVED_TTBR0_SIZE)
225	bl	__inval_dcache_area
226
227	/*
228	 * Clear the idmap and swapper page tables.
229	 */
230	adrp	x0, idmap_pg_dir
231	ldr	x1, =(IDMAP_DIR_SIZE + SWAPPER_DIR_SIZE + RESERVED_TTBR0_SIZE)
2321:	stp	xzr, xzr, [x0], #16
233	stp	xzr, xzr, [x0], #16
234	stp	xzr, xzr, [x0], #16
235	stp	xzr, xzr, [x0], #16
236	subs	x1, x1, #64
237	b.ne	1b
238
239	mov	x7, SWAPPER_MM_MMUFLAGS
240
241	/*
242	 * Create the identity mapping.
243	 */
244	adrp	x0, idmap_pg_dir
245	adrp	x3, __idmap_text_start		// __pa(__idmap_text_start)
246
247#ifndef CONFIG_ARM64_VA_BITS_48
248#define EXTRA_SHIFT	(PGDIR_SHIFT + PAGE_SHIFT - 3)
249#define EXTRA_PTRS	(1 << (48 - EXTRA_SHIFT))
250
251	/*
252	 * If VA_BITS < 48, it may be too small to allow for an ID mapping to be
253	 * created that covers system RAM if that is located sufficiently high
254	 * in the physical address space. So for the ID map, use an extended
255	 * virtual range in that case, by configuring an additional translation
256	 * level.
257	 * First, we have to verify our assumption that the current value of
258	 * VA_BITS was chosen such that all translation levels are fully
259	 * utilised, and that lowering T0SZ will always result in an additional
260	 * translation level to be configured.
261	 */
262#if VA_BITS != EXTRA_SHIFT
263#error "Mismatch between VA_BITS and page size/number of translation levels"
264#endif
265
266	/*
267	 * Calculate the maximum allowed value for TCR_EL1.T0SZ so that the
268	 * entire ID map region can be mapped. As T0SZ == (64 - #bits used),
269	 * this number conveniently equals the number of leading zeroes in
270	 * the physical address of __idmap_text_end.
271	 */
272	adrp	x5, __idmap_text_end
273	clz	x5, x5
274	cmp	x5, TCR_T0SZ(VA_BITS)	// default T0SZ small enough?
275	b.ge	1f			// .. then skip additional level
276
277	adr_l	x6, idmap_t0sz
278	str	x5, [x6]
279	dmb	sy
280	dc	ivac, x6		// Invalidate potentially stale cache line
281
282	create_table_entry x0, x3, EXTRA_SHIFT, EXTRA_PTRS, x5, x6
2831:
284#endif
285
286	create_pgd_entry x0, x3, x5, x6
287	mov	x5, x3				// __pa(__idmap_text_start)
288	adr_l	x6, __idmap_text_end		// __pa(__idmap_text_end)
289	create_block_map x0, x7, x3, x5, x6
290
291	/*
292	 * Map the kernel image (starting with PHYS_OFFSET).
293	 */
294	adrp	x0, swapper_pg_dir
295	mov_q	x5, KIMAGE_VADDR + TEXT_OFFSET	// compile time __va(_text)
296	add	x5, x5, x23			// add KASLR displacement
297	create_pgd_entry x0, x5, x3, x6
298	adrp	x6, _end			// runtime __pa(_end)
299	adrp	x3, _text			// runtime __pa(_text)
300	sub	x6, x6, x3			// _end - _text
301	add	x6, x6, x5			// runtime __va(_end)
302	create_block_map x0, x7, x3, x5, x6
303
304	/*
305	 * Since the page tables have been populated with non-cacheable
306	 * accesses (MMU disabled), invalidate the idmap and swapper page
307	 * tables again to remove any speculatively loaded cache lines.
308	 */
309	adrp	x0, idmap_pg_dir
310	ldr	x1, =(IDMAP_DIR_SIZE + SWAPPER_DIR_SIZE + RESERVED_TTBR0_SIZE)
311	dmb	sy
312	bl	__inval_dcache_area
313
314	ret	x28
315ENDPROC(__create_page_tables)
316	.ltorg
317
318/*
319 * The following fragment of code is executed with the MMU enabled.
320 *
321 *   x0 = __PHYS_OFFSET
322 */
323__primary_switched:
324	adrp	x4, init_thread_union
325	add	sp, x4, #THREAD_SIZE
326	adr_l	x5, init_task
327	msr	sp_el0, x5			// Save thread_info
328
329	adr_l	x8, vectors			// load VBAR_EL1 with virtual
330	msr	vbar_el1, x8			// vector table address
331	isb
332
333	stp	xzr, x30, [sp, #-16]!
334	mov	x29, sp
335
336	str_l	x21, __fdt_pointer, x5		// Save FDT pointer
337
338	ldr_l	x4, kimage_vaddr		// Save the offset between
339	sub	x4, x4, x0			// the kernel virtual and
340	str_l	x4, kimage_voffset, x5		// physical mappings
341
342	// Clear BSS
343	adr_l	x0, __bss_start
344	mov	x1, xzr
345	adr_l	x2, __bss_stop
346	sub	x2, x2, x0
347	bl	__pi_memset
348	dsb	ishst				// Make zero page visible to PTW
349
350#ifdef CONFIG_KASAN
351	bl	kasan_early_init
352#endif
353#ifdef CONFIG_RANDOMIZE_BASE
354	tst	x23, ~(MIN_KIMG_ALIGN - 1)	// already running randomized?
355	b.ne	0f
356	mov	x0, x21				// pass FDT address in x0
357	bl	kaslr_early_init		// parse FDT for KASLR options
358	cbz	x0, 0f				// KASLR disabled? just proceed
359	orr	x23, x23, x0			// record KASLR offset
360	ldp	x29, x30, [sp], #16		// we must enable KASLR, return
361	ret					// to __primary_switch()
3620:
363#endif
364	add	sp, sp, #16
365	mov	x29, #0
366	mov	x30, #0
367	b	start_kernel
368ENDPROC(__primary_switched)
369
370/*
371 * end early head section, begin head code that is also used for
372 * hotplug and needs to have the same protections as the text region
373 */
374	.section ".idmap.text","ax"
375
376ENTRY(kimage_vaddr)
377	.quad		_text - TEXT_OFFSET
378
379/*
380 * If we're fortunate enough to boot at EL2, ensure that the world is
381 * sane before dropping to EL1.
382 *
383 * Returns either BOOT_CPU_MODE_EL1 or BOOT_CPU_MODE_EL2 in w0 if
384 * booted in EL1 or EL2 respectively.
385 */
386ENTRY(el2_setup)
387	msr	SPsel, #1			// We want to use SP_EL{1,2}
388	mrs	x0, CurrentEL
389	cmp	x0, #CurrentEL_EL2
390	b.eq	1f
391	mrs	x0, sctlr_el1
392CPU_BE(	orr	x0, x0, #(3 << 24)	)	// Set the EE and E0E bits for EL1
393CPU_LE(	bic	x0, x0, #(3 << 24)	)	// Clear the EE and E0E bits for EL1
394	msr	sctlr_el1, x0
395	mov	w0, #BOOT_CPU_MODE_EL1		// This cpu booted in EL1
396	isb
397	ret
398
3991:	mrs	x0, sctlr_el2
400CPU_BE(	orr	x0, x0, #(1 << 25)	)	// Set the EE bit for EL2
401CPU_LE(	bic	x0, x0, #(1 << 25)	)	// Clear the EE bit for EL2
402	msr	sctlr_el2, x0
403
404#ifdef CONFIG_ARM64_VHE
405	/*
406	 * Check for VHE being present. For the rest of the EL2 setup,
407	 * x2 being non-zero indicates that we do have VHE, and that the
408	 * kernel is intended to run at EL2.
409	 */
410	mrs	x2, id_aa64mmfr1_el1
411	ubfx	x2, x2, #8, #4
412#else
413	mov	x2, xzr
414#endif
415
416	/* Hyp configuration. */
417	mov	x0, #HCR_RW			// 64-bit EL1
418	cbz	x2, set_hcr
419	orr	x0, x0, #HCR_TGE		// Enable Host Extensions
420	orr	x0, x0, #HCR_E2H
421set_hcr:
422	msr	hcr_el2, x0
423	isb
424
425	/*
426	 * Allow Non-secure EL1 and EL0 to access physical timer and counter.
427	 * This is not necessary for VHE, since the host kernel runs in EL2,
428	 * and EL0 accesses are configured in the later stage of boot process.
429	 * Note that when HCR_EL2.E2H == 1, CNTHCTL_EL2 has the same bit layout
430	 * as CNTKCTL_EL1, and CNTKCTL_EL1 accessing instructions are redefined
431	 * to access CNTHCTL_EL2. This allows the kernel designed to run at EL1
432	 * to transparently mess with the EL0 bits via CNTKCTL_EL1 access in
433	 * EL2.
434	 */
435	cbnz	x2, 1f
436	mrs	x0, cnthctl_el2
437	orr	x0, x0, #3			// Enable EL1 physical timers
438	msr	cnthctl_el2, x0
4391:
440	msr	cntvoff_el2, xzr		// Clear virtual offset
441
442#ifdef CONFIG_ARM_GIC_V3
443	/* GICv3 system register access */
444	mrs	x0, id_aa64pfr0_el1
445	ubfx	x0, x0, #24, #4
446	cmp	x0, #1
447	b.ne	3f
448
449	mrs_s	x0, SYS_ICC_SRE_EL2
450	orr	x0, x0, #ICC_SRE_EL2_SRE	// Set ICC_SRE_EL2.SRE==1
451	orr	x0, x0, #ICC_SRE_EL2_ENABLE	// Set ICC_SRE_EL2.Enable==1
452	msr_s	SYS_ICC_SRE_EL2, x0
453	isb					// Make sure SRE is now set
454	mrs_s	x0, SYS_ICC_SRE_EL2		// Read SRE back,
455	tbz	x0, #0, 3f			// and check that it sticks
456	msr_s	SYS_ICH_HCR_EL2, xzr		// Reset ICC_HCR_EL2 to defaults
457
4583:
459#endif
460
461	/* Populate ID registers. */
462	mrs	x0, midr_el1
463	mrs	x1, mpidr_el1
464	msr	vpidr_el2, x0
465	msr	vmpidr_el2, x1
466
467#ifdef CONFIG_COMPAT
468	msr	hstr_el2, xzr			// Disable CP15 traps to EL2
469#endif
470
471	/* EL2 debug */
472	mrs	x1, id_aa64dfr0_el1		// Check ID_AA64DFR0_EL1 PMUVer
473	sbfx	x0, x1, #8, #4
474	cmp	x0, #1
475	b.lt	4f				// Skip if no PMU present
476	mrs	x0, pmcr_el0			// Disable debug access traps
477	ubfx	x0, x0, #11, #5			// to EL2 and allow access to
4784:
479	csel	x3, xzr, x0, lt			// all PMU counters from EL1
480
481	/* Statistical profiling */
482	ubfx	x0, x1, #32, #4			// Check ID_AA64DFR0_EL1 PMSVer
483	cbz	x0, 7f				// Skip if SPE not present
484	cbnz	x2, 6f				// VHE?
485	mrs_s	x4, SYS_PMBIDR_EL1		// If SPE available at EL2,
486	and	x4, x4, #(1 << SYS_PMBIDR_EL1_P_SHIFT)
487	cbnz	x4, 5f				// then permit sampling of physical
488	mov	x4, #(1 << SYS_PMSCR_EL2_PCT_SHIFT | \
489		      1 << SYS_PMSCR_EL2_PA_SHIFT)
490	msr_s	SYS_PMSCR_EL2, x4		// addresses and physical counter
4915:
492	mov	x1, #(MDCR_EL2_E2PB_MASK << MDCR_EL2_E2PB_SHIFT)
493	orr	x3, x3, x1			// If we don't have VHE, then
494	b	7f				// use EL1&0 translation.
4956:						// For VHE, use EL2 translation
496	orr	x3, x3, #MDCR_EL2_TPMS		// and disable access from EL1
4977:
498	msr	mdcr_el2, x3			// Configure debug traps
499
500	/* Stage-2 translation */
501	msr	vttbr_el2, xzr
502
503	cbz	x2, install_el2_stub
504
505	mov	w0, #BOOT_CPU_MODE_EL2		// This CPU booted in EL2
506	isb
507	ret
508
509install_el2_stub:
510	/*
511	 * When VHE is not in use, early init of EL2 and EL1 needs to be
512	 * done here.
513	 * When VHE _is_ in use, EL1 will not be used in the host and
514	 * requires no configuration, and all non-hyp-specific EL2 setup
515	 * will be done via the _EL1 system register aliases in __cpu_setup.
516	 */
517	/* sctlr_el1 */
518	mov	x0, #0x0800			// Set/clear RES{1,0} bits
519CPU_BE(	movk	x0, #0x33d0, lsl #16	)	// Set EE and E0E on BE systems
520CPU_LE(	movk	x0, #0x30d0, lsl #16	)	// Clear EE and E0E on LE systems
521	msr	sctlr_el1, x0
522
523	/* Coprocessor traps. */
524	mov	x0, #0x33ff
525	msr	cptr_el2, x0			// Disable copro. traps to EL2
526
527	/* SVE register access */
528	mrs	x1, id_aa64pfr0_el1
529	ubfx	x1, x1, #ID_AA64PFR0_SVE_SHIFT, #4
530	cbz	x1, 7f
531
532	bic	x0, x0, #CPTR_EL2_TZ		// Also disable SVE traps
533	msr	cptr_el2, x0			// Disable copro. traps to EL2
534	isb
535	mov	x1, #ZCR_ELx_LEN_MASK		// SVE: Enable full vector
536	msr_s	SYS_ZCR_EL2, x1			// length for EL1.
537
538	/* Hypervisor stub */
5397:	adr_l	x0, __hyp_stub_vectors
540	msr	vbar_el2, x0
541
542	/* spsr */
543	mov	x0, #(PSR_F_BIT | PSR_I_BIT | PSR_A_BIT | PSR_D_BIT |\
544		      PSR_MODE_EL1h)
545	msr	spsr_el2, x0
546	msr	elr_el2, lr
547	mov	w0, #BOOT_CPU_MODE_EL2		// This CPU booted in EL2
548	eret
549ENDPROC(el2_setup)
550
551/*
552 * Sets the __boot_cpu_mode flag depending on the CPU boot mode passed
553 * in w0. See arch/arm64/include/asm/virt.h for more info.
554 */
555set_cpu_boot_mode_flag:
556	adr_l	x1, __boot_cpu_mode
557	cmp	w0, #BOOT_CPU_MODE_EL2
558	b.ne	1f
559	add	x1, x1, #4
5601:	str	w0, [x1]			// This CPU has booted in EL1
561	dmb	sy
562	dc	ivac, x1			// Invalidate potentially stale cache line
563	ret
564ENDPROC(set_cpu_boot_mode_flag)
565
566/*
567 * These values are written with the MMU off, but read with the MMU on.
568 * Writers will invalidate the corresponding address, discarding up to a
569 * 'Cache Writeback Granule' (CWG) worth of data. The linker script ensures
570 * sufficient alignment that the CWG doesn't overlap another section.
571 */
572	.pushsection ".mmuoff.data.write", "aw"
573/*
574 * We need to find out the CPU boot mode long after boot, so we need to
575 * store it in a writable variable.
576 *
577 * This is not in .bss, because we set it sufficiently early that the boot-time
578 * zeroing of .bss would clobber it.
579 */
580ENTRY(__boot_cpu_mode)
581	.long	BOOT_CPU_MODE_EL2
582	.long	BOOT_CPU_MODE_EL1
583/*
584 * The booting CPU updates the failed status @__early_cpu_boot_status,
585 * with MMU turned off.
586 */
587ENTRY(__early_cpu_boot_status)
588	.long 	0
589
590	.popsection
591
592	/*
593	 * This provides a "holding pen" for platforms to hold all secondary
594	 * cores are held until we're ready for them to initialise.
595	 */
596ENTRY(secondary_holding_pen)
597	bl	el2_setup			// Drop to EL1, w0=cpu_boot_mode
598	bl	set_cpu_boot_mode_flag
599	mrs	x0, mpidr_el1
600	mov_q	x1, MPIDR_HWID_BITMASK
601	and	x0, x0, x1
602	adr_l	x3, secondary_holding_pen_release
603pen:	ldr	x4, [x3]
604	cmp	x4, x0
605	b.eq	secondary_startup
606	wfe
607	b	pen
608ENDPROC(secondary_holding_pen)
609
610	/*
611	 * Secondary entry point that jumps straight into the kernel. Only to
612	 * be used where CPUs are brought online dynamically by the kernel.
613	 */
614ENTRY(secondary_entry)
615	bl	el2_setup			// Drop to EL1
616	bl	set_cpu_boot_mode_flag
617	b	secondary_startup
618ENDPROC(secondary_entry)
619
620secondary_startup:
621	/*
622	 * Common entry point for secondary CPUs.
623	 */
624	bl	__cpu_setup			// initialise processor
625	bl	__enable_mmu
626	ldr	x8, =__secondary_switched
627	br	x8
628ENDPROC(secondary_startup)
629
630__secondary_switched:
631	adr_l	x5, vectors
632	msr	vbar_el1, x5
633	isb
634
635	adr_l	x0, secondary_data
636	ldr	x1, [x0, #CPU_BOOT_STACK]	// get secondary_data.stack
637	mov	sp, x1
638	ldr	x2, [x0, #CPU_BOOT_TASK]
639	msr	sp_el0, x2
640	mov	x29, #0
641	mov	x30, #0
642	b	secondary_start_kernel
643ENDPROC(__secondary_switched)
644
645/*
646 * The booting CPU updates the failed status @__early_cpu_boot_status,
647 * with MMU turned off.
648 *
649 * update_early_cpu_boot_status tmp, status
650 *  - Corrupts tmp1, tmp2
651 *  - Writes 'status' to __early_cpu_boot_status and makes sure
652 *    it is committed to memory.
653 */
654
655	.macro	update_early_cpu_boot_status status, tmp1, tmp2
656	mov	\tmp2, #\status
657	adr_l	\tmp1, __early_cpu_boot_status
658	str	\tmp2, [\tmp1]
659	dmb	sy
660	dc	ivac, \tmp1			// Invalidate potentially stale cache line
661	.endm
662
663/*
664 * Enable the MMU.
665 *
666 *  x0  = SCTLR_EL1 value for turning on the MMU.
667 *
668 * Returns to the caller via x30/lr. This requires the caller to be covered
669 * by the .idmap.text section.
670 *
671 * Checks if the selected granule size is supported by the CPU.
672 * If it isn't, park the CPU
673 */
674ENTRY(__enable_mmu)
675	mrs	x1, ID_AA64MMFR0_EL1
676	ubfx	x2, x1, #ID_AA64MMFR0_TGRAN_SHIFT, 4
677	cmp	x2, #ID_AA64MMFR0_TGRAN_SUPPORTED
678	b.ne	__no_granule_support
679	update_early_cpu_boot_status 0, x1, x2
680	adrp	x1, idmap_pg_dir
681	adrp	x2, swapper_pg_dir
682	msr	ttbr0_el1, x1			// load TTBR0
683	msr	ttbr1_el1, x2			// load TTBR1
684	isb
685	msr	sctlr_el1, x0
686	isb
687	/*
688	 * Invalidate the local I-cache so that any instructions fetched
689	 * speculatively from the PoC are discarded, since they may have
690	 * been dynamically patched at the PoU.
691	 */
692	ic	iallu
693	dsb	nsh
694	isb
695	ret
696ENDPROC(__enable_mmu)
697
698__no_granule_support:
699	/* Indicate that this CPU can't boot and is stuck in the kernel */
700	update_early_cpu_boot_status CPU_STUCK_IN_KERNEL, x1, x2
7011:
702	wfe
703	wfi
704	b	1b
705ENDPROC(__no_granule_support)
706
707#ifdef CONFIG_RELOCATABLE
708__relocate_kernel:
709	/*
710	 * Iterate over each entry in the relocation table, and apply the
711	 * relocations in place.
712	 */
713	ldr	w9, =__rela_offset		// offset to reloc table
714	ldr	w10, =__rela_size		// size of reloc table
715
716	mov_q	x11, KIMAGE_VADDR		// default virtual offset
717	add	x11, x11, x23			// actual virtual offset
718	add	x9, x9, x11			// __va(.rela)
719	add	x10, x9, x10			// __va(.rela) + sizeof(.rela)
720
7210:	cmp	x9, x10
722	b.hs	1f
723	ldp	x11, x12, [x9], #24
724	ldr	x13, [x9, #-8]
725	cmp	w12, #R_AARCH64_RELATIVE
726	b.ne	0b
727	add	x13, x13, x23			// relocate
728	str	x13, [x11, x23]
729	b	0b
7301:	ret
731ENDPROC(__relocate_kernel)
732#endif
733
734__primary_switch:
735#ifdef CONFIG_RANDOMIZE_BASE
736	mov	x19, x0				// preserve new SCTLR_EL1 value
737	mrs	x20, sctlr_el1			// preserve old SCTLR_EL1 value
738#endif
739
740	bl	__enable_mmu
741#ifdef CONFIG_RELOCATABLE
742	bl	__relocate_kernel
743#ifdef CONFIG_RANDOMIZE_BASE
744	ldr	x8, =__primary_switched
745	adrp	x0, __PHYS_OFFSET
746	blr	x8
747
748	/*
749	 * If we return here, we have a KASLR displacement in x23 which we need
750	 * to take into account by discarding the current kernel mapping and
751	 * creating a new one.
752	 */
753	pre_disable_mmu_workaround
754	msr	sctlr_el1, x20			// disable the MMU
755	isb
756	bl	__create_page_tables		// recreate kernel mapping
757
758	tlbi	vmalle1				// Remove any stale TLB entries
759	dsb	nsh
760
761	msr	sctlr_el1, x19			// re-enable the MMU
762	isb
763	ic	iallu				// flush instructions fetched
764	dsb	nsh				// via old mapping
765	isb
766
767	bl	__relocate_kernel
768#endif
769#endif
770	ldr	x8, =__primary_switched
771	adrp	x0, __PHYS_OFFSET
772	br	x8
773ENDPROC(__primary_switch)
774