xref: /openbmc/linux/arch/arm64/kernel/head.S (revision 861e10be) 
1/*
2 * Low-level CPU initialisation
3 * Based on arch/arm/kernel/head.S
4 *
5 * Copyright (C) 1994-2002 Russell King
6 * Copyright (C) 2003-2012 ARM Ltd.
7 * Authors:	Catalin Marinas <catalin.marinas@arm.com>
8 *		Will Deacon <will.deacon@arm.com>
9 *
10 * This program is free software; you can redistribute it and/or modify
11 * it under the terms of the GNU General Public License version 2 as
12 * published by the Free Software Foundation.
13 *
14 * This program is distributed in the hope that it will be useful,
15 * but WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
17 * GNU General Public License for more details.
18 *
19 * You should have received a copy of the GNU General Public License
20 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
21 */
22
23#include <linux/linkage.h>
24#include <linux/init.h>
25
26#include <asm/assembler.h>
27#include <asm/ptrace.h>
28#include <asm/asm-offsets.h>
29#include <asm/memory.h>
30#include <asm/thread_info.h>
31#include <asm/pgtable-hwdef.h>
32#include <asm/pgtable.h>
33#include <asm/page.h>
34#include <asm/virt.h>
35
36/*
37 * swapper_pg_dir is the virtual address of the initial page table. We place
38 * the page tables 3 * PAGE_SIZE below KERNEL_RAM_VADDR. The idmap_pg_dir has
39 * 2 pages and is placed below swapper_pg_dir.
40 */
41#define KERNEL_RAM_VADDR	(PAGE_OFFSET + TEXT_OFFSET)
42
43#if (KERNEL_RAM_VADDR & 0xfffff) != 0x80000
44#error KERNEL_RAM_VADDR must start at 0xXXX80000
45#endif
46
47#define SWAPPER_DIR_SIZE	(3 * PAGE_SIZE)
48#define IDMAP_DIR_SIZE		(2 * PAGE_SIZE)
49
50	.globl	swapper_pg_dir
51	.equ	swapper_pg_dir, KERNEL_RAM_VADDR - SWAPPER_DIR_SIZE
52
53	.globl	idmap_pg_dir
54	.equ	idmap_pg_dir, swapper_pg_dir - IDMAP_DIR_SIZE
55
56	.macro	pgtbl, ttb0, ttb1, phys
57	add	\ttb1, \phys, #TEXT_OFFSET - SWAPPER_DIR_SIZE
58	sub	\ttb0, \ttb1, #IDMAP_DIR_SIZE
59	.endm
60
61#ifdef CONFIG_ARM64_64K_PAGES
62#define BLOCK_SHIFT	PAGE_SHIFT
63#define BLOCK_SIZE	PAGE_SIZE
64#else
65#define BLOCK_SHIFT	SECTION_SHIFT
66#define BLOCK_SIZE	SECTION_SIZE
67#endif
68
69#define KERNEL_START	KERNEL_RAM_VADDR
70#define KERNEL_END	_end
71
72/*
73 * Initial memory map attributes.
74 */
75#ifndef CONFIG_SMP
76#define PTE_FLAGS	PTE_TYPE_PAGE | PTE_AF
77#define PMD_FLAGS	PMD_TYPE_SECT | PMD_SECT_AF
78#else
79#define PTE_FLAGS	PTE_TYPE_PAGE | PTE_AF | PTE_SHARED
80#define PMD_FLAGS	PMD_TYPE_SECT | PMD_SECT_AF | PMD_SECT_S
81#endif
82
83#ifdef CONFIG_ARM64_64K_PAGES
84#define MM_MMUFLAGS	PTE_ATTRINDX(MT_NORMAL) | PTE_FLAGS
85#define IO_MMUFLAGS	PTE_ATTRINDX(MT_DEVICE_nGnRE) | PTE_XN | PTE_FLAGS
86#else
87#define MM_MMUFLAGS	PMD_ATTRINDX(MT_NORMAL) | PMD_FLAGS
88#define IO_MMUFLAGS	PMD_ATTRINDX(MT_DEVICE_nGnRE) | PMD_SECT_XN | PMD_FLAGS
89#endif
90
91/*
92 * Kernel startup entry point.
93 * ---------------------------
94 *
95 * The requirements are:
96 *   MMU = off, D-cache = off, I-cache = on or off,
97 *   x0 = physical address to the FDT blob.
98 *
99 * This code is mostly position independent so you call this at
100 * __pa(PAGE_OFFSET + TEXT_OFFSET).
101 *
102 * Note that the callee-saved registers are used for storing variables
103 * that are useful before the MMU is enabled. The allocations are described
104 * in the entry routines.
105 */
106	__HEAD
107
108	/*
109	 * DO NOT MODIFY. Image header expected by Linux boot-loaders.
110	 */
111	b	stext				// branch to kernel start, magic
112	.long	0				// reserved
113	.quad	TEXT_OFFSET			// Image load offset from start of RAM
114	.quad	0				// reserved
115	.quad	0				// reserved
116
117ENTRY(stext)
118	mov	x21, x0				// x21=FDT
119	bl	__calc_phys_offset		// x24=PHYS_OFFSET, x28=PHYS_OFFSET-PAGE_OFFSET
120	bl	el2_setup			// Drop to EL1
121	mrs	x22, midr_el1			// x22=cpuid
122	mov	x0, x22
123	bl	lookup_processor_type
124	mov	x23, x0				// x23=current cpu_table
125	cbz	x23, __error_p			// invalid processor (x23=0)?
126	bl	__vet_fdt
127	bl	__create_page_tables		// x25=TTBR0, x26=TTBR1
128	/*
129	 * The following calls CPU specific code in a position independent
130	 * manner. See arch/arm64/mm/proc.S for details. x23 = base of
131	 * cpu_info structure selected by lookup_processor_type above.
132	 * On return, the CPU will be ready for the MMU to be turned on and
133	 * the TCR will have been set.
134	 */
135	ldr	x27, __switch_data		// address to jump to after
136						// MMU has been enabled
137	adr	lr, __enable_mmu		// return (PIC) address
138	ldr	x12, [x23, #CPU_INFO_SETUP]
139	add	x12, x12, x28			// __virt_to_phys
140	br	x12				// initialise processor
141ENDPROC(stext)
142
143/*
144 * If we're fortunate enough to boot at EL2, ensure that the world is
145 * sane before dropping to EL1.
146 */
147ENTRY(el2_setup)
148	mrs	x0, CurrentEL
149	cmp	x0, #PSR_MODE_EL2t
150	ccmp	x0, #PSR_MODE_EL2h, #0x4, ne
151	ldr	x0, =__boot_cpu_mode		// Compute __boot_cpu_mode
152	add	x0, x0, x28
153	b.eq	1f
154	str	wzr, [x0]			// Remember we don't have EL2...
155	ret
156
157	/* Hyp configuration. */
1581:	ldr	w1, =BOOT_CPU_MODE_EL2
159	str	w1, [x0, #4]			// This CPU has EL2
160	mov	x0, #(1 << 31)			// 64-bit EL1
161	msr	hcr_el2, x0
162
163	/* Generic timers. */
164	mrs	x0, cnthctl_el2
165	orr	x0, x0, #3			// Enable EL1 physical timers
166	msr	cnthctl_el2, x0
167	msr	cntvoff_el2, xzr		// Clear virtual offset
168
169	/* Populate ID registers. */
170	mrs	x0, midr_el1
171	mrs	x1, mpidr_el1
172	msr	vpidr_el2, x0
173	msr	vmpidr_el2, x1
174
175	/* sctlr_el1 */
176	mov	x0, #0x0800			// Set/clear RES{1,0} bits
177	movk	x0, #0x30d0, lsl #16
178	msr	sctlr_el1, x0
179
180	/* Coprocessor traps. */
181	mov	x0, #0x33ff
182	msr	cptr_el2, x0			// Disable copro. traps to EL2
183
184#ifdef CONFIG_COMPAT
185	msr	hstr_el2, xzr			// Disable CP15 traps to EL2
186#endif
187
188	/* Stage-2 translation */
189	msr	vttbr_el2, xzr
190
191	/* Hypervisor stub */
192	adr	x0, __hyp_stub_vectors
193	msr	vbar_el2, x0
194
195	/* spsr */
196	mov	x0, #(PSR_F_BIT | PSR_I_BIT | PSR_A_BIT | PSR_D_BIT |\
197		      PSR_MODE_EL1h)
198	msr	spsr_el2, x0
199	msr	elr_el2, lr
200	eret
201ENDPROC(el2_setup)
202
203/*
204 * We need to find out the CPU boot mode long after boot, so we need to
205 * store it in a writable variable.
206 *
207 * This is not in .bss, because we set it sufficiently early that the boot-time
208 * zeroing of .bss would clobber it.
209 */
210	.pushsection	.data
211ENTRY(__boot_cpu_mode)
212	.long	BOOT_CPU_MODE_EL2
213	.long	0
214	.popsection
215
216	.align	3
2172:	.quad	.
218	.quad	PAGE_OFFSET
219
220#ifdef CONFIG_SMP
221	.pushsection    .smp.pen.text, "ax"
222	.align	3
2231:	.quad	.
224	.quad	secondary_holding_pen_release
225
226	/*
227	 * This provides a "holding pen" for platforms to hold all secondary
228	 * cores are held until we're ready for them to initialise.
229	 */
230ENTRY(secondary_holding_pen)
231	bl	__calc_phys_offset		// x24=phys offset
232	bl	el2_setup			// Drop to EL1
233	mrs	x0, mpidr_el1
234	and	x0, x0, #15			// CPU number
235	adr	x1, 1b
236	ldp	x2, x3, [x1]
237	sub	x1, x1, x2
238	add	x3, x3, x1
239pen:	ldr	x4, [x3]
240	cmp	x4, x0
241	b.eq	secondary_startup
242	wfe
243	b	pen
244ENDPROC(secondary_holding_pen)
245	.popsection
246
247ENTRY(secondary_startup)
248	/*
249	 * Common entry point for secondary CPUs.
250	 */
251	mrs	x22, midr_el1			// x22=cpuid
252	mov	x0, x22
253	bl	lookup_processor_type
254	mov	x23, x0				// x23=current cpu_table
255	cbz	x23, __error_p			// invalid processor (x23=0)?
256
257	pgtbl	x25, x26, x24			// x25=TTBR0, x26=TTBR1
258	ldr	x12, [x23, #CPU_INFO_SETUP]
259	add	x12, x12, x28			// __virt_to_phys
260	blr	x12				// initialise processor
261
262	ldr	x21, =secondary_data
263	ldr	x27, =__secondary_switched	// address to jump to after enabling the MMU
264	b	__enable_mmu
265ENDPROC(secondary_startup)
266
267ENTRY(__secondary_switched)
268	ldr	x0, [x21]			// get secondary_data.stack
269	mov	sp, x0
270	mov	x29, #0
271	b	secondary_start_kernel
272ENDPROC(__secondary_switched)
273#endif	/* CONFIG_SMP */
274
275/*
276 * Setup common bits before finally enabling the MMU. Essentially this is just
277 * loading the page table pointer and vector base registers.
278 *
279 * On entry to this code, x0 must contain the SCTLR_EL1 value for turning on
280 * the MMU.
281 */
282__enable_mmu:
283	ldr	x5, =vectors
284	msr	vbar_el1, x5
285	msr	ttbr0_el1, x25			// load TTBR0
286	msr	ttbr1_el1, x26			// load TTBR1
287	isb
288	b	__turn_mmu_on
289ENDPROC(__enable_mmu)
290
291/*
292 * Enable the MMU. This completely changes the structure of the visible memory
293 * space. You will not be able to trace execution through this.
294 *
295 *  x0  = system control register
296 *  x27 = *virtual* address to jump to upon completion
297 *
298 * other registers depend on the function called upon completion
299 */
300	.align	6
301__turn_mmu_on:
302	msr	sctlr_el1, x0
303	isb
304	br	x27
305ENDPROC(__turn_mmu_on)
306
307/*
308 * Calculate the start of physical memory.
309 */
310__calc_phys_offset:
311	adr	x0, 1f
312	ldp	x1, x2, [x0]
313	sub	x28, x0, x1			// x28 = PHYS_OFFSET - PAGE_OFFSET
314	add	x24, x2, x28			// x24 = PHYS_OFFSET
315	ret
316ENDPROC(__calc_phys_offset)
317
318	.align 3
3191:	.quad	.
320	.quad	PAGE_OFFSET
321
322/*
323 * Macro to populate the PGD for the corresponding block entry in the next
324 * level (tbl) for the given virtual address.
325 *
326 * Preserves:	pgd, tbl, virt
327 * Corrupts:	tmp1, tmp2
328 */
329	.macro	create_pgd_entry, pgd, tbl, virt, tmp1, tmp2
330	lsr	\tmp1, \virt, #PGDIR_SHIFT
331	and	\tmp1, \tmp1, #PTRS_PER_PGD - 1	// PGD index
332	orr	\tmp2, \tbl, #3			// PGD entry table type
333	str	\tmp2, [\pgd, \tmp1, lsl #3]
334	.endm
335
336/*
337 * Macro to populate block entries in the page table for the start..end
338 * virtual range (inclusive).
339 *
340 * Preserves:	tbl, flags
341 * Corrupts:	phys, start, end, pstate
342 */
343	.macro	create_block_map, tbl, flags, phys, start, end, idmap=0
344	lsr	\phys, \phys, #BLOCK_SHIFT
345	.if	\idmap
346	and	\start, \phys, #PTRS_PER_PTE - 1	// table index
347	.else
348	lsr	\start, \start, #BLOCK_SHIFT
349	and	\start, \start, #PTRS_PER_PTE - 1	// table index
350	.endif
351	orr	\phys, \flags, \phys, lsl #BLOCK_SHIFT	// table entry
352	.ifnc	\start,\end
353	lsr	\end, \end, #BLOCK_SHIFT
354	and	\end, \end, #PTRS_PER_PTE - 1		// table end index
355	.endif
3569999:	str	\phys, [\tbl, \start, lsl #3]		// store the entry
357	.ifnc	\start,\end
358	add	\start, \start, #1			// next entry
359	add	\phys, \phys, #BLOCK_SIZE		// next block
360	cmp	\start, \end
361	b.ls	9999b
362	.endif
363	.endm
364
365/*
366 * Setup the initial page tables. We only setup the barest amount which is
367 * required to get the kernel running. The following sections are required:
368 *   - identity mapping to enable the MMU (low address, TTBR0)
369 *   - first few MB of the kernel linear mapping to jump to once the MMU has
370 *     been enabled, including the FDT blob (TTBR1)
371 */
372__create_page_tables:
373	pgtbl	x25, x26, x24			// idmap_pg_dir and swapper_pg_dir addresses
374
375	/*
376	 * Clear the idmap and swapper page tables.
377	 */
378	mov	x0, x25
379	add	x6, x26, #SWAPPER_DIR_SIZE
3801:	stp	xzr, xzr, [x0], #16
381	stp	xzr, xzr, [x0], #16
382	stp	xzr, xzr, [x0], #16
383	stp	xzr, xzr, [x0], #16
384	cmp	x0, x6
385	b.lo	1b
386
387	ldr	x7, =MM_MMUFLAGS
388
389	/*
390	 * Create the identity mapping.
391	 */
392	add	x0, x25, #PAGE_SIZE		// section table address
393	adr	x3, __turn_mmu_on		// virtual/physical address
394	create_pgd_entry x25, x0, x3, x5, x6
395	create_block_map x0, x7, x3, x5, x5, idmap=1
396
397	/*
398	 * Map the kernel image (starting with PHYS_OFFSET).
399	 */
400	add	x0, x26, #PAGE_SIZE		// section table address
401	mov	x5, #PAGE_OFFSET
402	create_pgd_entry x26, x0, x5, x3, x6
403	ldr	x6, =KERNEL_END - 1
404	mov	x3, x24				// phys offset
405	create_block_map x0, x7, x3, x5, x6
406
407	/*
408	 * Map the FDT blob (maximum 2MB; must be within 512MB of
409	 * PHYS_OFFSET).
410	 */
411	mov	x3, x21				// FDT phys address
412	and	x3, x3, #~((1 << 21) - 1)	// 2MB aligned
413	mov	x6, #PAGE_OFFSET
414	sub	x5, x3, x24			// subtract PHYS_OFFSET
415	tst	x5, #~((1 << 29) - 1)		// within 512MB?
416	csel	x21, xzr, x21, ne		// zero the FDT pointer
417	b.ne	1f
418	add	x5, x5, x6			// __va(FDT blob)
419	add	x6, x5, #1 << 21		// 2MB for the FDT blob
420	sub	x6, x6, #1			// inclusive range
421	create_block_map x0, x7, x3, x5, x6
4221:
423	ret
424ENDPROC(__create_page_tables)
425	.ltorg
426
427	.align	3
428	.type	__switch_data, %object
429__switch_data:
430	.quad	__mmap_switched
431	.quad	__data_loc			// x4
432	.quad	_data				// x5
433	.quad	__bss_start			// x6
434	.quad	_end				// x7
435	.quad	processor_id			// x4
436	.quad	__fdt_pointer			// x5
437	.quad	memstart_addr			// x6
438	.quad	init_thread_union + THREAD_START_SP // sp
439
440/*
441 * The following fragment of code is executed with the MMU on in MMU mode, and
442 * uses absolute addresses; this is not position independent.
443 */
444__mmap_switched:
445	adr	x3, __switch_data + 8
446
447	ldp	x4, x5, [x3], #16
448	ldp	x6, x7, [x3], #16
449	cmp	x4, x5				// Copy data segment if needed
4501:	ccmp	x5, x6, #4, ne
451	b.eq	2f
452	ldr	x16, [x4], #8
453	str	x16, [x5], #8
454	b	1b
4552:
4561:	cmp	x6, x7
457	b.hs	2f
458	str	xzr, [x6], #8			// Clear BSS
459	b	1b
4602:
461	ldp	x4, x5, [x3], #16
462	ldr	x6, [x3], #8
463	ldr	x16, [x3]
464	mov	sp, x16
465	str	x22, [x4]			// Save processor ID
466	str	x21, [x5]			// Save FDT pointer
467	str	x24, [x6]			// Save PHYS_OFFSET
468	mov	x29, #0
469	b	start_kernel
470ENDPROC(__mmap_switched)
471
472/*
473 * Exception handling. Something went wrong and we can't proceed. We ought to
474 * tell the user, but since we don't have any guarantee that we're even
475 * running on the right architecture, we do virtually nothing.
476 */
477__error_p:
478ENDPROC(__error_p)
479
480__error:
4811:	nop
482	b	1b
483ENDPROC(__error)
484
485/*
486 * This function gets the processor ID in w0 and searches the cpu_table[] for
487 * a match. It returns a pointer to the struct cpu_info it found. The
488 * cpu_table[] must end with an empty (all zeros) structure.
489 *
490 * This routine can be called via C code and it needs to work with the MMU
491 * both disabled and enabled (the offset is calculated automatically).
492 */
493ENTRY(lookup_processor_type)
494	adr	x1, __lookup_processor_type_data
495	ldp	x2, x3, [x1]
496	sub	x1, x1, x2			// get offset between VA and PA
497	add	x3, x3, x1			// convert VA to PA
4981:
499	ldp	w5, w6, [x3]			// load cpu_id_val and cpu_id_mask
500	cbz	w5, 2f				// end of list?
501	and	w6, w6, w0
502	cmp	w5, w6
503	b.eq	3f
504	add	x3, x3, #CPU_INFO_SZ
505	b	1b
5062:
507	mov	x3, #0				// unknown processor
5083:
509	mov	x0, x3
510	ret
511ENDPROC(lookup_processor_type)
512
513	.align	3
514	.type	__lookup_processor_type_data, %object
515__lookup_processor_type_data:
516	.quad	.
517	.quad	cpu_table
518	.size	__lookup_processor_type_data, . - __lookup_processor_type_data
519
520/*
521 * Determine validity of the x21 FDT pointer.
522 * The dtb must be 8-byte aligned and live in the first 512M of memory.
523 */
524__vet_fdt:
525	tst	x21, #0x7
526	b.ne	1f
527	cmp	x21, x24
528	b.lt	1f
529	mov	x0, #(1 << 29)
530	add	x0, x0, x24
531	cmp	x21, x0
532	b.ge	1f
533	ret
5341:
535	mov	x21, #0
536	ret
537ENDPROC(__vet_fdt)
538