xref: /openbmc/linux/arch/arm64/kernel/entry.S (revision f7d84fa7) 
1/*
2 * Low-level exception handling code
3 *
4 * Copyright (C) 2012 ARM Ltd.
5 * Authors:	Catalin Marinas <catalin.marinas@arm.com>
6 *		Will Deacon <will.deacon@arm.com>
7 *
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License version 2 as
10 * published by the Free Software Foundation.
11 *
12 * This program is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
19 */
20
21#include <linux/init.h>
22#include <linux/linkage.h>
23
24#include <asm/alternative.h>
25#include <asm/assembler.h>
26#include <asm/asm-offsets.h>
27#include <asm/cpufeature.h>
28#include <asm/errno.h>
29#include <asm/esr.h>
30#include <asm/irq.h>
31#include <asm/memory.h>
32#include <asm/ptrace.h>
33#include <asm/thread_info.h>
34#include <asm/asm-uaccess.h>
35#include <asm/unistd.h>
36
37/*
38 * Context tracking subsystem.  Used to instrument transitions
39 * between user and kernel mode.
40 */
41	.macro ct_user_exit, syscall = 0
42#ifdef CONFIG_CONTEXT_TRACKING
43	bl	context_tracking_user_exit
44	.if \syscall == 1
45	/*
46	 * Save/restore needed during syscalls.  Restore syscall arguments from
47	 * the values already saved on stack during kernel_entry.
48	 */
49	ldp	x0, x1, [sp]
50	ldp	x2, x3, [sp, #S_X2]
51	ldp	x4, x5, [sp, #S_X4]
52	ldp	x6, x7, [sp, #S_X6]
53	.endif
54#endif
55	.endm
56
57	.macro ct_user_enter
58#ifdef CONFIG_CONTEXT_TRACKING
59	bl	context_tracking_user_enter
60#endif
61	.endm
62
63/*
64 * Bad Abort numbers
65 *-----------------
66 */
67#define BAD_SYNC	0
68#define BAD_IRQ		1
69#define BAD_FIQ		2
70#define BAD_ERROR	3
71
72	.macro	kernel_entry, el, regsize = 64
73	sub	sp, sp, #S_FRAME_SIZE
74	.if	\regsize == 32
75	mov	w0, w0				// zero upper 32 bits of x0
76	.endif
77	stp	x0, x1, [sp, #16 * 0]
78	stp	x2, x3, [sp, #16 * 1]
79	stp	x4, x5, [sp, #16 * 2]
80	stp	x6, x7, [sp, #16 * 3]
81	stp	x8, x9, [sp, #16 * 4]
82	stp	x10, x11, [sp, #16 * 5]
83	stp	x12, x13, [sp, #16 * 6]
84	stp	x14, x15, [sp, #16 * 7]
85	stp	x16, x17, [sp, #16 * 8]
86	stp	x18, x19, [sp, #16 * 9]
87	stp	x20, x21, [sp, #16 * 10]
88	stp	x22, x23, [sp, #16 * 11]
89	stp	x24, x25, [sp, #16 * 12]
90	stp	x26, x27, [sp, #16 * 13]
91	stp	x28, x29, [sp, #16 * 14]
92
93	.if	\el == 0
94	mrs	x21, sp_el0
95	ldr_this_cpu	tsk, __entry_task, x20	// Ensure MDSCR_EL1.SS is clear,
96	ldr	x19, [tsk, #TSK_TI_FLAGS]	// since we can unmask debug
97	disable_step_tsk x19, x20		// exceptions when scheduling.
98
99	mov	x29, xzr			// fp pointed to user-space
100	.else
101	add	x21, sp, #S_FRAME_SIZE
102	get_thread_info tsk
103	/* Save the task's original addr_limit and set USER_DS (TASK_SIZE_64) */
104	ldr	x20, [tsk, #TSK_TI_ADDR_LIMIT]
105	str	x20, [sp, #S_ORIG_ADDR_LIMIT]
106	mov	x20, #TASK_SIZE_64
107	str	x20, [tsk, #TSK_TI_ADDR_LIMIT]
108	/* No need to reset PSTATE.UAO, hardware's already set it to 0 for us */
109	.endif /* \el == 0 */
110	mrs	x22, elr_el1
111	mrs	x23, spsr_el1
112	stp	lr, x21, [sp, #S_LR]
113
114#ifdef CONFIG_ARM64_SW_TTBR0_PAN
115	/*
116	 * Set the TTBR0 PAN bit in SPSR. When the exception is taken from
117	 * EL0, there is no need to check the state of TTBR0_EL1 since
118	 * accesses are always enabled.
119	 * Note that the meaning of this bit differs from the ARMv8.1 PAN
120	 * feature as all TTBR0_EL1 accesses are disabled, not just those to
121	 * user mappings.
122	 */
123alternative_if ARM64_HAS_PAN
124	b	1f				// skip TTBR0 PAN
125alternative_else_nop_endif
126
127	.if	\el != 0
128	mrs	x21, ttbr0_el1
129	tst	x21, #0xffff << 48		// Check for the reserved ASID
130	orr	x23, x23, #PSR_PAN_BIT		// Set the emulated PAN in the saved SPSR
131	b.eq	1f				// TTBR0 access already disabled
132	and	x23, x23, #~PSR_PAN_BIT		// Clear the emulated PAN in the saved SPSR
133	.endif
134
135	__uaccess_ttbr0_disable x21
1361:
137#endif
138
139	stp	x22, x23, [sp, #S_PC]
140
141	/*
142	 * Set syscallno to -1 by default (overridden later if real syscall).
143	 */
144	.if	\el == 0
145	mvn	x21, xzr
146	str	x21, [sp, #S_SYSCALLNO]
147	.endif
148
149	/*
150	 * Set sp_el0 to current thread_info.
151	 */
152	.if	\el == 0
153	msr	sp_el0, tsk
154	.endif
155
156	/*
157	 * Registers that may be useful after this macro is invoked:
158	 *
159	 * x21 - aborted SP
160	 * x22 - aborted PC
161	 * x23 - aborted PSTATE
162	*/
163	.endm
164
165	.macro	kernel_exit, el
166	.if	\el != 0
167	/* Restore the task's original addr_limit. */
168	ldr	x20, [sp, #S_ORIG_ADDR_LIMIT]
169	str	x20, [tsk, #TSK_TI_ADDR_LIMIT]
170
171	/* No need to restore UAO, it will be restored from SPSR_EL1 */
172	.endif
173
174	ldp	x21, x22, [sp, #S_PC]		// load ELR, SPSR
175	.if	\el == 0
176	ct_user_enter
177	.endif
178
179#ifdef CONFIG_ARM64_SW_TTBR0_PAN
180	/*
181	 * Restore access to TTBR0_EL1. If returning to EL0, no need for SPSR
182	 * PAN bit checking.
183	 */
184alternative_if ARM64_HAS_PAN
185	b	2f				// skip TTBR0 PAN
186alternative_else_nop_endif
187
188	.if	\el != 0
189	tbnz	x22, #22, 1f			// Skip re-enabling TTBR0 access if the PSR_PAN_BIT is set
190	.endif
191
192	__uaccess_ttbr0_enable x0
193
194	.if	\el == 0
195	/*
196	 * Enable errata workarounds only if returning to user. The only
197	 * workaround currently required for TTBR0_EL1 changes are for the
198	 * Cavium erratum 27456 (broadcast TLBI instructions may cause I-cache
199	 * corruption).
200	 */
201	post_ttbr0_update_workaround
202	.endif
2031:
204	.if	\el != 0
205	and	x22, x22, #~PSR_PAN_BIT		// ARMv8.0 CPUs do not understand this bit
206	.endif
2072:
208#endif
209
210	.if	\el == 0
211	ldr	x23, [sp, #S_SP]		// load return stack pointer
212	msr	sp_el0, x23
213#ifdef CONFIG_ARM64_ERRATUM_845719
214alternative_if ARM64_WORKAROUND_845719
215	tbz	x22, #4, 1f
216#ifdef CONFIG_PID_IN_CONTEXTIDR
217	mrs	x29, contextidr_el1
218	msr	contextidr_el1, x29
219#else
220	msr contextidr_el1, xzr
221#endif
2221:
223alternative_else_nop_endif
224#endif
225	.endif
226
227	msr	elr_el1, x21			// set up the return data
228	msr	spsr_el1, x22
229	ldp	x0, x1, [sp, #16 * 0]
230	ldp	x2, x3, [sp, #16 * 1]
231	ldp	x4, x5, [sp, #16 * 2]
232	ldp	x6, x7, [sp, #16 * 3]
233	ldp	x8, x9, [sp, #16 * 4]
234	ldp	x10, x11, [sp, #16 * 5]
235	ldp	x12, x13, [sp, #16 * 6]
236	ldp	x14, x15, [sp, #16 * 7]
237	ldp	x16, x17, [sp, #16 * 8]
238	ldp	x18, x19, [sp, #16 * 9]
239	ldp	x20, x21, [sp, #16 * 10]
240	ldp	x22, x23, [sp, #16 * 11]
241	ldp	x24, x25, [sp, #16 * 12]
242	ldp	x26, x27, [sp, #16 * 13]
243	ldp	x28, x29, [sp, #16 * 14]
244	ldr	lr, [sp, #S_LR]
245	add	sp, sp, #S_FRAME_SIZE		// restore sp
246	eret					// return to kernel
247	.endm
248
249	.macro	irq_stack_entry
250	mov	x19, sp			// preserve the original sp
251
252	/*
253	 * Compare sp with the base of the task stack.
254	 * If the top ~(THREAD_SIZE - 1) bits match, we are on a task stack,
255	 * and should switch to the irq stack.
256	 */
257	ldr	x25, [tsk, TSK_STACK]
258	eor	x25, x25, x19
259	and	x25, x25, #~(THREAD_SIZE - 1)
260	cbnz	x25, 9998f
261
262	adr_this_cpu x25, irq_stack, x26
263	mov	x26, #IRQ_STACK_START_SP
264	add	x26, x25, x26
265
266	/* switch to the irq stack */
267	mov	sp, x26
268
269	/*
270	 * Add a dummy stack frame, this non-standard format is fixed up
271	 * by unwind_frame()
272	 */
273	stp     x29, x19, [sp, #-16]!
274	mov	x29, sp
275
2769998:
277	.endm
278
279	/*
280	 * x19 should be preserved between irq_stack_entry and
281	 * irq_stack_exit.
282	 */
283	.macro	irq_stack_exit
284	mov	sp, x19
285	.endm
286
287/*
288 * These are the registers used in the syscall handler, and allow us to
289 * have in theory up to 7 arguments to a function - x0 to x6.
290 *
291 * x7 is reserved for the system call number in 32-bit mode.
292 */
293sc_nr	.req	x25		// number of system calls
294scno	.req	x26		// syscall number
295stbl	.req	x27		// syscall table pointer
296tsk	.req	x28		// current thread_info
297
298/*
299 * Interrupt handling.
300 */
301	.macro	irq_handler
302	ldr_l	x1, handle_arch_irq
303	mov	x0, sp
304	irq_stack_entry
305	blr	x1
306	irq_stack_exit
307	.endm
308
309	.text
310
311/*
312 * Exception vectors.
313 */
314	.pushsection ".entry.text", "ax"
315
316	.align	11
317ENTRY(vectors)
318	ventry	el1_sync_invalid		// Synchronous EL1t
319	ventry	el1_irq_invalid			// IRQ EL1t
320	ventry	el1_fiq_invalid			// FIQ EL1t
321	ventry	el1_error_invalid		// Error EL1t
322
323	ventry	el1_sync			// Synchronous EL1h
324	ventry	el1_irq				// IRQ EL1h
325	ventry	el1_fiq_invalid			// FIQ EL1h
326	ventry	el1_error_invalid		// Error EL1h
327
328	ventry	el0_sync			// Synchronous 64-bit EL0
329	ventry	el0_irq				// IRQ 64-bit EL0
330	ventry	el0_fiq_invalid			// FIQ 64-bit EL0
331	ventry	el0_error_invalid		// Error 64-bit EL0
332
333#ifdef CONFIG_COMPAT
334	ventry	el0_sync_compat			// Synchronous 32-bit EL0
335	ventry	el0_irq_compat			// IRQ 32-bit EL0
336	ventry	el0_fiq_invalid_compat		// FIQ 32-bit EL0
337	ventry	el0_error_invalid_compat	// Error 32-bit EL0
338#else
339	ventry	el0_sync_invalid		// Synchronous 32-bit EL0
340	ventry	el0_irq_invalid			// IRQ 32-bit EL0
341	ventry	el0_fiq_invalid			// FIQ 32-bit EL0
342	ventry	el0_error_invalid		// Error 32-bit EL0
343#endif
344END(vectors)
345
346/*
347 * Invalid mode handlers
348 */
349	.macro	inv_entry, el, reason, regsize = 64
350	kernel_entry \el, \regsize
351	mov	x0, sp
352	mov	x1, #\reason
353	mrs	x2, esr_el1
354	b	bad_mode
355	.endm
356
357el0_sync_invalid:
358	inv_entry 0, BAD_SYNC
359ENDPROC(el0_sync_invalid)
360
361el0_irq_invalid:
362	inv_entry 0, BAD_IRQ
363ENDPROC(el0_irq_invalid)
364
365el0_fiq_invalid:
366	inv_entry 0, BAD_FIQ
367ENDPROC(el0_fiq_invalid)
368
369el0_error_invalid:
370	inv_entry 0, BAD_ERROR
371ENDPROC(el0_error_invalid)
372
373#ifdef CONFIG_COMPAT
374el0_fiq_invalid_compat:
375	inv_entry 0, BAD_FIQ, 32
376ENDPROC(el0_fiq_invalid_compat)
377
378el0_error_invalid_compat:
379	inv_entry 0, BAD_ERROR, 32
380ENDPROC(el0_error_invalid_compat)
381#endif
382
383el1_sync_invalid:
384	inv_entry 1, BAD_SYNC
385ENDPROC(el1_sync_invalid)
386
387el1_irq_invalid:
388	inv_entry 1, BAD_IRQ
389ENDPROC(el1_irq_invalid)
390
391el1_fiq_invalid:
392	inv_entry 1, BAD_FIQ
393ENDPROC(el1_fiq_invalid)
394
395el1_error_invalid:
396	inv_entry 1, BAD_ERROR
397ENDPROC(el1_error_invalid)
398
399/*
400 * EL1 mode handlers.
401 */
402	.align	6
403el1_sync:
404	kernel_entry 1
405	mrs	x1, esr_el1			// read the syndrome register
406	lsr	x24, x1, #ESR_ELx_EC_SHIFT	// exception class
407	cmp	x24, #ESR_ELx_EC_DABT_CUR	// data abort in EL1
408	b.eq	el1_da
409	cmp	x24, #ESR_ELx_EC_IABT_CUR	// instruction abort in EL1
410	b.eq	el1_ia
411	cmp	x24, #ESR_ELx_EC_SYS64		// configurable trap
412	b.eq	el1_undef
413	cmp	x24, #ESR_ELx_EC_SP_ALIGN	// stack alignment exception
414	b.eq	el1_sp_pc
415	cmp	x24, #ESR_ELx_EC_PC_ALIGN	// pc alignment exception
416	b.eq	el1_sp_pc
417	cmp	x24, #ESR_ELx_EC_UNKNOWN	// unknown exception in EL1
418	b.eq	el1_undef
419	cmp	x24, #ESR_ELx_EC_BREAKPT_CUR	// debug exception in EL1
420	b.ge	el1_dbg
421	b	el1_inv
422
423el1_ia:
424	/*
425	 * Fall through to the Data abort case
426	 */
427el1_da:
428	/*
429	 * Data abort handling
430	 */
431	mrs	x3, far_el1
432	enable_dbg
433	// re-enable interrupts if they were enabled in the aborted context
434	tbnz	x23, #7, 1f			// PSR_I_BIT
435	enable_irq
4361:
437	clear_address_tag x0, x3
438	mov	x2, sp				// struct pt_regs
439	bl	do_mem_abort
440
441	// disable interrupts before pulling preserved data off the stack
442	disable_irq
443	kernel_exit 1
444el1_sp_pc:
445	/*
446	 * Stack or PC alignment exception handling
447	 */
448	mrs	x0, far_el1
449	enable_dbg
450	mov	x2, sp
451	b	do_sp_pc_abort
452el1_undef:
453	/*
454	 * Undefined instruction
455	 */
456	enable_dbg
457	mov	x0, sp
458	b	do_undefinstr
459el1_dbg:
460	/*
461	 * Debug exception handling
462	 */
463	cmp	x24, #ESR_ELx_EC_BRK64		// if BRK64
464	cinc	x24, x24, eq			// set bit '0'
465	tbz	x24, #0, el1_inv		// EL1 only
466	mrs	x0, far_el1
467	mov	x2, sp				// struct pt_regs
468	bl	do_debug_exception
469	kernel_exit 1
470el1_inv:
471	// TODO: add support for undefined instructions in kernel mode
472	enable_dbg
473	mov	x0, sp
474	mov	x2, x1
475	mov	x1, #BAD_SYNC
476	b	bad_mode
477ENDPROC(el1_sync)
478
479	.align	6
480el1_irq:
481	kernel_entry 1
482	enable_dbg
483#ifdef CONFIG_TRACE_IRQFLAGS
484	bl	trace_hardirqs_off
485#endif
486
487	irq_handler
488
489#ifdef CONFIG_PREEMPT
490	ldr	w24, [tsk, #TSK_TI_PREEMPT]	// get preempt count
491	cbnz	w24, 1f				// preempt count != 0
492	ldr	x0, [tsk, #TSK_TI_FLAGS]	// get flags
493	tbz	x0, #TIF_NEED_RESCHED, 1f	// needs rescheduling?
494	bl	el1_preempt
4951:
496#endif
497#ifdef CONFIG_TRACE_IRQFLAGS
498	bl	trace_hardirqs_on
499#endif
500	kernel_exit 1
501ENDPROC(el1_irq)
502
503#ifdef CONFIG_PREEMPT
504el1_preempt:
505	mov	x24, lr
5061:	bl	preempt_schedule_irq		// irq en/disable is done inside
507	ldr	x0, [tsk, #TSK_TI_FLAGS]	// get new tasks TI_FLAGS
508	tbnz	x0, #TIF_NEED_RESCHED, 1b	// needs rescheduling?
509	ret	x24
510#endif
511
512/*
513 * EL0 mode handlers.
514 */
515	.align	6
516el0_sync:
517	kernel_entry 0
518	mrs	x25, esr_el1			// read the syndrome register
519	lsr	x24, x25, #ESR_ELx_EC_SHIFT	// exception class
520	cmp	x24, #ESR_ELx_EC_SVC64		// SVC in 64-bit state
521	b.eq	el0_svc
522	cmp	x24, #ESR_ELx_EC_DABT_LOW	// data abort in EL0
523	b.eq	el0_da
524	cmp	x24, #ESR_ELx_EC_IABT_LOW	// instruction abort in EL0
525	b.eq	el0_ia
526	cmp	x24, #ESR_ELx_EC_FP_ASIMD	// FP/ASIMD access
527	b.eq	el0_fpsimd_acc
528	cmp	x24, #ESR_ELx_EC_FP_EXC64	// FP/ASIMD exception
529	b.eq	el0_fpsimd_exc
530	cmp	x24, #ESR_ELx_EC_SYS64		// configurable trap
531	b.eq	el0_sys
532	cmp	x24, #ESR_ELx_EC_SP_ALIGN	// stack alignment exception
533	b.eq	el0_sp_pc
534	cmp	x24, #ESR_ELx_EC_PC_ALIGN	// pc alignment exception
535	b.eq	el0_sp_pc
536	cmp	x24, #ESR_ELx_EC_UNKNOWN	// unknown exception in EL0
537	b.eq	el0_undef
538	cmp	x24, #ESR_ELx_EC_BREAKPT_LOW	// debug exception in EL0
539	b.ge	el0_dbg
540	b	el0_inv
541
542#ifdef CONFIG_COMPAT
543	.align	6
544el0_sync_compat:
545	kernel_entry 0, 32
546	mrs	x25, esr_el1			// read the syndrome register
547	lsr	x24, x25, #ESR_ELx_EC_SHIFT	// exception class
548	cmp	x24, #ESR_ELx_EC_SVC32		// SVC in 32-bit state
549	b.eq	el0_svc_compat
550	cmp	x24, #ESR_ELx_EC_DABT_LOW	// data abort in EL0
551	b.eq	el0_da
552	cmp	x24, #ESR_ELx_EC_IABT_LOW	// instruction abort in EL0
553	b.eq	el0_ia
554	cmp	x24, #ESR_ELx_EC_FP_ASIMD	// FP/ASIMD access
555	b.eq	el0_fpsimd_acc
556	cmp	x24, #ESR_ELx_EC_FP_EXC32	// FP/ASIMD exception
557	b.eq	el0_fpsimd_exc
558	cmp	x24, #ESR_ELx_EC_PC_ALIGN	// pc alignment exception
559	b.eq	el0_sp_pc
560	cmp	x24, #ESR_ELx_EC_UNKNOWN	// unknown exception in EL0
561	b.eq	el0_undef
562	cmp	x24, #ESR_ELx_EC_CP15_32	// CP15 MRC/MCR trap
563	b.eq	el0_undef
564	cmp	x24, #ESR_ELx_EC_CP15_64	// CP15 MRRC/MCRR trap
565	b.eq	el0_undef
566	cmp	x24, #ESR_ELx_EC_CP14_MR	// CP14 MRC/MCR trap
567	b.eq	el0_undef
568	cmp	x24, #ESR_ELx_EC_CP14_LS	// CP14 LDC/STC trap
569	b.eq	el0_undef
570	cmp	x24, #ESR_ELx_EC_CP14_64	// CP14 MRRC/MCRR trap
571	b.eq	el0_undef
572	cmp	x24, #ESR_ELx_EC_BREAKPT_LOW	// debug exception in EL0
573	b.ge	el0_dbg
574	b	el0_inv
575el0_svc_compat:
576	/*
577	 * AArch32 syscall handling
578	 */
579	adrp	stbl, compat_sys_call_table	// load compat syscall table pointer
580	uxtw	scno, w7			// syscall number in w7 (r7)
581	mov     sc_nr, #__NR_compat_syscalls
582	b	el0_svc_naked
583
584	.align	6
585el0_irq_compat:
586	kernel_entry 0, 32
587	b	el0_irq_naked
588#endif
589
590el0_da:
591	/*
592	 * Data abort handling
593	 */
594	mrs	x26, far_el1
595	// enable interrupts before calling the main handler
596	enable_dbg_and_irq
597	ct_user_exit
598	clear_address_tag x0, x26
599	mov	x1, x25
600	mov	x2, sp
601	bl	do_mem_abort
602	b	ret_to_user
603el0_ia:
604	/*
605	 * Instruction abort handling
606	 */
607	mrs	x26, far_el1
608	// enable interrupts before calling the main handler
609	enable_dbg_and_irq
610	ct_user_exit
611	mov	x0, x26
612	mov	x1, x25
613	mov	x2, sp
614	bl	do_mem_abort
615	b	ret_to_user
616el0_fpsimd_acc:
617	/*
618	 * Floating Point or Advanced SIMD access
619	 */
620	enable_dbg
621	ct_user_exit
622	mov	x0, x25
623	mov	x1, sp
624	bl	do_fpsimd_acc
625	b	ret_to_user
626el0_fpsimd_exc:
627	/*
628	 * Floating Point or Advanced SIMD exception
629	 */
630	enable_dbg
631	ct_user_exit
632	mov	x0, x25
633	mov	x1, sp
634	bl	do_fpsimd_exc
635	b	ret_to_user
636el0_sp_pc:
637	/*
638	 * Stack or PC alignment exception handling
639	 */
640	mrs	x26, far_el1
641	// enable interrupts before calling the main handler
642	enable_dbg_and_irq
643	ct_user_exit
644	mov	x0, x26
645	mov	x1, x25
646	mov	x2, sp
647	bl	do_sp_pc_abort
648	b	ret_to_user
649el0_undef:
650	/*
651	 * Undefined instruction
652	 */
653	// enable interrupts before calling the main handler
654	enable_dbg_and_irq
655	ct_user_exit
656	mov	x0, sp
657	bl	do_undefinstr
658	b	ret_to_user
659el0_sys:
660	/*
661	 * System instructions, for trapped cache maintenance instructions
662	 */
663	enable_dbg_and_irq
664	ct_user_exit
665	mov	x0, x25
666	mov	x1, sp
667	bl	do_sysinstr
668	b	ret_to_user
669el0_dbg:
670	/*
671	 * Debug exception handling
672	 */
673	tbnz	x24, #0, el0_inv		// EL0 only
674	mrs	x0, far_el1
675	mov	x1, x25
676	mov	x2, sp
677	bl	do_debug_exception
678	enable_dbg
679	ct_user_exit
680	b	ret_to_user
681el0_inv:
682	enable_dbg
683	ct_user_exit
684	mov	x0, sp
685	mov	x1, #BAD_SYNC
686	mov	x2, x25
687	bl	bad_el0_sync
688	b	ret_to_user
689ENDPROC(el0_sync)
690
691	.align	6
692el0_irq:
693	kernel_entry 0
694el0_irq_naked:
695	enable_dbg
696#ifdef CONFIG_TRACE_IRQFLAGS
697	bl	trace_hardirqs_off
698#endif
699
700	ct_user_exit
701	irq_handler
702
703#ifdef CONFIG_TRACE_IRQFLAGS
704	bl	trace_hardirqs_on
705#endif
706	b	ret_to_user
707ENDPROC(el0_irq)
708
709/*
710 * Register switch for AArch64. The callee-saved registers need to be saved
711 * and restored. On entry:
712 *   x0 = previous task_struct (must be preserved across the switch)
713 *   x1 = next task_struct
714 * Previous and next are guaranteed not to be the same.
715 *
716 */
717ENTRY(cpu_switch_to)
718	mov	x10, #THREAD_CPU_CONTEXT
719	add	x8, x0, x10
720	mov	x9, sp
721	stp	x19, x20, [x8], #16		// store callee-saved registers
722	stp	x21, x22, [x8], #16
723	stp	x23, x24, [x8], #16
724	stp	x25, x26, [x8], #16
725	stp	x27, x28, [x8], #16
726	stp	x29, x9, [x8], #16
727	str	lr, [x8]
728	add	x8, x1, x10
729	ldp	x19, x20, [x8], #16		// restore callee-saved registers
730	ldp	x21, x22, [x8], #16
731	ldp	x23, x24, [x8], #16
732	ldp	x25, x26, [x8], #16
733	ldp	x27, x28, [x8], #16
734	ldp	x29, x9, [x8], #16
735	ldr	lr, [x8]
736	mov	sp, x9
737	msr	sp_el0, x1
738	ret
739ENDPROC(cpu_switch_to)
740
741/*
742 * This is the fast syscall return path.  We do as little as possible here,
743 * and this includes saving x0 back into the kernel stack.
744 */
745ret_fast_syscall:
746	disable_irq				// disable interrupts
747	str	x0, [sp, #S_X0]			// returned x0
748	ldr	x1, [tsk, #TSK_TI_FLAGS]	// re-check for syscall tracing
749	and	x2, x1, #_TIF_SYSCALL_WORK
750	cbnz	x2, ret_fast_syscall_trace
751	and	x2, x1, #_TIF_WORK_MASK
752	cbnz	x2, work_pending
753	enable_step_tsk x1, x2
754	kernel_exit 0
755ret_fast_syscall_trace:
756	enable_irq				// enable interrupts
757	b	__sys_trace_return_skipped	// we already saved x0
758
759/*
760 * Ok, we need to do extra processing, enter the slow path.
761 */
762work_pending:
763	mov	x0, sp				// 'regs'
764	bl	do_notify_resume
765#ifdef CONFIG_TRACE_IRQFLAGS
766	bl	trace_hardirqs_on		// enabled while in userspace
767#endif
768	ldr	x1, [tsk, #TSK_TI_FLAGS]	// re-check for single-step
769	b	finish_ret_to_user
770/*
771 * "slow" syscall return path.
772 */
773ret_to_user:
774	disable_irq				// disable interrupts
775	ldr	x1, [tsk, #TSK_TI_FLAGS]
776	and	x2, x1, #_TIF_WORK_MASK
777	cbnz	x2, work_pending
778finish_ret_to_user:
779	enable_step_tsk x1, x2
780	kernel_exit 0
781ENDPROC(ret_to_user)
782
783/*
784 * This is how we return from a fork.
785 */
786ENTRY(ret_from_fork)
787	bl	schedule_tail
788	cbz	x19, 1f				// not a kernel thread
789	mov	x0, x20
790	blr	x19
7911:	get_thread_info tsk
792	b	ret_to_user
793ENDPROC(ret_from_fork)
794
795/*
796 * SVC handler.
797 */
798	.align	6
799el0_svc:
800	adrp	stbl, sys_call_table		// load syscall table pointer
801	uxtw	scno, w8			// syscall number in w8
802	mov	sc_nr, #__NR_syscalls
803el0_svc_naked:					// compat entry point
804	stp	x0, scno, [sp, #S_ORIG_X0]	// save the original x0 and syscall number
805	enable_dbg_and_irq
806	ct_user_exit 1
807
808	ldr	x16, [tsk, #TSK_TI_FLAGS]	// check for syscall hooks
809	tst	x16, #_TIF_SYSCALL_WORK
810	b.ne	__sys_trace
811	cmp     scno, sc_nr                     // check upper syscall limit
812	b.hs	ni_sys
813	ldr	x16, [stbl, scno, lsl #3]	// address in the syscall table
814	blr	x16				// call sys_* routine
815	b	ret_fast_syscall
816ni_sys:
817	mov	x0, sp
818	bl	do_ni_syscall
819	b	ret_fast_syscall
820ENDPROC(el0_svc)
821
822	/*
823	 * This is the really slow path.  We're going to be doing context
824	 * switches, and waiting for our parent to respond.
825	 */
826__sys_trace:
827	mov	w0, #-1				// set default errno for
828	cmp     scno, x0			// user-issued syscall(-1)
829	b.ne	1f
830	mov	x0, #-ENOSYS
831	str	x0, [sp, #S_X0]
8321:	mov	x0, sp
833	bl	syscall_trace_enter
834	cmp	w0, #-1				// skip the syscall?
835	b.eq	__sys_trace_return_skipped
836	uxtw	scno, w0			// syscall number (possibly new)
837	mov	x1, sp				// pointer to regs
838	cmp	scno, sc_nr			// check upper syscall limit
839	b.hs	__ni_sys_trace
840	ldp	x0, x1, [sp]			// restore the syscall args
841	ldp	x2, x3, [sp, #S_X2]
842	ldp	x4, x5, [sp, #S_X4]
843	ldp	x6, x7, [sp, #S_X6]
844	ldr	x16, [stbl, scno, lsl #3]	// address in the syscall table
845	blr	x16				// call sys_* routine
846
847__sys_trace_return:
848	str	x0, [sp, #S_X0]			// save returned x0
849__sys_trace_return_skipped:
850	mov	x0, sp
851	bl	syscall_trace_exit
852	b	ret_to_user
853
854__ni_sys_trace:
855	mov	x0, sp
856	bl	do_ni_syscall
857	b	__sys_trace_return
858
859	.popsection				// .entry.text
860
861/*
862 * Special system call wrappers.
863 */
864ENTRY(sys_rt_sigreturn_wrapper)
865	mov	x0, sp
866	b	sys_rt_sigreturn
867ENDPROC(sys_rt_sigreturn_wrapper)
868