xref: /openbmc/linux/arch/riscv/kernel/entry.S (revision faffb083)
1/* SPDX-License-Identifier: GPL-2.0-only */
2/*
3 * Copyright (C) 2012 Regents of the University of California
4 * Copyright (C) 2017 SiFive
5 */
6
7#include <linux/init.h>
8#include <linux/linkage.h>
9
10#include <asm/asm.h>
11#include <asm/csr.h>
12#include <asm/unistd.h>
13#include <asm/thread_info.h>
14#include <asm/asm-offsets.h>
15#include <asm/errata_list.h>
16
17#if !IS_ENABLED(CONFIG_PREEMPTION)
18.set resume_kernel, restore_all
19#endif
20
21ENTRY(handle_exception)
22	/*
23	 * If coming from userspace, preserve the user thread pointer and load
24	 * the kernel thread pointer.  If we came from the kernel, the scratch
25	 * register will contain 0, and we should continue on the current TP.
26	 */
27	csrrw tp, CSR_SCRATCH, tp
28	bnez tp, _save_context
29
30_restore_kernel_tpsp:
31	csrr tp, CSR_SCRATCH
32	REG_S sp, TASK_TI_KERNEL_SP(tp)
33
34#ifdef CONFIG_VMAP_STACK
35	addi sp, sp, -(PT_SIZE_ON_STACK)
36	srli sp, sp, THREAD_SHIFT
37	andi sp, sp, 0x1
38	bnez sp, handle_kernel_stack_overflow
39	REG_L sp, TASK_TI_KERNEL_SP(tp)
40#endif
41
42_save_context:
43	REG_S sp, TASK_TI_USER_SP(tp)
44	REG_L sp, TASK_TI_KERNEL_SP(tp)
45	addi sp, sp, -(PT_SIZE_ON_STACK)
46	REG_S x1,  PT_RA(sp)
47	REG_S x3,  PT_GP(sp)
48	REG_S x5,  PT_T0(sp)
49	REG_S x6,  PT_T1(sp)
50	REG_S x7,  PT_T2(sp)
51	REG_S x8,  PT_S0(sp)
52	REG_S x9,  PT_S1(sp)
53	REG_S x10, PT_A0(sp)
54	REG_S x11, PT_A1(sp)
55	REG_S x12, PT_A2(sp)
56	REG_S x13, PT_A3(sp)
57	REG_S x14, PT_A4(sp)
58	REG_S x15, PT_A5(sp)
59	REG_S x16, PT_A6(sp)
60	REG_S x17, PT_A7(sp)
61	REG_S x18, PT_S2(sp)
62	REG_S x19, PT_S3(sp)
63	REG_S x20, PT_S4(sp)
64	REG_S x21, PT_S5(sp)
65	REG_S x22, PT_S6(sp)
66	REG_S x23, PT_S7(sp)
67	REG_S x24, PT_S8(sp)
68	REG_S x25, PT_S9(sp)
69	REG_S x26, PT_S10(sp)
70	REG_S x27, PT_S11(sp)
71	REG_S x28, PT_T3(sp)
72	REG_S x29, PT_T4(sp)
73	REG_S x30, PT_T5(sp)
74	REG_S x31, PT_T6(sp)
75
76	/*
77	 * Disable user-mode memory access as it should only be set in the
78	 * actual user copy routines.
79	 *
80	 * Disable the FPU to detect illegal usage of floating point in kernel
81	 * space.
82	 */
83	li t0, SR_SUM | SR_FS
84
85	REG_L s0, TASK_TI_USER_SP(tp)
86	csrrc s1, CSR_STATUS, t0
87	csrr s2, CSR_EPC
88	csrr s3, CSR_TVAL
89	csrr s4, CSR_CAUSE
90	csrr s5, CSR_SCRATCH
91	REG_S s0, PT_SP(sp)
92	REG_S s1, PT_STATUS(sp)
93	REG_S s2, PT_EPC(sp)
94	REG_S s3, PT_BADADDR(sp)
95	REG_S s4, PT_CAUSE(sp)
96	REG_S s5, PT_TP(sp)
97
98	/*
99	 * Set the scratch register to 0, so that if a recursive exception
100	 * occurs, the exception vector knows it came from the kernel
101	 */
102	csrw CSR_SCRATCH, x0
103
104	/* Load the global pointer */
105.option push
106.option norelax
107	la gp, __global_pointer$
108.option pop
109
110#ifdef CONFIG_TRACE_IRQFLAGS
111	call __trace_hardirqs_off
112#endif
113
114#ifdef CONFIG_CONTEXT_TRACKING_USER
115	/* If previous state is in user mode, call user_exit_callable(). */
116	li   a0, SR_PP
117	and a0, s1, a0
118	bnez a0, skip_context_tracking
119	call user_exit_callable
120skip_context_tracking:
121#endif
122
123	/*
124	 * MSB of cause differentiates between
125	 * interrupts and exceptions
126	 */
127	bge s4, zero, 1f
128
129	la ra, ret_from_exception
130
131	/* Handle interrupts */
132	move a0, sp /* pt_regs */
133	la a1, generic_handle_arch_irq
134	jr a1
1351:
136	/*
137	 * Exceptions run with interrupts enabled or disabled depending on the
138	 * state of SR_PIE in m/sstatus.
139	 */
140	andi t0, s1, SR_PIE
141	beqz t0, 1f
142	/* kprobes, entered via ebreak, must have interrupts disabled. */
143	li t0, EXC_BREAKPOINT
144	beq s4, t0, 1f
145#ifdef CONFIG_TRACE_IRQFLAGS
146	call __trace_hardirqs_on
147#endif
148	csrs CSR_STATUS, SR_IE
149
1501:
151	la ra, ret_from_exception
152	/* Handle syscalls */
153	li t0, EXC_SYSCALL
154	beq s4, t0, handle_syscall
155
156	/* Handle other exceptions */
157	slli t0, s4, RISCV_LGPTR
158	la t1, excp_vect_table
159	la t2, excp_vect_table_end
160	move a0, sp /* pt_regs */
161	add t0, t1, t0
162	/* Check if exception code lies within bounds */
163	bgeu t0, t2, 1f
164	REG_L t0, 0(t0)
165	jr t0
1661:
167	tail do_trap_unknown
168
169handle_syscall:
170#ifdef CONFIG_RISCV_M_MODE
171	/*
172	 * When running is M-Mode (no MMU config), MPIE does not get set.
173	 * As a result, we need to force enable interrupts here because
174	 * handle_exception did not do set SR_IE as it always sees SR_PIE
175	 * being cleared.
176	 */
177	csrs CSR_STATUS, SR_IE
178#endif
179#if defined(CONFIG_TRACE_IRQFLAGS) || defined(CONFIG_CONTEXT_TRACKING_USER)
180	/* Recover a0 - a7 for system calls */
181	REG_L a0, PT_A0(sp)
182	REG_L a1, PT_A1(sp)
183	REG_L a2, PT_A2(sp)
184	REG_L a3, PT_A3(sp)
185	REG_L a4, PT_A4(sp)
186	REG_L a5, PT_A5(sp)
187	REG_L a6, PT_A6(sp)
188	REG_L a7, PT_A7(sp)
189#endif
190	 /* save the initial A0 value (needed in signal handlers) */
191	REG_S a0, PT_ORIG_A0(sp)
192	/*
193	 * Advance SEPC to avoid executing the original
194	 * scall instruction on sret
195	 */
196	addi s2, s2, 0x4
197	REG_S s2, PT_EPC(sp)
198	/* Trace syscalls, but only if requested by the user. */
199	REG_L t0, TASK_TI_FLAGS(tp)
200	andi t0, t0, _TIF_SYSCALL_WORK
201	bnez t0, handle_syscall_trace_enter
202check_syscall_nr:
203	/* Check to make sure we don't jump to a bogus syscall number. */
204	li t0, __NR_syscalls
205	la s0, sys_ni_syscall
206	/*
207	 * Syscall number held in a7.
208	 * If syscall number is above allowed value, redirect to ni_syscall.
209	 */
210	bgeu a7, t0, 3f
211#ifdef CONFIG_COMPAT
212	REG_L s0, PT_STATUS(sp)
213	srli s0, s0, SR_UXL_SHIFT
214	andi s0, s0, (SR_UXL >> SR_UXL_SHIFT)
215	li t0, (SR_UXL_32 >> SR_UXL_SHIFT)
216	sub t0, s0, t0
217	bnez t0, 1f
218
219	/* Call compat_syscall */
220	la s0, compat_sys_call_table
221	j 2f
2221:
223#endif
224	/* Call syscall */
225	la s0, sys_call_table
2262:
227	slli t0, a7, RISCV_LGPTR
228	add s0, s0, t0
229	REG_L s0, 0(s0)
2303:
231	jalr s0
232
233ret_from_syscall:
234	/* Set user a0 to kernel a0 */
235	REG_S a0, PT_A0(sp)
236	/*
237	 * We didn't execute the actual syscall.
238	 * Seccomp already set return value for the current task pt_regs.
239	 * (If it was configured with SECCOMP_RET_ERRNO/TRACE)
240	 */
241ret_from_syscall_rejected:
242#ifdef CONFIG_DEBUG_RSEQ
243	move a0, sp
244	call rseq_syscall
245#endif
246	/* Trace syscalls, but only if requested by the user. */
247	REG_L t0, TASK_TI_FLAGS(tp)
248	andi t0, t0, _TIF_SYSCALL_WORK
249	bnez t0, handle_syscall_trace_exit
250
251SYM_CODE_START_NOALIGN(ret_from_exception)
252	REG_L s0, PT_STATUS(sp)
253	csrc CSR_STATUS, SR_IE
254#ifdef CONFIG_TRACE_IRQFLAGS
255	call __trace_hardirqs_off
256#endif
257#ifdef CONFIG_RISCV_M_MODE
258	/* the MPP value is too large to be used as an immediate arg for addi */
259	li t0, SR_MPP
260	and s0, s0, t0
261#else
262	andi s0, s0, SR_SPP
263#endif
264	bnez s0, resume_kernel
265SYM_CODE_END(ret_from_exception)
266
267	/* Interrupts must be disabled here so flags are checked atomically */
268	REG_L s0, TASK_TI_FLAGS(tp) /* current_thread_info->flags */
269	andi s1, s0, _TIF_WORK_MASK
270	bnez s1, resume_userspace_slow
271resume_userspace:
272#ifdef CONFIG_CONTEXT_TRACKING_USER
273	call user_enter_callable
274#endif
275
276	/* Save unwound kernel stack pointer in thread_info */
277	addi s0, sp, PT_SIZE_ON_STACK
278	REG_S s0, TASK_TI_KERNEL_SP(tp)
279
280	/*
281	 * Save TP into the scratch register , so we can find the kernel data
282	 * structures again.
283	 */
284	csrw CSR_SCRATCH, tp
285
286restore_all:
287#ifdef CONFIG_TRACE_IRQFLAGS
288	REG_L s1, PT_STATUS(sp)
289	andi t0, s1, SR_PIE
290	beqz t0, 1f
291	call __trace_hardirqs_on
292	j 2f
2931:
294	call __trace_hardirqs_off
2952:
296#endif
297	REG_L a0, PT_STATUS(sp)
298	/*
299	 * The current load reservation is effectively part of the processor's
300	 * state, in the sense that load reservations cannot be shared between
301	 * different hart contexts.  We can't actually save and restore a load
302	 * reservation, so instead here we clear any existing reservation --
303	 * it's always legal for implementations to clear load reservations at
304	 * any point (as long as the forward progress guarantee is kept, but
305	 * we'll ignore that here).
306	 *
307	 * Dangling load reservations can be the result of taking a trap in the
308	 * middle of an LR/SC sequence, but can also be the result of a taken
309	 * forward branch around an SC -- which is how we implement CAS.  As a
310	 * result we need to clear reservations between the last CAS and the
311	 * jump back to the new context.  While it is unlikely the store
312	 * completes, implementations are allowed to expand reservations to be
313	 * arbitrarily large.
314	 */
315	REG_L  a2, PT_EPC(sp)
316	REG_SC x0, a2, PT_EPC(sp)
317
318	csrw CSR_STATUS, a0
319	csrw CSR_EPC, a2
320
321	REG_L x1,  PT_RA(sp)
322	REG_L x3,  PT_GP(sp)
323	REG_L x4,  PT_TP(sp)
324	REG_L x5,  PT_T0(sp)
325	REG_L x6,  PT_T1(sp)
326	REG_L x7,  PT_T2(sp)
327	REG_L x8,  PT_S0(sp)
328	REG_L x9,  PT_S1(sp)
329	REG_L x10, PT_A0(sp)
330	REG_L x11, PT_A1(sp)
331	REG_L x12, PT_A2(sp)
332	REG_L x13, PT_A3(sp)
333	REG_L x14, PT_A4(sp)
334	REG_L x15, PT_A5(sp)
335	REG_L x16, PT_A6(sp)
336	REG_L x17, PT_A7(sp)
337	REG_L x18, PT_S2(sp)
338	REG_L x19, PT_S3(sp)
339	REG_L x20, PT_S4(sp)
340	REG_L x21, PT_S5(sp)
341	REG_L x22, PT_S6(sp)
342	REG_L x23, PT_S7(sp)
343	REG_L x24, PT_S8(sp)
344	REG_L x25, PT_S9(sp)
345	REG_L x26, PT_S10(sp)
346	REG_L x27, PT_S11(sp)
347	REG_L x28, PT_T3(sp)
348	REG_L x29, PT_T4(sp)
349	REG_L x30, PT_T5(sp)
350	REG_L x31, PT_T6(sp)
351
352	REG_L x2,  PT_SP(sp)
353
354#ifdef CONFIG_RISCV_M_MODE
355	mret
356#else
357	sret
358#endif
359
360#if IS_ENABLED(CONFIG_PREEMPTION)
361resume_kernel:
362	REG_L s0, TASK_TI_PREEMPT_COUNT(tp)
363	bnez s0, restore_all
364	REG_L s0, TASK_TI_FLAGS(tp)
365	andi s0, s0, _TIF_NEED_RESCHED
366	beqz s0, restore_all
367	call preempt_schedule_irq
368	j restore_all
369#endif
370
371resume_userspace_slow:
372	/* Enter slow path for supplementary processing */
373	move a0, sp /* pt_regs */
374	move a1, s0 /* current_thread_info->flags */
375	call do_work_pending
376	j resume_userspace
377
378/* Slow paths for ptrace. */
379handle_syscall_trace_enter:
380	move a0, sp
381	call do_syscall_trace_enter
382	move t0, a0
383	REG_L a0, PT_A0(sp)
384	REG_L a1, PT_A1(sp)
385	REG_L a2, PT_A2(sp)
386	REG_L a3, PT_A3(sp)
387	REG_L a4, PT_A4(sp)
388	REG_L a5, PT_A5(sp)
389	REG_L a6, PT_A6(sp)
390	REG_L a7, PT_A7(sp)
391	bnez t0, ret_from_syscall_rejected
392	j check_syscall_nr
393handle_syscall_trace_exit:
394	move a0, sp
395	call do_syscall_trace_exit
396	j ret_from_exception
397
398#ifdef CONFIG_VMAP_STACK
399handle_kernel_stack_overflow:
400	/*
401	 * Takes the psuedo-spinlock for the shadow stack, in case multiple
402	 * harts are concurrently overflowing their kernel stacks.  We could
403	 * store any value here, but since we're overflowing the kernel stack
404	 * already we only have SP to use as a scratch register.  So we just
405	 * swap in the address of the spinlock, as that's definately non-zero.
406	 *
407	 * Pairs with a store_release in handle_bad_stack().
408	 */
4091:	la sp, spin_shadow_stack
410	REG_AMOSWAP_AQ sp, sp, (sp)
411	bnez sp, 1b
412
413	la sp, shadow_stack
414	addi sp, sp, SHADOW_OVERFLOW_STACK_SIZE
415
416	//save caller register to shadow stack
417	addi sp, sp, -(PT_SIZE_ON_STACK)
418	REG_S x1,  PT_RA(sp)
419	REG_S x5,  PT_T0(sp)
420	REG_S x6,  PT_T1(sp)
421	REG_S x7,  PT_T2(sp)
422	REG_S x10, PT_A0(sp)
423	REG_S x11, PT_A1(sp)
424	REG_S x12, PT_A2(sp)
425	REG_S x13, PT_A3(sp)
426	REG_S x14, PT_A4(sp)
427	REG_S x15, PT_A5(sp)
428	REG_S x16, PT_A6(sp)
429	REG_S x17, PT_A7(sp)
430	REG_S x28, PT_T3(sp)
431	REG_S x29, PT_T4(sp)
432	REG_S x30, PT_T5(sp)
433	REG_S x31, PT_T6(sp)
434
435	la ra, restore_caller_reg
436	tail get_overflow_stack
437
438restore_caller_reg:
439	//save per-cpu overflow stack
440	REG_S a0, -8(sp)
441	//restore caller register from shadow_stack
442	REG_L x1,  PT_RA(sp)
443	REG_L x5,  PT_T0(sp)
444	REG_L x6,  PT_T1(sp)
445	REG_L x7,  PT_T2(sp)
446	REG_L x10, PT_A0(sp)
447	REG_L x11, PT_A1(sp)
448	REG_L x12, PT_A2(sp)
449	REG_L x13, PT_A3(sp)
450	REG_L x14, PT_A4(sp)
451	REG_L x15, PT_A5(sp)
452	REG_L x16, PT_A6(sp)
453	REG_L x17, PT_A7(sp)
454	REG_L x28, PT_T3(sp)
455	REG_L x29, PT_T4(sp)
456	REG_L x30, PT_T5(sp)
457	REG_L x31, PT_T6(sp)
458
459	//load per-cpu overflow stack
460	REG_L sp, -8(sp)
461	addi sp, sp, -(PT_SIZE_ON_STACK)
462
463	//save context to overflow stack
464	REG_S x1,  PT_RA(sp)
465	REG_S x3,  PT_GP(sp)
466	REG_S x5,  PT_T0(sp)
467	REG_S x6,  PT_T1(sp)
468	REG_S x7,  PT_T2(sp)
469	REG_S x8,  PT_S0(sp)
470	REG_S x9,  PT_S1(sp)
471	REG_S x10, PT_A0(sp)
472	REG_S x11, PT_A1(sp)
473	REG_S x12, PT_A2(sp)
474	REG_S x13, PT_A3(sp)
475	REG_S x14, PT_A4(sp)
476	REG_S x15, PT_A5(sp)
477	REG_S x16, PT_A6(sp)
478	REG_S x17, PT_A7(sp)
479	REG_S x18, PT_S2(sp)
480	REG_S x19, PT_S3(sp)
481	REG_S x20, PT_S4(sp)
482	REG_S x21, PT_S5(sp)
483	REG_S x22, PT_S6(sp)
484	REG_S x23, PT_S7(sp)
485	REG_S x24, PT_S8(sp)
486	REG_S x25, PT_S9(sp)
487	REG_S x26, PT_S10(sp)
488	REG_S x27, PT_S11(sp)
489	REG_S x28, PT_T3(sp)
490	REG_S x29, PT_T4(sp)
491	REG_S x30, PT_T5(sp)
492	REG_S x31, PT_T6(sp)
493
494	REG_L s0, TASK_TI_KERNEL_SP(tp)
495	csrr s1, CSR_STATUS
496	csrr s2, CSR_EPC
497	csrr s3, CSR_TVAL
498	csrr s4, CSR_CAUSE
499	csrr s5, CSR_SCRATCH
500	REG_S s0, PT_SP(sp)
501	REG_S s1, PT_STATUS(sp)
502	REG_S s2, PT_EPC(sp)
503	REG_S s3, PT_BADADDR(sp)
504	REG_S s4, PT_CAUSE(sp)
505	REG_S s5, PT_TP(sp)
506	move a0, sp
507	tail handle_bad_stack
508#endif
509
510END(handle_exception)
511
512ENTRY(ret_from_fork)
513	la ra, ret_from_exception
514	tail schedule_tail
515ENDPROC(ret_from_fork)
516
517ENTRY(ret_from_kernel_thread)
518	call schedule_tail
519	/* Call fn(arg) */
520	la ra, ret_from_exception
521	move a0, s1
522	jr s0
523ENDPROC(ret_from_kernel_thread)
524
525
526/*
527 * Integer register context switch
528 * The callee-saved registers must be saved and restored.
529 *
530 *   a0: previous task_struct (must be preserved across the switch)
531 *   a1: next task_struct
532 *
533 * The value of a0 and a1 must be preserved by this function, as that's how
534 * arguments are passed to schedule_tail.
535 */
536ENTRY(__switch_to)
537	/* Save context into prev->thread */
538	li    a4,  TASK_THREAD_RA
539	add   a3, a0, a4
540	add   a4, a1, a4
541	REG_S ra,  TASK_THREAD_RA_RA(a3)
542	REG_S sp,  TASK_THREAD_SP_RA(a3)
543	REG_S s0,  TASK_THREAD_S0_RA(a3)
544	REG_S s1,  TASK_THREAD_S1_RA(a3)
545	REG_S s2,  TASK_THREAD_S2_RA(a3)
546	REG_S s3,  TASK_THREAD_S3_RA(a3)
547	REG_S s4,  TASK_THREAD_S4_RA(a3)
548	REG_S s5,  TASK_THREAD_S5_RA(a3)
549	REG_S s6,  TASK_THREAD_S6_RA(a3)
550	REG_S s7,  TASK_THREAD_S7_RA(a3)
551	REG_S s8,  TASK_THREAD_S8_RA(a3)
552	REG_S s9,  TASK_THREAD_S9_RA(a3)
553	REG_S s10, TASK_THREAD_S10_RA(a3)
554	REG_S s11, TASK_THREAD_S11_RA(a3)
555	/* Restore context from next->thread */
556	REG_L ra,  TASK_THREAD_RA_RA(a4)
557	REG_L sp,  TASK_THREAD_SP_RA(a4)
558	REG_L s0,  TASK_THREAD_S0_RA(a4)
559	REG_L s1,  TASK_THREAD_S1_RA(a4)
560	REG_L s2,  TASK_THREAD_S2_RA(a4)
561	REG_L s3,  TASK_THREAD_S3_RA(a4)
562	REG_L s4,  TASK_THREAD_S4_RA(a4)
563	REG_L s5,  TASK_THREAD_S5_RA(a4)
564	REG_L s6,  TASK_THREAD_S6_RA(a4)
565	REG_L s7,  TASK_THREAD_S7_RA(a4)
566	REG_L s8,  TASK_THREAD_S8_RA(a4)
567	REG_L s9,  TASK_THREAD_S9_RA(a4)
568	REG_L s10, TASK_THREAD_S10_RA(a4)
569	REG_L s11, TASK_THREAD_S11_RA(a4)
570	/* The offset of thread_info in task_struct is zero. */
571	move tp, a1
572	ret
573ENDPROC(__switch_to)
574
575#ifndef CONFIG_MMU
576#define do_page_fault do_trap_unknown
577#endif
578
579	.section ".rodata"
580	.align LGREG
581	/* Exception vector table */
582ENTRY(excp_vect_table)
583	RISCV_PTR do_trap_insn_misaligned
584	ALT_INSN_FAULT(RISCV_PTR do_trap_insn_fault)
585	RISCV_PTR do_trap_insn_illegal
586	RISCV_PTR do_trap_break
587	RISCV_PTR do_trap_load_misaligned
588	RISCV_PTR do_trap_load_fault
589	RISCV_PTR do_trap_store_misaligned
590	RISCV_PTR do_trap_store_fault
591	RISCV_PTR do_trap_ecall_u /* system call, gets intercepted */
592	RISCV_PTR do_trap_ecall_s
593	RISCV_PTR do_trap_unknown
594	RISCV_PTR do_trap_ecall_m
595	/* instruciton page fault */
596	ALT_PAGE_FAULT(RISCV_PTR do_page_fault)
597	RISCV_PTR do_page_fault   /* load page fault */
598	RISCV_PTR do_trap_unknown
599	RISCV_PTR do_page_fault   /* store page fault */
600excp_vect_table_end:
601END(excp_vect_table)
602
603#ifndef CONFIG_MMU
604ENTRY(__user_rt_sigreturn)
605	li a7, __NR_rt_sigreturn
606	scall
607END(__user_rt_sigreturn)
608#endif
609