1#include <asm/asm-offsets.h>
2#include <asm/bug.h>
3#ifdef CONFIG_PPC_BOOK3S
4#include <asm/exception-64s.h>
5#else
6#include <asm/exception-64e.h>
7#endif
8#include <asm/feature-fixups.h>
9#include <asm/head-64.h>
10#include <asm/hw_irq.h>
11#include <asm/kup.h>
12#include <asm/mmu.h>
13#include <asm/ppc_asm.h>
14#include <asm/ptrace.h>
15
16	.align 7
17
18.macro DEBUG_SRR_VALID srr
19#ifdef CONFIG_PPC_RFI_SRR_DEBUG
20	.ifc \srr,srr
21	mfspr	r11,SPRN_SRR0
22	ld	r12,_NIP(r1)
23	clrrdi  r11,r11,2
24	clrrdi  r12,r12,2
25100:	tdne	r11,r12
26	EMIT_WARN_ENTRY 100b,__FILE__,__LINE__,(BUGFLAG_WARNING | BUGFLAG_ONCE)
27	mfspr	r11,SPRN_SRR1
28	ld	r12,_MSR(r1)
29100:	tdne	r11,r12
30	EMIT_WARN_ENTRY 100b,__FILE__,__LINE__,(BUGFLAG_WARNING | BUGFLAG_ONCE)
31	.else
32	mfspr	r11,SPRN_HSRR0
33	ld	r12,_NIP(r1)
34	clrrdi  r11,r11,2
35	clrrdi  r12,r12,2
36100:	tdne	r11,r12
37	EMIT_WARN_ENTRY 100b,__FILE__,__LINE__,(BUGFLAG_WARNING | BUGFLAG_ONCE)
38	mfspr	r11,SPRN_HSRR1
39	ld	r12,_MSR(r1)
40100:	tdne	r11,r12
41	EMIT_WARN_ENTRY 100b,__FILE__,__LINE__,(BUGFLAG_WARNING | BUGFLAG_ONCE)
42	.endif
43#endif
44.endm
45
46#ifdef CONFIG_PPC_BOOK3S
47.macro system_call_vectored name trapnr
48	.globl system_call_vectored_\name
49system_call_vectored_\name:
50_ASM_NOKPROBE_SYMBOL(system_call_vectored_\name)
51	SCV_INTERRUPT_TO_KERNEL
52	mr	r10,r1
53	ld	r1,PACAKSAVE(r13)
54	std	r10,0(r1)
55	std	r11,_NIP(r1)
56	std	r12,_MSR(r1)
57	std	r0,GPR0(r1)
58	std	r10,GPR1(r1)
59	std	r2,GPR2(r1)
60	LOAD_PACA_TOC()
61	mfcr	r12
62	li	r11,0
63	/* Save syscall parameters in r3-r8 */
64	SAVE_GPRS(3, 8, r1)
65	/* Zero r9-r12, this should only be required when restoring all GPRs */
66	std	r11,GPR9(r1)
67	std	r11,GPR10(r1)
68	std	r11,GPR11(r1)
69	std	r11,GPR12(r1)
70	std	r9,GPR13(r1)
71	SAVE_NVGPRS(r1)
72	std	r11,_XER(r1)
73	std	r11,_LINK(r1)
74	std	r11,_CTR(r1)
75
76	li	r11,\trapnr
77	std	r11,_TRAP(r1)
78	std	r12,_CCR(r1)
79	std	r3,ORIG_GPR3(r1)
80	LOAD_REG_IMMEDIATE(r11, STACK_FRAME_REGS_MARKER)
81	std	r11,STACK_INT_FRAME_MARKER(r1)		/* "regs" marker */
82	/* Calling convention has r3 = regs, r4 = orig r0 */
83	addi	r3,r1,STACK_INT_FRAME_REGS
84	mr	r4,r0
85
86BEGIN_FTR_SECTION
87	HMT_MEDIUM
88END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
89
90	/*
91	 * scv enters with MSR[EE]=1 and is immediately considered soft-masked.
92	 * The entry vector already sets PACAIRQSOFTMASK to IRQS_ALL_DISABLED,
93	 * and interrupts may be masked and pending already.
94	 * system_call_exception() will call trace_hardirqs_off() which means
95	 * interrupts could already have been blocked before trace_hardirqs_off,
96	 * but this is the best we can do.
97	 */
98
99	/*
100	 * Zero user registers to prevent influencing speculative execution
101	 * state of kernel code.
102	 */
103	SANITIZE_SYSCALL_GPRS()
104	bl	CFUNC(system_call_exception)
105
106.Lsyscall_vectored_\name\()_exit:
107	addi	r4,r1,STACK_INT_FRAME_REGS
108	li	r5,1 /* scv */
109	bl	CFUNC(syscall_exit_prepare)
110	std	r1,PACA_EXIT_SAVE_R1(r13) /* save r1 for restart */
111.Lsyscall_vectored_\name\()_rst_start:
112	lbz	r11,PACAIRQHAPPENED(r13)
113	andi.	r11,r11,(~PACA_IRQ_HARD_DIS)@l
114	bne-	syscall_vectored_\name\()_restart
115	li	r11,IRQS_ENABLED
116	stb	r11,PACAIRQSOFTMASK(r13)
117	li	r11,0
118	stb	r11,PACAIRQHAPPENED(r13) # clear out possible HARD_DIS
119
120	ld	r2,_CCR(r1)
121	ld	r4,_NIP(r1)
122	ld	r5,_MSR(r1)
123
124BEGIN_FTR_SECTION
125	stdcx.	r0,0,r1			/* to clear the reservation */
126END_FTR_SECTION_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS)
127
128BEGIN_FTR_SECTION
129	HMT_MEDIUM_LOW
130END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
131
132	SANITIZE_RESTORE_NVGPRS()
133	cmpdi	r3,0
134	bne	.Lsyscall_vectored_\name\()_restore_regs
135
136	/* rfscv returns with LR->NIA and CTR->MSR */
137	mtlr	r4
138	mtctr	r5
139
140	/* Could zero these as per ABI, but we may consider a stricter ABI
141	 * which preserves these if libc implementations can benefit, so
142	 * restore them for now until further measurement is done. */
143	REST_GPR(0, r1)
144	REST_GPRS(4, 8, r1)
145	/* Zero volatile regs that may contain sensitive kernel data */
146	ZEROIZE_GPRS(9, 12)
147	mtspr	SPRN_XER,r0
148
149	/*
150	 * We don't need to restore AMR on the way back to userspace for KUAP.
151	 * The value of AMR only matters while we're in the kernel.
152	 */
153	mtcr	r2
154	REST_GPRS(2, 3, r1)
155	REST_GPR(13, r1)
156	REST_GPR(1, r1)
157	RFSCV_TO_USER
158	b	.	/* prevent speculative execution */
159
160.Lsyscall_vectored_\name\()_restore_regs:
161	mtspr	SPRN_SRR0,r4
162	mtspr	SPRN_SRR1,r5
163
164	ld	r3,_CTR(r1)
165	ld	r4,_LINK(r1)
166	ld	r5,_XER(r1)
167
168	HANDLER_RESTORE_NVGPRS()
169	REST_GPR(0, r1)
170	mtcr	r2
171	mtctr	r3
172	mtlr	r4
173	mtspr	SPRN_XER,r5
174	REST_GPRS(2, 13, r1)
175	REST_GPR(1, r1)
176	RFI_TO_USER
177.Lsyscall_vectored_\name\()_rst_end:
178
179syscall_vectored_\name\()_restart:
180_ASM_NOKPROBE_SYMBOL(syscall_vectored_\name\()_restart)
181	GET_PACA(r13)
182	ld	r1,PACA_EXIT_SAVE_R1(r13)
183	LOAD_PACA_TOC()
184	ld	r3,RESULT(r1)
185	addi	r4,r1,STACK_INT_FRAME_REGS
186	li	r11,IRQS_ALL_DISABLED
187	stb	r11,PACAIRQSOFTMASK(r13)
188	bl	CFUNC(syscall_exit_restart)
189	std	r1,PACA_EXIT_SAVE_R1(r13) /* save r1 for restart */
190	b	.Lsyscall_vectored_\name\()_rst_start
1911:
192
193SOFT_MASK_TABLE(.Lsyscall_vectored_\name\()_rst_start, 1b)
194RESTART_TABLE(.Lsyscall_vectored_\name\()_rst_start, .Lsyscall_vectored_\name\()_rst_end, syscall_vectored_\name\()_restart)
195
196.endm
197
198system_call_vectored common 0x3000
199
200/*
201 * We instantiate another entry copy for the SIGILL variant, with TRAP=0x7ff0
202 * which is tested by system_call_exception when r0 is -1 (as set by vector
203 * entry code).
204 */
205system_call_vectored sigill 0x7ff0
206
207#endif /* CONFIG_PPC_BOOK3S */
208
209	.balign IFETCH_ALIGN_BYTES
210	.globl system_call_common_real
211system_call_common_real:
212_ASM_NOKPROBE_SYMBOL(system_call_common_real)
213	ld	r10,PACAKMSR(r13)	/* get MSR value for kernel */
214	mtmsrd	r10
215
216	.balign IFETCH_ALIGN_BYTES
217	.globl system_call_common
218system_call_common:
219_ASM_NOKPROBE_SYMBOL(system_call_common)
220	mr	r10,r1
221	ld	r1,PACAKSAVE(r13)
222	std	r10,0(r1)
223	std	r11,_NIP(r1)
224	std	r12,_MSR(r1)
225	std	r0,GPR0(r1)
226	std	r10,GPR1(r1)
227	std	r2,GPR2(r1)
228#ifdef CONFIG_PPC_E500
229START_BTB_FLUSH_SECTION
230	BTB_FLUSH(r10)
231END_BTB_FLUSH_SECTION
232#endif
233	LOAD_PACA_TOC()
234	mfcr	r12
235	li	r11,0
236	/* Save syscall parameters in r3-r8 */
237	SAVE_GPRS(3, 8, r1)
238	/* Zero r9-r12, this should only be required when restoring all GPRs */
239	std	r11,GPR9(r1)
240	std	r11,GPR10(r1)
241	std	r11,GPR11(r1)
242	std	r11,GPR12(r1)
243	std	r9,GPR13(r1)
244	SAVE_NVGPRS(r1)
245	std	r11,_XER(r1)
246	std	r11,_CTR(r1)
247	mflr	r10
248
249	/*
250	 * This clears CR0.SO (bit 28), which is the error indication on
251	 * return from this system call.
252	 */
253	rldimi	r12,r11,28,(63-28)
254	li	r11,0xc00
255	std	r10,_LINK(r1)
256	std	r11,_TRAP(r1)
257	std	r12,_CCR(r1)
258	std	r3,ORIG_GPR3(r1)
259	LOAD_REG_IMMEDIATE(r11, STACK_FRAME_REGS_MARKER)
260	std	r11,STACK_INT_FRAME_MARKER(r1)		/* "regs" marker */
261	/* Calling convention has r3 = regs, r4 = orig r0 */
262	addi	r3,r1,STACK_INT_FRAME_REGS
263	mr	r4,r0
264
265#ifdef CONFIG_PPC_BOOK3S
266	li	r11,1
267	stb	r11,PACASRR_VALID(r13)
268#endif
269
270	/*
271	 * We always enter kernel from userspace with irq soft-mask enabled and
272	 * nothing pending. system_call_exception() will call
273	 * trace_hardirqs_off().
274	 */
275	li	r11,IRQS_ALL_DISABLED
276	stb	r11,PACAIRQSOFTMASK(r13)
277#ifdef CONFIG_PPC_BOOK3S
278	li	r12,-1 /* Set MSR_EE and MSR_RI */
279	mtmsrd	r12,1
280#else
281	wrteei	1
282#endif
283
284	/*
285	 * Zero user registers to prevent influencing speculative execution
286	 * state of kernel code.
287	 */
288	SANITIZE_SYSCALL_GPRS()
289	bl	CFUNC(system_call_exception)
290
291.Lsyscall_exit:
292	addi	r4,r1,STACK_INT_FRAME_REGS
293	li	r5,0 /* !scv */
294	bl	CFUNC(syscall_exit_prepare)
295	std	r1,PACA_EXIT_SAVE_R1(r13) /* save r1 for restart */
296#ifdef CONFIG_PPC_BOOK3S
297.Lsyscall_rst_start:
298	lbz	r11,PACAIRQHAPPENED(r13)
299	andi.	r11,r11,(~PACA_IRQ_HARD_DIS)@l
300	bne-	syscall_restart
301#endif
302	li	r11,IRQS_ENABLED
303	stb	r11,PACAIRQSOFTMASK(r13)
304	li	r11,0
305	stb	r11,PACAIRQHAPPENED(r13) # clear out possible HARD_DIS
306
307	ld	r2,_CCR(r1)
308	ld	r6,_LINK(r1)
309	mtlr	r6
310
311#ifdef CONFIG_PPC_BOOK3S
312	lbz	r4,PACASRR_VALID(r13)
313	cmpdi	r4,0
314	bne	1f
315	li	r4,0
316	stb	r4,PACASRR_VALID(r13)
317#endif
318	ld	r4,_NIP(r1)
319	ld	r5,_MSR(r1)
320	mtspr	SPRN_SRR0,r4
321	mtspr	SPRN_SRR1,r5
3221:
323	DEBUG_SRR_VALID srr
324
325BEGIN_FTR_SECTION
326	stdcx.	r0,0,r1			/* to clear the reservation */
327END_FTR_SECTION_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS)
328
329	SANITIZE_RESTORE_NVGPRS()
330	cmpdi	r3,0
331	bne	.Lsyscall_restore_regs
332	/* Zero volatile regs that may contain sensitive kernel data */
333	ZEROIZE_GPR(0)
334	ZEROIZE_GPRS(4, 12)
335	mtctr	r0
336	mtspr	SPRN_XER,r0
337.Lsyscall_restore_regs_cont:
338
339BEGIN_FTR_SECTION
340	HMT_MEDIUM_LOW
341END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
342
343	/*
344	 * We don't need to restore AMR on the way back to userspace for KUAP.
345	 * The value of AMR only matters while we're in the kernel.
346	 */
347	mtcr	r2
348	REST_GPRS(2, 3, r1)
349	REST_GPR(13, r1)
350	REST_GPR(1, r1)
351	RFI_TO_USER
352	b	.	/* prevent speculative execution */
353
354.Lsyscall_restore_regs:
355	ld	r3,_CTR(r1)
356	ld	r4,_XER(r1)
357	HANDLER_RESTORE_NVGPRS()
358	mtctr	r3
359	mtspr	SPRN_XER,r4
360	REST_GPR(0, r1)
361	REST_GPRS(4, 12, r1)
362	b	.Lsyscall_restore_regs_cont
363.Lsyscall_rst_end:
364
365#ifdef CONFIG_PPC_BOOK3S
366syscall_restart:
367_ASM_NOKPROBE_SYMBOL(syscall_restart)
368	GET_PACA(r13)
369	ld	r1,PACA_EXIT_SAVE_R1(r13)
370	LOAD_PACA_TOC()
371	ld	r3,RESULT(r1)
372	addi	r4,r1,STACK_INT_FRAME_REGS
373	li	r11,IRQS_ALL_DISABLED
374	stb	r11,PACAIRQSOFTMASK(r13)
375	bl	CFUNC(syscall_exit_restart)
376	std	r1,PACA_EXIT_SAVE_R1(r13) /* save r1 for restart */
377	b	.Lsyscall_rst_start
3781:
379
380SOFT_MASK_TABLE(.Lsyscall_rst_start, 1b)
381RESTART_TABLE(.Lsyscall_rst_start, .Lsyscall_rst_end, syscall_restart)
382#endif
383
384	/*
385	 * If MSR EE/RI was never enabled, IRQs not reconciled, NVGPRs not
386	 * touched, no exit work created, then this can be used.
387	 */
388	.balign IFETCH_ALIGN_BYTES
389	.globl fast_interrupt_return_srr
390fast_interrupt_return_srr:
391_ASM_NOKPROBE_SYMBOL(fast_interrupt_return_srr)
392	kuap_check_amr r3, r4
393	ld	r5,_MSR(r1)
394	andi.	r0,r5,MSR_PR
395#ifdef CONFIG_PPC_BOOK3S
396	beq	1f
397	kuap_user_restore r3, r4
398	b	.Lfast_user_interrupt_return_srr
3991:	kuap_kernel_restore r3, r4
400	andi.	r0,r5,MSR_RI
401	li	r3,0 /* 0 return value, no EMULATE_STACK_STORE */
402	bne+	.Lfast_kernel_interrupt_return_srr
403	addi	r3,r1,STACK_INT_FRAME_REGS
404	bl	CFUNC(unrecoverable_exception)
405	b	. /* should not get here */
406#else
407	bne	.Lfast_user_interrupt_return_srr
408	b	.Lfast_kernel_interrupt_return_srr
409#endif
410
411.macro interrupt_return_macro srr
412	.balign IFETCH_ALIGN_BYTES
413	.globl interrupt_return_\srr
414interrupt_return_\srr\():
415_ASM_NOKPROBE_SYMBOL(interrupt_return_\srr\())
416	ld	r4,_MSR(r1)
417	andi.	r0,r4,MSR_PR
418	beq	interrupt_return_\srr\()_kernel
419interrupt_return_\srr\()_user: /* make backtraces match the _kernel variant */
420_ASM_NOKPROBE_SYMBOL(interrupt_return_\srr\()_user)
421	addi	r3,r1,STACK_INT_FRAME_REGS
422	bl	CFUNC(interrupt_exit_user_prepare)
423#ifndef CONFIG_INTERRUPT_SANITIZE_REGISTERS
424	cmpdi	r3,0
425	bne-	.Lrestore_nvgprs_\srr
426.Lrestore_nvgprs_\srr\()_cont:
427#endif
428	std	r1,PACA_EXIT_SAVE_R1(r13) /* save r1 for restart */
429#ifdef CONFIG_PPC_BOOK3S
430.Linterrupt_return_\srr\()_user_rst_start:
431	lbz	r11,PACAIRQHAPPENED(r13)
432	andi.	r11,r11,(~PACA_IRQ_HARD_DIS)@l
433	bne-	interrupt_return_\srr\()_user_restart
434#endif
435	li	r11,IRQS_ENABLED
436	stb	r11,PACAIRQSOFTMASK(r13)
437	li	r11,0
438	stb	r11,PACAIRQHAPPENED(r13) # clear out possible HARD_DIS
439
440.Lfast_user_interrupt_return_\srr\():
441	SANITIZE_RESTORE_NVGPRS()
442#ifdef CONFIG_PPC_BOOK3S
443	.ifc \srr,srr
444	lbz	r4,PACASRR_VALID(r13)
445	.else
446	lbz	r4,PACAHSRR_VALID(r13)
447	.endif
448	cmpdi	r4,0
449	li	r4,0
450	bne	1f
451#endif
452	ld	r11,_NIP(r1)
453	ld	r12,_MSR(r1)
454	.ifc \srr,srr
455	mtspr	SPRN_SRR0,r11
456	mtspr	SPRN_SRR1,r12
4571:
458#ifdef CONFIG_PPC_BOOK3S
459	stb	r4,PACASRR_VALID(r13)
460#endif
461	.else
462	mtspr	SPRN_HSRR0,r11
463	mtspr	SPRN_HSRR1,r12
4641:
465#ifdef CONFIG_PPC_BOOK3S
466	stb	r4,PACAHSRR_VALID(r13)
467#endif
468	.endif
469	DEBUG_SRR_VALID \srr
470
471#ifdef CONFIG_PPC_IRQ_SOFT_MASK_DEBUG
472	lbz	r4,PACAIRQSOFTMASK(r13)
473	tdnei	r4,IRQS_ENABLED
474#endif
475
476BEGIN_FTR_SECTION
477	ld	r10,_PPR(r1)
478	mtspr	SPRN_PPR,r10
479END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
480
481BEGIN_FTR_SECTION
482	stdcx.	r0,0,r1		/* to clear the reservation */
483FTR_SECTION_ELSE
484	ldarx	r0,0,r1
485ALT_FTR_SECTION_END_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS)
486
487	ld	r3,_CCR(r1)
488	ld	r4,_LINK(r1)
489	ld	r5,_CTR(r1)
490	ld	r6,_XER(r1)
491	li	r0,0
492
493	REST_GPRS(7, 13, r1)
494
495	mtcr	r3
496	mtlr	r4
497	mtctr	r5
498	mtspr	SPRN_XER,r6
499
500	REST_GPRS(2, 6, r1)
501	REST_GPR(0, r1)
502	REST_GPR(1, r1)
503	.ifc \srr,srr
504	RFI_TO_USER
505	.else
506	HRFI_TO_USER
507	.endif
508	b	.	/* prevent speculative execution */
509.Linterrupt_return_\srr\()_user_rst_end:
510
511#ifndef CONFIG_INTERRUPT_SANITIZE_REGISTERS
512.Lrestore_nvgprs_\srr\():
513	REST_NVGPRS(r1)
514	b	.Lrestore_nvgprs_\srr\()_cont
515#endif
516
517#ifdef CONFIG_PPC_BOOK3S
518interrupt_return_\srr\()_user_restart:
519_ASM_NOKPROBE_SYMBOL(interrupt_return_\srr\()_user_restart)
520	GET_PACA(r13)
521	ld	r1,PACA_EXIT_SAVE_R1(r13)
522	LOAD_PACA_TOC()
523	addi	r3,r1,STACK_INT_FRAME_REGS
524	li	r11,IRQS_ALL_DISABLED
525	stb	r11,PACAIRQSOFTMASK(r13)
526	bl	CFUNC(interrupt_exit_user_restart)
527	std	r1,PACA_EXIT_SAVE_R1(r13) /* save r1 for restart */
528	b	.Linterrupt_return_\srr\()_user_rst_start
5291:
530
531SOFT_MASK_TABLE(.Linterrupt_return_\srr\()_user_rst_start, 1b)
532RESTART_TABLE(.Linterrupt_return_\srr\()_user_rst_start, .Linterrupt_return_\srr\()_user_rst_end, interrupt_return_\srr\()_user_restart)
533#endif
534
535	.balign IFETCH_ALIGN_BYTES
536interrupt_return_\srr\()_kernel:
537_ASM_NOKPROBE_SYMBOL(interrupt_return_\srr\()_kernel)
538	addi	r3,r1,STACK_INT_FRAME_REGS
539	bl	CFUNC(interrupt_exit_kernel_prepare)
540
541	std	r1,PACA_EXIT_SAVE_R1(r13) /* save r1 for restart */
542.Linterrupt_return_\srr\()_kernel_rst_start:
543	ld	r11,SOFTE(r1)
544	cmpwi	r11,IRQS_ENABLED
545	stb	r11,PACAIRQSOFTMASK(r13)
546	beq	.Linterrupt_return_\srr\()_soft_enabled
547
548	/*
549	 * Returning to soft-disabled context.
550	 * Check if a MUST_HARD_MASK interrupt has become pending, in which
551	 * case we need to disable MSR[EE] in the return context.
552	 *
553	 * The MSR[EE] check catches among other things the short incoherency
554	 * in hard_irq_disable() between clearing MSR[EE] and setting
555	 * PACA_IRQ_HARD_DIS.
556	 */
557	ld	r12,_MSR(r1)
558	andi.	r10,r12,MSR_EE
559	beq	.Lfast_kernel_interrupt_return_\srr\() // EE already disabled
560	lbz	r11,PACAIRQHAPPENED(r13)
561	andi.	r10,r11,PACA_IRQ_MUST_HARD_MASK
562	bne	1f // HARD_MASK is pending
563	// No HARD_MASK pending, clear possible HARD_DIS set by interrupt
564	andi.	r11,r11,(~PACA_IRQ_HARD_DIS)@l
565	stb	r11,PACAIRQHAPPENED(r13)
566	b	.Lfast_kernel_interrupt_return_\srr\()
567
568
5691:	/* Must clear MSR_EE from _MSR */
570#ifdef CONFIG_PPC_BOOK3S
571	li	r10,0
572	/* Clear valid before changing _MSR */
573	.ifc \srr,srr
574	stb	r10,PACASRR_VALID(r13)
575	.else
576	stb	r10,PACAHSRR_VALID(r13)
577	.endif
578#endif
579	xori	r12,r12,MSR_EE
580	std	r12,_MSR(r1)
581	b	.Lfast_kernel_interrupt_return_\srr\()
582
583.Linterrupt_return_\srr\()_soft_enabled:
584	/*
585	 * In the soft-enabled case, need to double-check that we have no
586	 * pending interrupts that might have come in before we reached the
587	 * restart section of code, and restart the exit so those can be
588	 * handled.
589	 *
590	 * If there are none, it is be possible that the interrupt still
591	 * has PACA_IRQ_HARD_DIS set, which needs to be cleared for the
592	 * interrupted context. This clear will not clobber a new pending
593	 * interrupt coming in, because we're in the restart section, so
594	 * such would return to the restart location.
595	 */
596#ifdef CONFIG_PPC_BOOK3S
597	lbz	r11,PACAIRQHAPPENED(r13)
598	andi.	r11,r11,(~PACA_IRQ_HARD_DIS)@l
599	bne-	interrupt_return_\srr\()_kernel_restart
600#endif
601	li	r11,0
602	stb	r11,PACAIRQHAPPENED(r13) // clear the possible HARD_DIS
603
604.Lfast_kernel_interrupt_return_\srr\():
605	SANITIZE_RESTORE_NVGPRS()
606	cmpdi	cr1,r3,0
607#ifdef CONFIG_PPC_BOOK3S
608	.ifc \srr,srr
609	lbz	r4,PACASRR_VALID(r13)
610	.else
611	lbz	r4,PACAHSRR_VALID(r13)
612	.endif
613	cmpdi	r4,0
614	li	r4,0
615	bne	1f
616#endif
617	ld	r11,_NIP(r1)
618	ld	r12,_MSR(r1)
619	.ifc \srr,srr
620	mtspr	SPRN_SRR0,r11
621	mtspr	SPRN_SRR1,r12
6221:
623#ifdef CONFIG_PPC_BOOK3S
624	stb	r4,PACASRR_VALID(r13)
625#endif
626	.else
627	mtspr	SPRN_HSRR0,r11
628	mtspr	SPRN_HSRR1,r12
6291:
630#ifdef CONFIG_PPC_BOOK3S
631	stb	r4,PACAHSRR_VALID(r13)
632#endif
633	.endif
634	DEBUG_SRR_VALID \srr
635
636BEGIN_FTR_SECTION
637	stdcx.	r0,0,r1		/* to clear the reservation */
638FTR_SECTION_ELSE
639	ldarx	r0,0,r1
640ALT_FTR_SECTION_END_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS)
641
642	ld	r3,_LINK(r1)
643	ld	r4,_CTR(r1)
644	ld	r5,_XER(r1)
645	ld	r6,_CCR(r1)
646	li	r0,0
647
648	REST_GPRS(7, 12, r1)
649
650	mtlr	r3
651	mtctr	r4
652	mtspr	SPRN_XER,r5
653
654	/*
655	 * Leaving a stale STACK_FRAME_REGS_MARKER on the stack can confuse
656	 * the reliable stack unwinder later on. Clear it.
657	 */
658	std	r0,STACK_INT_FRAME_MARKER(r1)
659
660	REST_GPRS(2, 5, r1)
661
662	bne-	cr1,1f /* emulate stack store */
663	mtcr	r6
664	REST_GPR(6, r1)
665	REST_GPR(0, r1)
666	REST_GPR(1, r1)
667	.ifc \srr,srr
668	RFI_TO_KERNEL
669	.else
670	HRFI_TO_KERNEL
671	.endif
672	b	.	/* prevent speculative execution */
673
6741:	/*
675	 * Emulate stack store with update. New r1 value was already calculated
676	 * and updated in our interrupt regs by emulate_loadstore, but we can't
677	 * store the previous value of r1 to the stack before re-loading our
678	 * registers from it, otherwise they could be clobbered.  Use
679	 * PACA_EXGEN as temporary storage to hold the store data, as
680	 * interrupts are disabled here so it won't be clobbered.
681	 */
682	mtcr	r6
683	std	r9,PACA_EXGEN+0(r13)
684	addi	r9,r1,INT_FRAME_SIZE /* get original r1 */
685	REST_GPR(6, r1)
686	REST_GPR(0, r1)
687	REST_GPR(1, r1)
688	std	r9,0(r1) /* perform store component of stdu */
689	ld	r9,PACA_EXGEN+0(r13)
690
691	.ifc \srr,srr
692	RFI_TO_KERNEL
693	.else
694	HRFI_TO_KERNEL
695	.endif
696	b	.	/* prevent speculative execution */
697.Linterrupt_return_\srr\()_kernel_rst_end:
698
699#ifdef CONFIG_PPC_BOOK3S
700interrupt_return_\srr\()_kernel_restart:
701_ASM_NOKPROBE_SYMBOL(interrupt_return_\srr\()_kernel_restart)
702	GET_PACA(r13)
703	ld	r1,PACA_EXIT_SAVE_R1(r13)
704	LOAD_PACA_TOC()
705	addi	r3,r1,STACK_INT_FRAME_REGS
706	li	r11,IRQS_ALL_DISABLED
707	stb	r11,PACAIRQSOFTMASK(r13)
708	bl	CFUNC(interrupt_exit_kernel_restart)
709	std	r1,PACA_EXIT_SAVE_R1(r13) /* save r1 for restart */
710	b	.Linterrupt_return_\srr\()_kernel_rst_start
7111:
712
713SOFT_MASK_TABLE(.Linterrupt_return_\srr\()_kernel_rst_start, 1b)
714RESTART_TABLE(.Linterrupt_return_\srr\()_kernel_rst_start, .Linterrupt_return_\srr\()_kernel_rst_end, interrupt_return_\srr\()_kernel_restart)
715#endif
716
717.endm
718
719interrupt_return_macro srr
720#ifdef CONFIG_PPC_BOOK3S
721interrupt_return_macro hsrr
722
723	.globl __end_soft_masked
724__end_soft_masked:
725DEFINE_FIXED_SYMBOL(__end_soft_masked, text)
726#endif /* CONFIG_PPC_BOOK3S */
727
728#ifdef CONFIG_PPC_BOOK3S
729_GLOBAL(ret_from_fork_scv)
730	bl	CFUNC(schedule_tail)
731	HANDLER_RESTORE_NVGPRS()
732	li	r3,0	/* fork() return value */
733	b	.Lsyscall_vectored_common_exit
734#endif
735
736_GLOBAL(ret_from_fork)
737	bl	CFUNC(schedule_tail)
738	HANDLER_RESTORE_NVGPRS()
739	li	r3,0	/* fork() return value */
740	b	.Lsyscall_exit
741
742_GLOBAL(ret_from_kernel_user_thread)
743	bl	CFUNC(schedule_tail)
744	mtctr	r14
745	mr	r3,r15
746#ifdef CONFIG_PPC64_ELF_ABI_V2
747	mr	r12,r14
748#endif
749	bctrl
750	li	r3,0
751	/*
752	 * It does not matter whether this returns via the scv or sc path
753	 * because it returns as execve() and therefore has no calling ABI
754	 * (i.e., it sets registers according to the exec()ed entry point).
755	 */
756	b	.Lsyscall_exit
757
758_GLOBAL(start_kernel_thread)
759	bl	CFUNC(schedule_tail)
760	mtctr	r14
761	mr	r3,r15
762#ifdef CONFIG_PPC64_ELF_ABI_V2
763	mr	r12,r14
764#endif
765	bctrl
766	/*
767	 * This must not return. We actually want to BUG here, not WARN,
768	 * because BUG will exit the process which is what the kernel thread
769	 * should have done, which may give some hope of continuing.
770	 */
771100:	trap
772	EMIT_BUG_ENTRY 100b,__FILE__,__LINE__,0
773