1#include <asm/asm-offsets.h>
2#include <asm/bug.h>
3#ifdef CONFIG_PPC_BOOK3S
4#include <asm/exception-64s.h>
5#else
6#include <asm/exception-64e.h>
7#endif
8#include <asm/feature-fixups.h>
9#include <asm/head-64.h>
10#include <asm/hw_irq.h>
11#include <asm/kup.h>
12#include <asm/mmu.h>
13#include <asm/ppc_asm.h>
14#include <asm/ptrace.h>
15
16	.align 7
17
18.macro DEBUG_SRR_VALID srr
19#ifdef CONFIG_PPC_RFI_SRR_DEBUG
20	.ifc \srr,srr
21	mfspr	r11,SPRN_SRR0
22	ld	r12,_NIP(r1)
23	clrrdi  r11,r11,2
24	clrrdi  r12,r12,2
25100:	tdne	r11,r12
26	EMIT_WARN_ENTRY 100b,__FILE__,__LINE__,(BUGFLAG_WARNING | BUGFLAG_ONCE)
27	mfspr	r11,SPRN_SRR1
28	ld	r12,_MSR(r1)
29100:	tdne	r11,r12
30	EMIT_WARN_ENTRY 100b,__FILE__,__LINE__,(BUGFLAG_WARNING | BUGFLAG_ONCE)
31	.else
32	mfspr	r11,SPRN_HSRR0
33	ld	r12,_NIP(r1)
34	clrrdi  r11,r11,2
35	clrrdi  r12,r12,2
36100:	tdne	r11,r12
37	EMIT_WARN_ENTRY 100b,__FILE__,__LINE__,(BUGFLAG_WARNING | BUGFLAG_ONCE)
38	mfspr	r11,SPRN_HSRR1
39	ld	r12,_MSR(r1)
40100:	tdne	r11,r12
41	EMIT_WARN_ENTRY 100b,__FILE__,__LINE__,(BUGFLAG_WARNING | BUGFLAG_ONCE)
42	.endif
43#endif
44.endm
45
46#ifdef CONFIG_PPC_BOOK3S
47.macro system_call_vectored name trapnr
48	.globl system_call_vectored_\name
49system_call_vectored_\name:
50_ASM_NOKPROBE_SYMBOL(system_call_vectored_\name)
51	SCV_INTERRUPT_TO_KERNEL
52	mr	r10,r1
53	ld	r1,PACAKSAVE(r13)
54	std	r10,0(r1)
55	std	r11,_NIP(r1)
56	std	r12,_MSR(r1)
57	std	r0,GPR0(r1)
58	std	r10,GPR1(r1)
59	std	r2,GPR2(r1)
60	LOAD_PACA_TOC()
61	mfcr	r12
62	li	r11,0
63	/* Save syscall parameters in r3-r8 */
64	SAVE_GPRS(3, 8, r1)
65	/* Zero r9-r12, this should only be required when restoring all GPRs */
66	std	r11,GPR9(r1)
67	std	r11,GPR10(r1)
68	std	r11,GPR11(r1)
69	std	r11,GPR12(r1)
70	std	r9,GPR13(r1)
71	SAVE_NVGPRS(r1)
72	std	r11,_XER(r1)
73	std	r11,_LINK(r1)
74	std	r11,_CTR(r1)
75
76	li	r11,\trapnr
77	std	r11,_TRAP(r1)
78	std	r12,_CCR(r1)
79	std	r3,ORIG_GPR3(r1)
80	/* Calling convention has r3 = regs, r4 = orig r0 */
81	addi	r3,r1,STACK_FRAME_OVERHEAD
82	mr	r4,r0
83	LOAD_REG_IMMEDIATE(r11, STACK_FRAME_REGS_MARKER)
84	std	r11,-16(r3)		/* "regshere" marker */
85
86BEGIN_FTR_SECTION
87	HMT_MEDIUM
88END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
89
90	/*
91	 * scv enters with MSR[EE]=1 and is immediately considered soft-masked.
92	 * The entry vector already sets PACAIRQSOFTMASK to IRQS_ALL_DISABLED,
93	 * and interrupts may be masked and pending already.
94	 * system_call_exception() will call trace_hardirqs_off() which means
95	 * interrupts could already have been blocked before trace_hardirqs_off,
96	 * but this is the best we can do.
97	 */
98
99	bl	system_call_exception
100
101.Lsyscall_vectored_\name\()_exit:
102	addi	r4,r1,STACK_FRAME_OVERHEAD
103	li	r5,1 /* scv */
104	bl	syscall_exit_prepare
105	std	r1,PACA_EXIT_SAVE_R1(r13) /* save r1 for restart */
106.Lsyscall_vectored_\name\()_rst_start:
107	lbz	r11,PACAIRQHAPPENED(r13)
108	andi.	r11,r11,(~PACA_IRQ_HARD_DIS)@l
109	bne-	syscall_vectored_\name\()_restart
110	li	r11,IRQS_ENABLED
111	stb	r11,PACAIRQSOFTMASK(r13)
112	li	r11,0
113	stb	r11,PACAIRQHAPPENED(r13) # clear out possible HARD_DIS
114
115	ld	r2,_CCR(r1)
116	ld	r4,_NIP(r1)
117	ld	r5,_MSR(r1)
118
119BEGIN_FTR_SECTION
120	stdcx.	r0,0,r1			/* to clear the reservation */
121END_FTR_SECTION_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS)
122
123BEGIN_FTR_SECTION
124	HMT_MEDIUM_LOW
125END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
126
127	cmpdi	r3,0
128	bne	.Lsyscall_vectored_\name\()_restore_regs
129
130	/* rfscv returns with LR->NIA and CTR->MSR */
131	mtlr	r4
132	mtctr	r5
133
134	/* Could zero these as per ABI, but we may consider a stricter ABI
135	 * which preserves these if libc implementations can benefit, so
136	 * restore them for now until further measurement is done. */
137	REST_GPR(0, r1)
138	REST_GPRS(4, 8, r1)
139	/* Zero volatile regs that may contain sensitive kernel data */
140	ZEROIZE_GPRS(9, 12)
141	mtspr	SPRN_XER,r0
142
143	/*
144	 * We don't need to restore AMR on the way back to userspace for KUAP.
145	 * The value of AMR only matters while we're in the kernel.
146	 */
147	mtcr	r2
148	REST_GPRS(2, 3, r1)
149	REST_GPR(13, r1)
150	REST_GPR(1, r1)
151	RFSCV_TO_USER
152	b	.	/* prevent speculative execution */
153
154.Lsyscall_vectored_\name\()_restore_regs:
155	mtspr	SPRN_SRR0,r4
156	mtspr	SPRN_SRR1,r5
157
158	ld	r3,_CTR(r1)
159	ld	r4,_LINK(r1)
160	ld	r5,_XER(r1)
161
162	REST_NVGPRS(r1)
163	REST_GPR(0, r1)
164	mtcr	r2
165	mtctr	r3
166	mtlr	r4
167	mtspr	SPRN_XER,r5
168	REST_GPRS(2, 13, r1)
169	REST_GPR(1, r1)
170	RFI_TO_USER
171.Lsyscall_vectored_\name\()_rst_end:
172
173syscall_vectored_\name\()_restart:
174_ASM_NOKPROBE_SYMBOL(syscall_vectored_\name\()_restart)
175	GET_PACA(r13)
176	ld	r1,PACA_EXIT_SAVE_R1(r13)
177	LOAD_PACA_TOC()
178	ld	r3,RESULT(r1)
179	addi	r4,r1,STACK_FRAME_OVERHEAD
180	li	r11,IRQS_ALL_DISABLED
181	stb	r11,PACAIRQSOFTMASK(r13)
182	bl	syscall_exit_restart
183	std	r1,PACA_EXIT_SAVE_R1(r13) /* save r1 for restart */
184	b	.Lsyscall_vectored_\name\()_rst_start
1851:
186
187SOFT_MASK_TABLE(.Lsyscall_vectored_\name\()_rst_start, 1b)
188RESTART_TABLE(.Lsyscall_vectored_\name\()_rst_start, .Lsyscall_vectored_\name\()_rst_end, syscall_vectored_\name\()_restart)
189
190.endm
191
192system_call_vectored common 0x3000
193
194/*
195 * We instantiate another entry copy for the SIGILL variant, with TRAP=0x7ff0
196 * which is tested by system_call_exception when r0 is -1 (as set by vector
197 * entry code).
198 */
199system_call_vectored sigill 0x7ff0
200
201#endif /* CONFIG_PPC_BOOK3S */
202
203	.balign IFETCH_ALIGN_BYTES
204	.globl system_call_common_real
205system_call_common_real:
206_ASM_NOKPROBE_SYMBOL(system_call_common_real)
207	ld	r10,PACAKMSR(r13)	/* get MSR value for kernel */
208	mtmsrd	r10
209
210	.balign IFETCH_ALIGN_BYTES
211	.globl system_call_common
212system_call_common:
213_ASM_NOKPROBE_SYMBOL(system_call_common)
214	mr	r10,r1
215	ld	r1,PACAKSAVE(r13)
216	std	r10,0(r1)
217	std	r11,_NIP(r1)
218	std	r12,_MSR(r1)
219	std	r0,GPR0(r1)
220	std	r10,GPR1(r1)
221	std	r2,GPR2(r1)
222#ifdef CONFIG_PPC_E500
223START_BTB_FLUSH_SECTION
224	BTB_FLUSH(r10)
225END_BTB_FLUSH_SECTION
226#endif
227	LOAD_PACA_TOC()
228	mfcr	r12
229	li	r11,0
230	/* Save syscall parameters in r3-r8 */
231	SAVE_GPRS(3, 8, r1)
232	/* Zero r9-r12, this should only be required when restoring all GPRs */
233	std	r11,GPR9(r1)
234	std	r11,GPR10(r1)
235	std	r11,GPR11(r1)
236	std	r11,GPR12(r1)
237	std	r9,GPR13(r1)
238	SAVE_NVGPRS(r1)
239	std	r11,_XER(r1)
240	std	r11,_CTR(r1)
241	mflr	r10
242
243	/*
244	 * This clears CR0.SO (bit 28), which is the error indication on
245	 * return from this system call.
246	 */
247	rldimi	r12,r11,28,(63-28)
248	li	r11,0xc00
249	std	r10,_LINK(r1)
250	std	r11,_TRAP(r1)
251	std	r12,_CCR(r1)
252	std	r3,ORIG_GPR3(r1)
253	/* Calling convention has r3 = regs, r4 = orig r0 */
254	addi	r3,r1,STACK_FRAME_OVERHEAD
255	mr	r4,r0
256	LOAD_REG_IMMEDIATE(r11, STACK_FRAME_REGS_MARKER)
257	std	r11,-16(r3)		/* "regshere" marker */
258
259#ifdef CONFIG_PPC_BOOK3S
260	li	r11,1
261	stb	r11,PACASRR_VALID(r13)
262#endif
263
264	/*
265	 * We always enter kernel from userspace with irq soft-mask enabled and
266	 * nothing pending. system_call_exception() will call
267	 * trace_hardirqs_off().
268	 */
269	li	r11,IRQS_ALL_DISABLED
270	stb	r11,PACAIRQSOFTMASK(r13)
271#ifdef CONFIG_PPC_BOOK3S
272	li	r12,-1 /* Set MSR_EE and MSR_RI */
273	mtmsrd	r12,1
274#else
275	wrteei	1
276#endif
277
278	bl	system_call_exception
279
280.Lsyscall_exit:
281	addi	r4,r1,STACK_FRAME_OVERHEAD
282	li	r5,0 /* !scv */
283	bl	syscall_exit_prepare
284	std	r1,PACA_EXIT_SAVE_R1(r13) /* save r1 for restart */
285#ifdef CONFIG_PPC_BOOK3S
286.Lsyscall_rst_start:
287	lbz	r11,PACAIRQHAPPENED(r13)
288	andi.	r11,r11,(~PACA_IRQ_HARD_DIS)@l
289	bne-	syscall_restart
290#endif
291	li	r11,IRQS_ENABLED
292	stb	r11,PACAIRQSOFTMASK(r13)
293	li	r11,0
294	stb	r11,PACAIRQHAPPENED(r13) # clear out possible HARD_DIS
295
296	ld	r2,_CCR(r1)
297	ld	r6,_LINK(r1)
298	mtlr	r6
299
300#ifdef CONFIG_PPC_BOOK3S
301	lbz	r4,PACASRR_VALID(r13)
302	cmpdi	r4,0
303	bne	1f
304	li	r4,0
305	stb	r4,PACASRR_VALID(r13)
306#endif
307	ld	r4,_NIP(r1)
308	ld	r5,_MSR(r1)
309	mtspr	SPRN_SRR0,r4
310	mtspr	SPRN_SRR1,r5
3111:
312	DEBUG_SRR_VALID srr
313
314BEGIN_FTR_SECTION
315	stdcx.	r0,0,r1			/* to clear the reservation */
316END_FTR_SECTION_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS)
317
318	cmpdi	r3,0
319	bne	.Lsyscall_restore_regs
320	/* Zero volatile regs that may contain sensitive kernel data */
321	ZEROIZE_GPR(0)
322	ZEROIZE_GPRS(4, 12)
323	mtctr	r0
324	mtspr	SPRN_XER,r0
325.Lsyscall_restore_regs_cont:
326
327BEGIN_FTR_SECTION
328	HMT_MEDIUM_LOW
329END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
330
331	/*
332	 * We don't need to restore AMR on the way back to userspace for KUAP.
333	 * The value of AMR only matters while we're in the kernel.
334	 */
335	mtcr	r2
336	REST_GPRS(2, 3, r1)
337	REST_GPR(13, r1)
338	REST_GPR(1, r1)
339	RFI_TO_USER
340	b	.	/* prevent speculative execution */
341
342.Lsyscall_restore_regs:
343	ld	r3,_CTR(r1)
344	ld	r4,_XER(r1)
345	REST_NVGPRS(r1)
346	mtctr	r3
347	mtspr	SPRN_XER,r4
348	REST_GPR(0, r1)
349	REST_GPRS(4, 12, r1)
350	b	.Lsyscall_restore_regs_cont
351.Lsyscall_rst_end:
352
353#ifdef CONFIG_PPC_BOOK3S
354syscall_restart:
355_ASM_NOKPROBE_SYMBOL(syscall_restart)
356	GET_PACA(r13)
357	ld	r1,PACA_EXIT_SAVE_R1(r13)
358	LOAD_PACA_TOC()
359	ld	r3,RESULT(r1)
360	addi	r4,r1,STACK_FRAME_OVERHEAD
361	li	r11,IRQS_ALL_DISABLED
362	stb	r11,PACAIRQSOFTMASK(r13)
363	bl	syscall_exit_restart
364	std	r1,PACA_EXIT_SAVE_R1(r13) /* save r1 for restart */
365	b	.Lsyscall_rst_start
3661:
367
368SOFT_MASK_TABLE(.Lsyscall_rst_start, 1b)
369RESTART_TABLE(.Lsyscall_rst_start, .Lsyscall_rst_end, syscall_restart)
370#endif
371
372	/*
373	 * If MSR EE/RI was never enabled, IRQs not reconciled, NVGPRs not
374	 * touched, no exit work created, then this can be used.
375	 */
376	.balign IFETCH_ALIGN_BYTES
377	.globl fast_interrupt_return_srr
378fast_interrupt_return_srr:
379_ASM_NOKPROBE_SYMBOL(fast_interrupt_return_srr)
380	kuap_check_amr r3, r4
381	ld	r5,_MSR(r1)
382	andi.	r0,r5,MSR_PR
383#ifdef CONFIG_PPC_BOOK3S
384	beq	1f
385	kuap_user_restore r3, r4
386	b	.Lfast_user_interrupt_return_srr
3871:	kuap_kernel_restore r3, r4
388	andi.	r0,r5,MSR_RI
389	li	r3,0 /* 0 return value, no EMULATE_STACK_STORE */
390	bne+	.Lfast_kernel_interrupt_return_srr
391	addi	r3,r1,STACK_FRAME_OVERHEAD
392	bl	unrecoverable_exception
393	b	. /* should not get here */
394#else
395	bne	.Lfast_user_interrupt_return_srr
396	b	.Lfast_kernel_interrupt_return_srr
397#endif
398
399.macro interrupt_return_macro srr
400	.balign IFETCH_ALIGN_BYTES
401	.globl interrupt_return_\srr
402interrupt_return_\srr\():
403_ASM_NOKPROBE_SYMBOL(interrupt_return_\srr\())
404	ld	r4,_MSR(r1)
405	andi.	r0,r4,MSR_PR
406	beq	interrupt_return_\srr\()_kernel
407interrupt_return_\srr\()_user: /* make backtraces match the _kernel variant */
408_ASM_NOKPROBE_SYMBOL(interrupt_return_\srr\()_user)
409	addi	r3,r1,STACK_FRAME_OVERHEAD
410	bl	interrupt_exit_user_prepare
411	cmpdi	r3,0
412	bne-	.Lrestore_nvgprs_\srr
413.Lrestore_nvgprs_\srr\()_cont:
414	std	r1,PACA_EXIT_SAVE_R1(r13) /* save r1 for restart */
415#ifdef CONFIG_PPC_BOOK3S
416.Linterrupt_return_\srr\()_user_rst_start:
417	lbz	r11,PACAIRQHAPPENED(r13)
418	andi.	r11,r11,(~PACA_IRQ_HARD_DIS)@l
419	bne-	interrupt_return_\srr\()_user_restart
420#endif
421	li	r11,IRQS_ENABLED
422	stb	r11,PACAIRQSOFTMASK(r13)
423	li	r11,0
424	stb	r11,PACAIRQHAPPENED(r13) # clear out possible HARD_DIS
425
426.Lfast_user_interrupt_return_\srr\():
427#ifdef CONFIG_PPC_BOOK3S
428	.ifc \srr,srr
429	lbz	r4,PACASRR_VALID(r13)
430	.else
431	lbz	r4,PACAHSRR_VALID(r13)
432	.endif
433	cmpdi	r4,0
434	li	r4,0
435	bne	1f
436#endif
437	ld	r11,_NIP(r1)
438	ld	r12,_MSR(r1)
439	.ifc \srr,srr
440	mtspr	SPRN_SRR0,r11
441	mtspr	SPRN_SRR1,r12
4421:
443#ifdef CONFIG_PPC_BOOK3S
444	stb	r4,PACASRR_VALID(r13)
445#endif
446	.else
447	mtspr	SPRN_HSRR0,r11
448	mtspr	SPRN_HSRR1,r12
4491:
450#ifdef CONFIG_PPC_BOOK3S
451	stb	r4,PACAHSRR_VALID(r13)
452#endif
453	.endif
454	DEBUG_SRR_VALID \srr
455
456#ifdef CONFIG_PPC_IRQ_SOFT_MASK_DEBUG
457	lbz	r4,PACAIRQSOFTMASK(r13)
458	tdnei	r4,IRQS_ENABLED
459#endif
460
461BEGIN_FTR_SECTION
462	ld	r10,_PPR(r1)
463	mtspr	SPRN_PPR,r10
464END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
465
466BEGIN_FTR_SECTION
467	stdcx.	r0,0,r1		/* to clear the reservation */
468FTR_SECTION_ELSE
469	ldarx	r0,0,r1
470ALT_FTR_SECTION_END_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS)
471
472	ld	r3,_CCR(r1)
473	ld	r4,_LINK(r1)
474	ld	r5,_CTR(r1)
475	ld	r6,_XER(r1)
476	li	r0,0
477
478	REST_GPRS(7, 13, r1)
479
480	mtcr	r3
481	mtlr	r4
482	mtctr	r5
483	mtspr	SPRN_XER,r6
484
485	REST_GPRS(2, 6, r1)
486	REST_GPR(0, r1)
487	REST_GPR(1, r1)
488	.ifc \srr,srr
489	RFI_TO_USER
490	.else
491	HRFI_TO_USER
492	.endif
493	b	.	/* prevent speculative execution */
494.Linterrupt_return_\srr\()_user_rst_end:
495
496.Lrestore_nvgprs_\srr\():
497	REST_NVGPRS(r1)
498	b	.Lrestore_nvgprs_\srr\()_cont
499
500#ifdef CONFIG_PPC_BOOK3S
501interrupt_return_\srr\()_user_restart:
502_ASM_NOKPROBE_SYMBOL(interrupt_return_\srr\()_user_restart)
503	GET_PACA(r13)
504	ld	r1,PACA_EXIT_SAVE_R1(r13)
505	LOAD_PACA_TOC()
506	addi	r3,r1,STACK_FRAME_OVERHEAD
507	li	r11,IRQS_ALL_DISABLED
508	stb	r11,PACAIRQSOFTMASK(r13)
509	bl	interrupt_exit_user_restart
510	std	r1,PACA_EXIT_SAVE_R1(r13) /* save r1 for restart */
511	b	.Linterrupt_return_\srr\()_user_rst_start
5121:
513
514SOFT_MASK_TABLE(.Linterrupt_return_\srr\()_user_rst_start, 1b)
515RESTART_TABLE(.Linterrupt_return_\srr\()_user_rst_start, .Linterrupt_return_\srr\()_user_rst_end, interrupt_return_\srr\()_user_restart)
516#endif
517
518	.balign IFETCH_ALIGN_BYTES
519interrupt_return_\srr\()_kernel:
520_ASM_NOKPROBE_SYMBOL(interrupt_return_\srr\()_kernel)
521	addi	r3,r1,STACK_FRAME_OVERHEAD
522	bl	interrupt_exit_kernel_prepare
523
524	std	r1,PACA_EXIT_SAVE_R1(r13) /* save r1 for restart */
525.Linterrupt_return_\srr\()_kernel_rst_start:
526	ld	r11,SOFTE(r1)
527	cmpwi	r11,IRQS_ENABLED
528	stb	r11,PACAIRQSOFTMASK(r13)
529	beq	.Linterrupt_return_\srr\()_soft_enabled
530
531	/*
532	 * Returning to soft-disabled context.
533	 * Check if a MUST_HARD_MASK interrupt has become pending, in which
534	 * case we need to disable MSR[EE] in the return context.
535	 */
536	ld	r12,_MSR(r1)
537	andi.	r10,r12,MSR_EE
538	beq	.Lfast_kernel_interrupt_return_\srr\() // EE already disabled
539	lbz	r11,PACAIRQHAPPENED(r13)
540	andi.	r10,r11,PACA_IRQ_MUST_HARD_MASK
541	beq	.Lfast_kernel_interrupt_return_\srr\() // No HARD_MASK pending
542
543	/* Must clear MSR_EE from _MSR */
544#ifdef CONFIG_PPC_BOOK3S
545	li	r10,0
546	/* Clear valid before changing _MSR */
547	.ifc \srr,srr
548	stb	r10,PACASRR_VALID(r13)
549	.else
550	stb	r10,PACAHSRR_VALID(r13)
551	.endif
552#endif
553	xori	r12,r12,MSR_EE
554	std	r12,_MSR(r1)
555	b	.Lfast_kernel_interrupt_return_\srr\()
556
557.Linterrupt_return_\srr\()_soft_enabled:
558	/*
559	 * In the soft-enabled case, need to double-check that we have no
560	 * pending interrupts that might have come in before we reached the
561	 * restart section of code, and restart the exit so those can be
562	 * handled.
563	 *
564	 * If there are none, it is be possible that the interrupt still
565	 * has PACA_IRQ_HARD_DIS set, which needs to be cleared for the
566	 * interrupted context. This clear will not clobber a new pending
567	 * interrupt coming in, because we're in the restart section, so
568	 * such would return to the restart location.
569	 */
570#ifdef CONFIG_PPC_BOOK3S
571	lbz	r11,PACAIRQHAPPENED(r13)
572	andi.	r11,r11,(~PACA_IRQ_HARD_DIS)@l
573	bne-	interrupt_return_\srr\()_kernel_restart
574#endif
575	li	r11,0
576	stb	r11,PACAIRQHAPPENED(r13) // clear the possible HARD_DIS
577
578.Lfast_kernel_interrupt_return_\srr\():
579	cmpdi	cr1,r3,0
580#ifdef CONFIG_PPC_BOOK3S
581	.ifc \srr,srr
582	lbz	r4,PACASRR_VALID(r13)
583	.else
584	lbz	r4,PACAHSRR_VALID(r13)
585	.endif
586	cmpdi	r4,0
587	li	r4,0
588	bne	1f
589#endif
590	ld	r11,_NIP(r1)
591	ld	r12,_MSR(r1)
592	.ifc \srr,srr
593	mtspr	SPRN_SRR0,r11
594	mtspr	SPRN_SRR1,r12
5951:
596#ifdef CONFIG_PPC_BOOK3S
597	stb	r4,PACASRR_VALID(r13)
598#endif
599	.else
600	mtspr	SPRN_HSRR0,r11
601	mtspr	SPRN_HSRR1,r12
6021:
603#ifdef CONFIG_PPC_BOOK3S
604	stb	r4,PACAHSRR_VALID(r13)
605#endif
606	.endif
607	DEBUG_SRR_VALID \srr
608
609BEGIN_FTR_SECTION
610	stdcx.	r0,0,r1		/* to clear the reservation */
611FTR_SECTION_ELSE
612	ldarx	r0,0,r1
613ALT_FTR_SECTION_END_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS)
614
615	ld	r3,_LINK(r1)
616	ld	r4,_CTR(r1)
617	ld	r5,_XER(r1)
618	ld	r6,_CCR(r1)
619	li	r0,0
620
621	REST_GPRS(7, 12, r1)
622
623	mtlr	r3
624	mtctr	r4
625	mtspr	SPRN_XER,r5
626
627	/*
628	 * Leaving a stale STACK_FRAME_REGS_MARKER on the stack can confuse
629	 * the reliable stack unwinder later on. Clear it.
630	 */
631	std	r0,STACK_FRAME_OVERHEAD-16(r1)
632
633	REST_GPRS(2, 5, r1)
634
635	bne-	cr1,1f /* emulate stack store */
636	mtcr	r6
637	REST_GPR(6, r1)
638	REST_GPR(0, r1)
639	REST_GPR(1, r1)
640	.ifc \srr,srr
641	RFI_TO_KERNEL
642	.else
643	HRFI_TO_KERNEL
644	.endif
645	b	.	/* prevent speculative execution */
646
6471:	/*
648	 * Emulate stack store with update. New r1 value was already calculated
649	 * and updated in our interrupt regs by emulate_loadstore, but we can't
650	 * store the previous value of r1 to the stack before re-loading our
651	 * registers from it, otherwise they could be clobbered.  Use
652	 * PACA_EXGEN as temporary storage to hold the store data, as
653	 * interrupts are disabled here so it won't be clobbered.
654	 */
655	mtcr	r6
656	std	r9,PACA_EXGEN+0(r13)
657	addi	r9,r1,INT_FRAME_SIZE /* get original r1 */
658	REST_GPR(6, r1)
659	REST_GPR(0, r1)
660	REST_GPR(1, r1)
661	std	r9,0(r1) /* perform store component of stdu */
662	ld	r9,PACA_EXGEN+0(r13)
663
664	.ifc \srr,srr
665	RFI_TO_KERNEL
666	.else
667	HRFI_TO_KERNEL
668	.endif
669	b	.	/* prevent speculative execution */
670.Linterrupt_return_\srr\()_kernel_rst_end:
671
672#ifdef CONFIG_PPC_BOOK3S
673interrupt_return_\srr\()_kernel_restart:
674_ASM_NOKPROBE_SYMBOL(interrupt_return_\srr\()_kernel_restart)
675	GET_PACA(r13)
676	ld	r1,PACA_EXIT_SAVE_R1(r13)
677	LOAD_PACA_TOC()
678	addi	r3,r1,STACK_FRAME_OVERHEAD
679	li	r11,IRQS_ALL_DISABLED
680	stb	r11,PACAIRQSOFTMASK(r13)
681	bl	interrupt_exit_kernel_restart
682	std	r1,PACA_EXIT_SAVE_R1(r13) /* save r1 for restart */
683	b	.Linterrupt_return_\srr\()_kernel_rst_start
6841:
685
686SOFT_MASK_TABLE(.Linterrupt_return_\srr\()_kernel_rst_start, 1b)
687RESTART_TABLE(.Linterrupt_return_\srr\()_kernel_rst_start, .Linterrupt_return_\srr\()_kernel_rst_end, interrupt_return_\srr\()_kernel_restart)
688#endif
689
690.endm
691
692interrupt_return_macro srr
693#ifdef CONFIG_PPC_BOOK3S
694interrupt_return_macro hsrr
695
696	.globl __end_soft_masked
697__end_soft_masked:
698DEFINE_FIXED_SYMBOL(__end_soft_masked, text)
699#endif /* CONFIG_PPC_BOOK3S */
700
701#ifdef CONFIG_PPC_BOOK3S
702_GLOBAL(ret_from_fork_scv)
703	bl	schedule_tail
704	REST_NVGPRS(r1)
705	li	r3,0	/* fork() return value */
706	b	.Lsyscall_vectored_common_exit
707#endif
708
709_GLOBAL(ret_from_fork)
710	bl	schedule_tail
711	REST_NVGPRS(r1)
712	li	r3,0	/* fork() return value */
713	b	.Lsyscall_exit
714
715_GLOBAL(ret_from_kernel_thread)
716	bl	schedule_tail
717	REST_NVGPRS(r1)
718	mtctr	r14
719	mr	r3,r15
720#ifdef CONFIG_PPC64_ELF_ABI_V2
721	mr	r12,r14
722#endif
723	bctrl
724	li	r3,0
725	b	.Lsyscall_exit
726