xref: /openbmc/linux/arch/powerpc/kernel/vector.S (revision a44e4f3a) 
1/* SPDX-License-Identifier: GPL-2.0 */
2#include <asm/processor.h>
3#include <asm/ppc_asm.h>
4#include <asm/reg.h>
5#include <asm/asm-offsets.h>
6#include <asm/cputable.h>
7#include <asm/thread_info.h>
8#include <asm/page.h>
9#include <asm/ptrace.h>
10#include <asm/export.h>
11#include <asm/asm-compat.h>
12
13/*
14 * Load state from memory into VMX registers including VSCR.
15 * Assumes the caller has enabled VMX in the MSR.
16 */
17_GLOBAL(load_vr_state)
18	li	r4,VRSTATE_VSCR
19	lvx	v0,r4,r3
20	mtvscr	v0
21	REST_32VRS(0,r4,r3)
22	blr
23EXPORT_SYMBOL(load_vr_state)
24_ASM_NOKPROBE_SYMBOL(load_vr_state); /* used by restore_math */
25
26/*
27 * Store VMX state into memory, including VSCR.
28 * Assumes the caller has enabled VMX in the MSR.
29 */
30_GLOBAL(store_vr_state)
31	SAVE_32VRS(0, r4, r3)
32	mfvscr	v0
33	li	r4, VRSTATE_VSCR
34	stvx	v0, r4, r3
35	blr
36EXPORT_SYMBOL(store_vr_state)
37
38/*
39 * Disable VMX for the task which had it previously,
40 * and save its vector registers in its thread_struct.
41 * Enables the VMX for use in the kernel on return.
42 * On SMP we know the VMX is free, since we give it up every
43 * switch (ie, no lazy save of the vector registers).
44 *
45 * Note that on 32-bit this can only use registers that will be
46 * restored by fast_exception_return, i.e. r3 - r6, r10 and r11.
47 */
48_GLOBAL(load_up_altivec)
49	mfmsr	r5			/* grab the current MSR */
50	oris	r5,r5,MSR_VEC@h
51	MTMSRD(r5)			/* enable use of AltiVec now */
52	isync
53
54	/*
55	 * While userspace in general ignores VRSAVE, glibc uses it as a boolean
56	 * to optimise userspace context save/restore. Whenever we take an
57	 * altivec unavailable exception we must set VRSAVE to something non
58	 * zero. Set it to all 1s. See also the programming note in the ISA.
59	 */
60	mfspr	r4,SPRN_VRSAVE
61	cmpwi	0,r4,0
62	bne+	1f
63	li	r4,-1
64	mtspr	SPRN_VRSAVE,r4
651:
66	/* enable use of VMX after return */
67#ifdef CONFIG_PPC32
68	mfspr	r5,SPRN_SPRG_THREAD		/* current task's THREAD (phys) */
69	oris	r9,r9,MSR_VEC@h
70#else
71	ld	r4,PACACURRENT(r13)
72	addi	r5,r4,THREAD		/* Get THREAD */
73	oris	r12,r12,MSR_VEC@h
74	std	r12,_MSR(r1)
75#endif
76	/* Don't care if r4 overflows, this is desired behaviour */
77	lbz	r4,THREAD_LOAD_VEC(r5)
78	addi	r4,r4,1
79	stb	r4,THREAD_LOAD_VEC(r5)
80	addi	r6,r5,THREAD_VRSTATE
81	li	r4,1
82	li	r10,VRSTATE_VSCR
83	stw	r4,THREAD_USED_VR(r5)
84	lvx	v0,r10,r6
85	mtvscr	v0
86	REST_32VRS(0,r4,r6)
87	/* restore registers and return */
88	blr
89
90/*
91 * save_altivec(tsk)
92 * Save the vector registers to its thread_struct
93 */
94_GLOBAL(save_altivec)
95	addi	r3,r3,THREAD		/* want THREAD of task */
96	PPC_LL	r7,THREAD_VRSAVEAREA(r3)
97	PPC_LL	r5,PT_REGS(r3)
98	PPC_LCMPI	0,r7,0
99	bne	2f
100	addi	r7,r3,THREAD_VRSTATE
1012:	SAVE_32VRS(0,r4,r7)
102	mfvscr	v0
103	li	r4,VRSTATE_VSCR
104	stvx	v0,r4,r7
105	blr
106
107#ifdef CONFIG_VSX
108
109#ifdef CONFIG_PPC32
110#error This asm code isn't ready for 32-bit kernels
111#endif
112
113/*
114 * load_up_vsx(unused, unused, tsk)
115 * Disable VSX for the task which had it previously,
116 * and save its vector registers in its thread_struct.
117 * Reuse the fp and vsx saves, but first check to see if they have
118 * been saved already.
119 */
120_GLOBAL(load_up_vsx)
121/* Load FP and VSX registers if they haven't been done yet */
122	andi.	r5,r12,MSR_FP
123	beql+	load_up_fpu		/* skip if already loaded */
124	andis.	r5,r12,MSR_VEC@h
125	beql+	load_up_altivec		/* skip if already loaded */
126
127	ld	r4,PACACURRENT(r13)
128	addi	r4,r4,THREAD		/* Get THREAD */
129	li	r6,1
130	stw	r6,THREAD_USED_VSR(r4) /* ... also set thread used vsr */
131	/* enable use of VSX after return */
132	oris	r12,r12,MSR_VSX@h
133	std	r12,_MSR(r1)
134	b	fast_exception_return
135
136#endif /* CONFIG_VSX */
137
138
139/*
140 * The routines below are in assembler so we can closely control the
141 * usage of floating-point registers.  These routines must be called
142 * with preempt disabled.
143 */
144#ifdef CONFIG_PPC32
145	.data
146fpzero:
147	.long	0
148fpone:
149	.long	0x3f800000	/* 1.0 in single-precision FP */
150fphalf:
151	.long	0x3f000000	/* 0.5 in single-precision FP */
152
153#define LDCONST(fr, name)	\
154	lis	r11,name@ha;	\
155	lfs	fr,name@l(r11)
156#else
157
158	.section ".toc","aw"
159fpzero:
160	.tc	FD_0_0[TC],0
161fpone:
162	.tc	FD_3ff00000_0[TC],0x3ff0000000000000	/* 1.0 */
163fphalf:
164	.tc	FD_3fe00000_0[TC],0x3fe0000000000000	/* 0.5 */
165
166#define LDCONST(fr, name)	\
167	lfd	fr,name@toc(r2)
168#endif
169
170	.text
171/*
172 * Internal routine to enable floating point and set FPSCR to 0.
173 * Don't call it from C; it doesn't use the normal calling convention.
174 */
175fpenable:
176#ifdef CONFIG_PPC32
177	stwu	r1,-64(r1)
178#else
179	stdu	r1,-64(r1)
180#endif
181	mfmsr	r10
182	ori	r11,r10,MSR_FP
183	mtmsr	r11
184	isync
185	stfd	fr0,24(r1)
186	stfd	fr1,16(r1)
187	stfd	fr31,8(r1)
188	LDCONST(fr1, fpzero)
189	mffs	fr31
190	MTFSF_L(fr1)
191	blr
192
193fpdisable:
194	mtlr	r12
195	MTFSF_L(fr31)
196	lfd	fr31,8(r1)
197	lfd	fr1,16(r1)
198	lfd	fr0,24(r1)
199	mtmsr	r10
200	isync
201	addi	r1,r1,64
202	blr
203
204/*
205 * Vector add, floating point.
206 */
207_GLOBAL(vaddfp)
208	mflr	r12
209	bl	fpenable
210	li	r0,4
211	mtctr	r0
212	li	r6,0
2131:	lfsx	fr0,r4,r6
214	lfsx	fr1,r5,r6
215	fadds	fr0,fr0,fr1
216	stfsx	fr0,r3,r6
217	addi	r6,r6,4
218	bdnz	1b
219	b	fpdisable
220
221/*
222 * Vector subtract, floating point.
223 */
224_GLOBAL(vsubfp)
225	mflr	r12
226	bl	fpenable
227	li	r0,4
228	mtctr	r0
229	li	r6,0
2301:	lfsx	fr0,r4,r6
231	lfsx	fr1,r5,r6
232	fsubs	fr0,fr0,fr1
233	stfsx	fr0,r3,r6
234	addi	r6,r6,4
235	bdnz	1b
236	b	fpdisable
237
238/*
239 * Vector multiply and add, floating point.
240 */
241_GLOBAL(vmaddfp)
242	mflr	r12
243	bl	fpenable
244	stfd	fr2,32(r1)
245	li	r0,4
246	mtctr	r0
247	li	r7,0
2481:	lfsx	fr0,r4,r7
249	lfsx	fr1,r5,r7
250	lfsx	fr2,r6,r7
251	fmadds	fr0,fr0,fr2,fr1
252	stfsx	fr0,r3,r7
253	addi	r7,r7,4
254	bdnz	1b
255	lfd	fr2,32(r1)
256	b	fpdisable
257
258/*
259 * Vector negative multiply and subtract, floating point.
260 */
261_GLOBAL(vnmsubfp)
262	mflr	r12
263	bl	fpenable
264	stfd	fr2,32(r1)
265	li	r0,4
266	mtctr	r0
267	li	r7,0
2681:	lfsx	fr0,r4,r7
269	lfsx	fr1,r5,r7
270	lfsx	fr2,r6,r7
271	fnmsubs	fr0,fr0,fr2,fr1
272	stfsx	fr0,r3,r7
273	addi	r7,r7,4
274	bdnz	1b
275	lfd	fr2,32(r1)
276	b	fpdisable
277
278/*
279 * Vector reciprocal estimate.  We just compute 1.0/x.
280 * r3 -> destination, r4 -> source.
281 */
282_GLOBAL(vrefp)
283	mflr	r12
284	bl	fpenable
285	li	r0,4
286	LDCONST(fr1, fpone)
287	mtctr	r0
288	li	r6,0
2891:	lfsx	fr0,r4,r6
290	fdivs	fr0,fr1,fr0
291	stfsx	fr0,r3,r6
292	addi	r6,r6,4
293	bdnz	1b
294	b	fpdisable
295
296/*
297 * Vector reciprocal square-root estimate, floating point.
298 * We use the frsqrte instruction for the initial estimate followed
299 * by 2 iterations of Newton-Raphson to get sufficient accuracy.
300 * r3 -> destination, r4 -> source.
301 */
302_GLOBAL(vrsqrtefp)
303	mflr	r12
304	bl	fpenable
305	stfd	fr2,32(r1)
306	stfd	fr3,40(r1)
307	stfd	fr4,48(r1)
308	stfd	fr5,56(r1)
309	li	r0,4
310	LDCONST(fr4, fpone)
311	LDCONST(fr5, fphalf)
312	mtctr	r0
313	li	r6,0
3141:	lfsx	fr0,r4,r6
315	frsqrte	fr1,fr0		/* r = frsqrte(s) */
316	fmuls	fr3,fr1,fr0	/* r * s */
317	fmuls	fr2,fr1,fr5	/* r * 0.5 */
318	fnmsubs	fr3,fr1,fr3,fr4	/* 1 - s * r * r */
319	fmadds	fr1,fr2,fr3,fr1	/* r = r + 0.5 * r * (1 - s * r * r) */
320	fmuls	fr3,fr1,fr0	/* r * s */
321	fmuls	fr2,fr1,fr5	/* r * 0.5 */
322	fnmsubs	fr3,fr1,fr3,fr4	/* 1 - s * r * r */
323	fmadds	fr1,fr2,fr3,fr1	/* r = r + 0.5 * r * (1 - s * r * r) */
324	stfsx	fr1,r3,r6
325	addi	r6,r6,4
326	bdnz	1b
327	lfd	fr5,56(r1)
328	lfd	fr4,48(r1)
329	lfd	fr3,40(r1)
330	lfd	fr2,32(r1)
331	b	fpdisable
332