xref: /openbmc/linux/arch/powerpc/kernel/vector.S (revision ddc141e5)
1/* SPDX-License-Identifier: GPL-2.0 */
2#include <asm/processor.h>
3#include <asm/ppc_asm.h>
4#include <asm/reg.h>
5#include <asm/asm-offsets.h>
6#include <asm/cputable.h>
7#include <asm/thread_info.h>
8#include <asm/page.h>
9#include <asm/ptrace.h>
10#include <asm/export.h>
11
12/*
13 * Load state from memory into VMX registers including VSCR.
14 * Assumes the caller has enabled VMX in the MSR.
15 */
16_GLOBAL(load_vr_state)
17	li	r4,VRSTATE_VSCR
18	lvx	v0,r4,r3
19	mtvscr	v0
20	REST_32VRS(0,r4,r3)
21	blr
22EXPORT_SYMBOL(load_vr_state)
23
24/*
25 * Store VMX state into memory, including VSCR.
26 * Assumes the caller has enabled VMX in the MSR.
27 */
28_GLOBAL(store_vr_state)
29	SAVE_32VRS(0, r4, r3)
30	mfvscr	v0
31	li	r4, VRSTATE_VSCR
32	stvx	v0, r4, r3
33	blr
34EXPORT_SYMBOL(store_vr_state)
35
36/*
37 * Disable VMX for the task which had it previously,
38 * and save its vector registers in its thread_struct.
39 * Enables the VMX for use in the kernel on return.
40 * On SMP we know the VMX is free, since we give it up every
41 * switch (ie, no lazy save of the vector registers).
42 *
43 * Note that on 32-bit this can only use registers that will be
44 * restored by fast_exception_return, i.e. r3 - r6, r10 and r11.
45 */
46_GLOBAL(load_up_altivec)
47	mfmsr	r5			/* grab the current MSR */
48	oris	r5,r5,MSR_VEC@h
49	MTMSRD(r5)			/* enable use of AltiVec now */
50	isync
51
52	/*
53	 * While userspace in general ignores VRSAVE, glibc uses it as a boolean
54	 * to optimise userspace context save/restore. Whenever we take an
55	 * altivec unavailable exception we must set VRSAVE to something non
56	 * zero. Set it to all 1s. See also the programming note in the ISA.
57	 */
58	mfspr	r4,SPRN_VRSAVE
59	cmpwi	0,r4,0
60	bne+	1f
61	li	r4,-1
62	mtspr	SPRN_VRSAVE,r4
631:
64	/* enable use of VMX after return */
65#ifdef CONFIG_PPC32
66	mfspr	r5,SPRN_SPRG_THREAD		/* current task's THREAD (phys) */
67	oris	r9,r9,MSR_VEC@h
68#else
69	ld	r4,PACACURRENT(r13)
70	addi	r5,r4,THREAD		/* Get THREAD */
71	oris	r12,r12,MSR_VEC@h
72	std	r12,_MSR(r1)
73#endif
74	/* Don't care if r4 overflows, this is desired behaviour */
75	lbz	r4,THREAD_LOAD_VEC(r5)
76	addi	r4,r4,1
77	stb	r4,THREAD_LOAD_VEC(r5)
78	addi	r6,r5,THREAD_VRSTATE
79	li	r4,1
80	li	r10,VRSTATE_VSCR
81	stw	r4,THREAD_USED_VR(r5)
82	lvx	v0,r10,r6
83	mtvscr	v0
84	REST_32VRS(0,r4,r6)
85	/* restore registers and return */
86	blr
87
88/*
89 * save_altivec(tsk)
90 * Save the vector registers to its thread_struct
91 */
92_GLOBAL(save_altivec)
93	addi	r3,r3,THREAD		/* want THREAD of task */
94	PPC_LL	r7,THREAD_VRSAVEAREA(r3)
95	PPC_LL	r5,PT_REGS(r3)
96	PPC_LCMPI	0,r7,0
97	bne	2f
98	addi	r7,r3,THREAD_VRSTATE
992:	SAVE_32VRS(0,r4,r7)
100	mfvscr	v0
101	li	r4,VRSTATE_VSCR
102	stvx	v0,r4,r7
103	blr
104
105#ifdef CONFIG_VSX
106
107#ifdef CONFIG_PPC32
108#error This asm code isn't ready for 32-bit kernels
109#endif
110
111/*
112 * load_up_vsx(unused, unused, tsk)
113 * Disable VSX for the task which had it previously,
114 * and save its vector registers in its thread_struct.
115 * Reuse the fp and vsx saves, but first check to see if they have
116 * been saved already.
117 */
118_GLOBAL(load_up_vsx)
119/* Load FP and VSX registers if they haven't been done yet */
120	andi.	r5,r12,MSR_FP
121	beql+	load_up_fpu		/* skip if already loaded */
122	andis.	r5,r12,MSR_VEC@h
123	beql+	load_up_altivec		/* skip if already loaded */
124
125	ld	r4,PACACURRENT(r13)
126	addi	r4,r4,THREAD		/* Get THREAD */
127	li	r6,1
128	stw	r6,THREAD_USED_VSR(r4) /* ... also set thread used vsr */
129	/* enable use of VSX after return */
130	oris	r12,r12,MSR_VSX@h
131	std	r12,_MSR(r1)
132	b	fast_exception_return
133
134#endif /* CONFIG_VSX */
135
136
137/*
138 * The routines below are in assembler so we can closely control the
139 * usage of floating-point registers.  These routines must be called
140 * with preempt disabled.
141 */
142#ifdef CONFIG_PPC32
143	.data
144fpzero:
145	.long	0
146fpone:
147	.long	0x3f800000	/* 1.0 in single-precision FP */
148fphalf:
149	.long	0x3f000000	/* 0.5 in single-precision FP */
150
151#define LDCONST(fr, name)	\
152	lis	r11,name@ha;	\
153	lfs	fr,name@l(r11)
154#else
155
156	.section ".toc","aw"
157fpzero:
158	.tc	FD_0_0[TC],0
159fpone:
160	.tc	FD_3ff00000_0[TC],0x3ff0000000000000	/* 1.0 */
161fphalf:
162	.tc	FD_3fe00000_0[TC],0x3fe0000000000000	/* 0.5 */
163
164#define LDCONST(fr, name)	\
165	lfd	fr,name@toc(r2)
166#endif
167
168	.text
169/*
170 * Internal routine to enable floating point and set FPSCR to 0.
171 * Don't call it from C; it doesn't use the normal calling convention.
172 */
173fpenable:
174#ifdef CONFIG_PPC32
175	stwu	r1,-64(r1)
176#else
177	stdu	r1,-64(r1)
178#endif
179	mfmsr	r10
180	ori	r11,r10,MSR_FP
181	mtmsr	r11
182	isync
183	stfd	fr0,24(r1)
184	stfd	fr1,16(r1)
185	stfd	fr31,8(r1)
186	LDCONST(fr1, fpzero)
187	mffs	fr31
188	MTFSF_L(fr1)
189	blr
190
191fpdisable:
192	mtlr	r12
193	MTFSF_L(fr31)
194	lfd	fr31,8(r1)
195	lfd	fr1,16(r1)
196	lfd	fr0,24(r1)
197	mtmsr	r10
198	isync
199	addi	r1,r1,64
200	blr
201
202/*
203 * Vector add, floating point.
204 */
205_GLOBAL(vaddfp)
206	mflr	r12
207	bl	fpenable
208	li	r0,4
209	mtctr	r0
210	li	r6,0
2111:	lfsx	fr0,r4,r6
212	lfsx	fr1,r5,r6
213	fadds	fr0,fr0,fr1
214	stfsx	fr0,r3,r6
215	addi	r6,r6,4
216	bdnz	1b
217	b	fpdisable
218
219/*
220 * Vector subtract, floating point.
221 */
222_GLOBAL(vsubfp)
223	mflr	r12
224	bl	fpenable
225	li	r0,4
226	mtctr	r0
227	li	r6,0
2281:	lfsx	fr0,r4,r6
229	lfsx	fr1,r5,r6
230	fsubs	fr0,fr0,fr1
231	stfsx	fr0,r3,r6
232	addi	r6,r6,4
233	bdnz	1b
234	b	fpdisable
235
236/*
237 * Vector multiply and add, floating point.
238 */
239_GLOBAL(vmaddfp)
240	mflr	r12
241	bl	fpenable
242	stfd	fr2,32(r1)
243	li	r0,4
244	mtctr	r0
245	li	r7,0
2461:	lfsx	fr0,r4,r7
247	lfsx	fr1,r5,r7
248	lfsx	fr2,r6,r7
249	fmadds	fr0,fr0,fr2,fr1
250	stfsx	fr0,r3,r7
251	addi	r7,r7,4
252	bdnz	1b
253	lfd	fr2,32(r1)
254	b	fpdisable
255
256/*
257 * Vector negative multiply and subtract, floating point.
258 */
259_GLOBAL(vnmsubfp)
260	mflr	r12
261	bl	fpenable
262	stfd	fr2,32(r1)
263	li	r0,4
264	mtctr	r0
265	li	r7,0
2661:	lfsx	fr0,r4,r7
267	lfsx	fr1,r5,r7
268	lfsx	fr2,r6,r7
269	fnmsubs	fr0,fr0,fr2,fr1
270	stfsx	fr0,r3,r7
271	addi	r7,r7,4
272	bdnz	1b
273	lfd	fr2,32(r1)
274	b	fpdisable
275
276/*
277 * Vector reciprocal estimate.  We just compute 1.0/x.
278 * r3 -> destination, r4 -> source.
279 */
280_GLOBAL(vrefp)
281	mflr	r12
282	bl	fpenable
283	li	r0,4
284	LDCONST(fr1, fpone)
285	mtctr	r0
286	li	r6,0
2871:	lfsx	fr0,r4,r6
288	fdivs	fr0,fr1,fr0
289	stfsx	fr0,r3,r6
290	addi	r6,r6,4
291	bdnz	1b
292	b	fpdisable
293
294/*
295 * Vector reciprocal square-root estimate, floating point.
296 * We use the frsqrte instruction for the initial estimate followed
297 * by 2 iterations of Newton-Raphson to get sufficient accuracy.
298 * r3 -> destination, r4 -> source.
299 */
300_GLOBAL(vrsqrtefp)
301	mflr	r12
302	bl	fpenable
303	stfd	fr2,32(r1)
304	stfd	fr3,40(r1)
305	stfd	fr4,48(r1)
306	stfd	fr5,56(r1)
307	li	r0,4
308	LDCONST(fr4, fpone)
309	LDCONST(fr5, fphalf)
310	mtctr	r0
311	li	r6,0
3121:	lfsx	fr0,r4,r6
313	frsqrte	fr1,fr0		/* r = frsqrte(s) */
314	fmuls	fr3,fr1,fr0	/* r * s */
315	fmuls	fr2,fr1,fr5	/* r * 0.5 */
316	fnmsubs	fr3,fr1,fr3,fr4	/* 1 - s * r * r */
317	fmadds	fr1,fr2,fr3,fr1	/* r = r + 0.5 * r * (1 - s * r * r) */
318	fmuls	fr3,fr1,fr0	/* r * s */
319	fmuls	fr2,fr1,fr5	/* r * 0.5 */
320	fnmsubs	fr3,fr1,fr3,fr4	/* 1 - s * r * r */
321	fmadds	fr1,fr2,fr3,fr1	/* r = r + 0.5 * r * (1 - s * r * r) */
322	stfsx	fr1,r3,r6
323	addi	r6,r6,4
324	bdnz	1b
325	lfd	fr5,56(r1)
326	lfd	fr4,48(r1)
327	lfd	fr3,40(r1)
328	lfd	fr2,32(r1)
329	b	fpdisable
330